Repository: MyCATApache/Mycat-Server Branch: main Commit: 243539fb74bb Files: 1154 Total size: 4.9 MB Directory structure: gitextract_61vp_fxc/ ├── .github/ │ └── ISSUE_TEMPLATE/ │ ├── bug-report.md │ ├── new_feture.md │ └── others.md ├── .gitignore ├── .travis.yml ├── .vscode/ │ ├── launch.json │ └── settings.json ├── Dockerfile ├── LICENSE ├── README.md ├── README_Chinese.md ├── catlet/ │ └── readme.txt ├── eclipse-java-google-style.xml ├── intellij-java-google-style.xml ├── pom.xml ├── src/ │ ├── main/ │ │ ├── assembly/ │ │ │ ├── assembly-linux.xml │ │ │ ├── assembly-mac.xml │ │ │ ├── assembly-solaris.xml │ │ │ ├── assembly-testtool.xml │ │ │ ├── assembly-unix.xml │ │ │ ├── assembly-win.xml │ │ │ ├── conf/ │ │ │ │ └── log4j2.xml │ │ │ └── testtool/ │ │ │ ├── test_globalseq_insert_perf.bat │ │ │ ├── test_globaltable_insert_perf.bat │ │ │ ├── test_globaltable_insert_perf.sh │ │ │ ├── test_stand_insert_perf.bat │ │ │ ├── test_stand_insert_perf.sh │ │ │ ├── test_stand_merge_sel_perf.bat │ │ │ ├── test_stand_merge_sel_perf.sh │ │ │ ├── test_stand_select_perf.bat │ │ │ ├── test_stand_select_perf.sh │ │ │ ├── test_stand_update_perf.bat │ │ │ └── test_stand_update_perf.sh │ │ ├── java/ │ │ │ └── io/ │ │ │ └── mycat/ │ │ │ ├── MycatServer.java │ │ │ ├── MycatShutdown.java │ │ │ ├── MycatStartup.java │ │ │ ├── backend/ │ │ │ │ ├── BackendConnection.java │ │ │ │ ├── ConMap.java │ │ │ │ ├── ConQueue.java │ │ │ │ ├── ConnectionMeta.java │ │ │ │ ├── datasource/ │ │ │ │ │ ├── PhysicalDBNode.java │ │ │ │ │ ├── PhysicalDBPool.java │ │ │ │ │ └── PhysicalDatasource.java │ │ │ │ ├── heartbeat/ │ │ │ │ │ ├── ConsistenCollectHandler.java │ │ │ │ │ ├── DBHeartbeat.java │ │ │ │ │ ├── MySQLConsistencyChecker.java │ │ │ │ │ ├── MySQLConsistencyCheckerHandler.java │ │ │ │ │ ├── MySQLConsistencyHelper.java │ │ │ │ │ ├── MySQLDetector.java │ │ │ │ │ ├── MySQLHeartbeat.java │ │ │ │ │ └── zkprocess/ │ │ │ │ │ ├── ManageHeartBeatChange.java │ │ │ │ │ ├── MycatLeaderLatch.java │ │ │ │ │ └── SwitchStatueToZK.java │ │ │ │ ├── jdbc/ │ │ │ │ │ ├── JDBCConnection.java │ │ │ │ │ ├── JDBCDatasource.java │ │ │ │ │ ├── JDBCHeartbeat.java │ │ │ │ │ ├── ShowVariables.java │ │ │ │ │ ├── mongodb/ │ │ │ │ │ │ ├── DriverPropertyInfoHelper.java │ │ │ │ │ │ ├── MongoClientPropertyHelper.java │ │ │ │ │ │ ├── MongoConnection.java │ │ │ │ │ │ ├── MongoData.java │ │ │ │ │ │ ├── MongoDriver.java │ │ │ │ │ │ ├── MongoEmbeddedObjectProcessor.java │ │ │ │ │ │ ├── MongoPreparedStatement.java │ │ │ │ │ │ ├── MongoResultSet.java │ │ │ │ │ │ ├── MongoResultSetMetaData.java │ │ │ │ │ │ ├── MongoSQLException.java │ │ │ │ │ │ ├── MongoSQLParser.java │ │ │ │ │ │ ├── MongoStatement.java │ │ │ │ │ │ └── StringUtils.java │ │ │ │ │ └── sequoiadb/ │ │ │ │ │ ├── DriverPropertyInfoHelper.java │ │ │ │ │ ├── SequoiaConnection.java │ │ │ │ │ ├── SequoiaData.java │ │ │ │ │ ├── SequoiaDriver.java │ │ │ │ │ ├── SequoiaPreparedStatement.java │ │ │ │ │ ├── SequoiaResultSet.java │ │ │ │ │ ├── SequoiaResultSetMetaData.java │ │ │ │ │ ├── SequoiaSQLException.java │ │ │ │ │ ├── SequoiaSQLParser.java │ │ │ │ │ ├── SequoiaStatement.java │ │ │ │ │ └── StringUtils.java │ │ │ │ ├── loadbalance/ │ │ │ │ │ ├── LeastActiveLoadBalance.java │ │ │ │ │ ├── LoadBalance.java │ │ │ │ │ ├── RandomLoadBalance.java │ │ │ │ │ └── WeightedRoundRobinLoadBalance.java │ │ │ │ ├── mysql/ │ │ │ │ │ ├── BindValue.java │ │ │ │ │ ├── BindValueUtil.java │ │ │ │ │ ├── BufferUtil.java │ │ │ │ │ ├── ByteUtil.java │ │ │ │ │ ├── CharsetUtil.java │ │ │ │ │ ├── DataType.java │ │ │ │ │ ├── LoadDataUtil.java │ │ │ │ │ ├── MySQLMessage.java │ │ │ │ │ ├── PacketUtil.java │ │ │ │ │ ├── PreparedStatement.java │ │ │ │ │ ├── SecurityUtil.java │ │ │ │ │ ├── StreamUtil.java │ │ │ │ │ ├── listener/ │ │ │ │ │ │ ├── DefaultSqlExecuteStageListener.java │ │ │ │ │ │ ├── SqlExecuteStage.java │ │ │ │ │ │ └── SqlExecuteStageListener.java │ │ │ │ │ ├── nio/ │ │ │ │ │ │ ├── MySQLConnection.java │ │ │ │ │ │ ├── MySQLConnectionAuthenticator.java │ │ │ │ │ │ ├── MySQLConnectionFactory.java │ │ │ │ │ │ ├── MySQLConnectionHandler.java │ │ │ │ │ │ ├── MySQLDataSource.java │ │ │ │ │ │ └── handler/ │ │ │ │ │ │ ├── CommitNodeHandler.java │ │ │ │ │ │ ├── ConnectionHeartBeatHandler.java │ │ │ │ │ │ ├── DelegateResponseHandler.java │ │ │ │ │ │ ├── FetchStoreNodeOfChildTableHandler.java │ │ │ │ │ │ ├── GetConnectionHandler.java │ │ │ │ │ │ ├── JDBCFetchStoreNodeOfChildTableHandler.java │ │ │ │ │ │ ├── KillConnectionHandler.java │ │ │ │ │ │ ├── LoadDataResponseHandler.java │ │ │ │ │ │ ├── LockTablesHandler.java │ │ │ │ │ │ ├── MiddlerQueryResultHandler.java │ │ │ │ │ │ ├── MiddlerResultHandler.java │ │ │ │ │ │ ├── MultiNodeCoordinator.java │ │ │ │ │ │ ├── MultiNodeHandler.java │ │ │ │ │ │ ├── MultiNodeQueryHandler.java │ │ │ │ │ │ ├── NewConnectionRespHandler.java │ │ │ │ │ │ ├── PrepareRequestHandler.java │ │ │ │ │ │ ├── ResponseHandler.java │ │ │ │ │ │ ├── RollbackNodeHandler.java │ │ │ │ │ │ ├── RollbackReleaseHandler.java │ │ │ │ │ │ ├── SecondHandler.java │ │ │ │ │ │ ├── SecondQueryHandler.java │ │ │ │ │ │ ├── SimpleLogHandler.java │ │ │ │ │ │ ├── SingleNodeHandler.java │ │ │ │ │ │ ├── Terminatable.java │ │ │ │ │ │ └── UnLockTablesHandler.java │ │ │ │ │ └── xa/ │ │ │ │ │ ├── CoordinatorLogEntry.java │ │ │ │ │ ├── Deserializer.java │ │ │ │ │ ├── LogFileLock.java │ │ │ │ │ ├── ParticipantLogEntry.java │ │ │ │ │ ├── Serializer.java │ │ │ │ │ ├── TxState.java │ │ │ │ │ ├── VersionedFile.java │ │ │ │ │ ├── XACommitCallback.java │ │ │ │ │ ├── XARollbackCallback.java │ │ │ │ │ └── recovery/ │ │ │ │ │ ├── DeserialisationException.java │ │ │ │ │ ├── LogException.java │ │ │ │ │ ├── LogReadException.java │ │ │ │ │ ├── LogWriteException.java │ │ │ │ │ ├── Repository.java │ │ │ │ │ └── impl/ │ │ │ │ │ ├── FileSystemRepository.java │ │ │ │ │ └── InMemoryRepository.java │ │ │ │ └── postgresql/ │ │ │ │ ├── PostgreSQLBackendConnection.java │ │ │ │ ├── PostgreSQLBackendConnectionFactory.java │ │ │ │ ├── PostgreSQLBackendConnectionHandler.java │ │ │ │ ├── PostgreSQLDataSource.java │ │ │ │ ├── heartbeat/ │ │ │ │ │ ├── PostgreSQLDetector.java │ │ │ │ │ └── PostgreSQLHeartbeat.java │ │ │ │ ├── package-info.java │ │ │ │ ├── packet/ │ │ │ │ │ ├── AuthenticationPacket.java │ │ │ │ │ ├── BackendKeyData.java │ │ │ │ │ ├── Bind.java │ │ │ │ │ ├── BindComplete.java │ │ │ │ │ ├── CancelRequest.java │ │ │ │ │ ├── CommandComplete.java │ │ │ │ │ ├── CopyInResponse.java │ │ │ │ │ ├── CopyOutResponse.java │ │ │ │ │ ├── DataRow.java │ │ │ │ │ ├── EmptyQueryResponse.java │ │ │ │ │ ├── ErrorResponse.java │ │ │ │ │ ├── NoticeResponse.java │ │ │ │ │ ├── NotificationResponse.java │ │ │ │ │ ├── ParameterDescription.java │ │ │ │ │ ├── ParameterStatus.java │ │ │ │ │ ├── Parse.java │ │ │ │ │ ├── ParseComplete.java │ │ │ │ │ ├── PasswordMessage.java │ │ │ │ │ ├── PostgreSQLPacket.java │ │ │ │ │ ├── Query.java │ │ │ │ │ ├── ReadyForQuery.java │ │ │ │ │ ├── RowDescription.java │ │ │ │ │ ├── SSLRequest.java │ │ │ │ │ ├── StartupMessage.java │ │ │ │ │ └── Terminate.java │ │ │ │ └── utils/ │ │ │ │ ├── MD5Digest.java │ │ │ │ ├── PIOUtils.java │ │ │ │ ├── PacketUtils.java │ │ │ │ ├── PgPacketApaterUtils.java │ │ │ │ └── PgSqlApaterUtils.java │ │ │ ├── buffer/ │ │ │ │ ├── BufferArray.java │ │ │ │ ├── BufferPool.java │ │ │ │ ├── ByteBufferArena.java │ │ │ │ ├── ByteBufferChunk.java │ │ │ │ ├── ByteBufferChunkList.java │ │ │ │ ├── ByteBufferPage.java │ │ │ │ ├── DirectByteBufferPool.java │ │ │ │ ├── MyCatMemoryAllocator.java │ │ │ │ └── NettyBufferPool.java │ │ │ ├── cache/ │ │ │ │ ├── CachePool.java │ │ │ │ ├── CachePoolFactory.java │ │ │ │ ├── CacheService.java │ │ │ │ ├── CacheStatic.java │ │ │ │ ├── DefaultLayedCachePool.java │ │ │ │ ├── LayerCachePool.java │ │ │ │ ├── MysqlDataSetCache.java │ │ │ │ ├── MysqlDataSetService.java │ │ │ │ ├── impl/ │ │ │ │ │ ├── EnchachePooFactory.java │ │ │ │ │ ├── EnchachePool.java │ │ │ │ │ ├── LevelDBCachePooFactory.java │ │ │ │ │ ├── LevelDBPool.java │ │ │ │ │ ├── MapDBCachePooFactory.java │ │ │ │ │ └── MapDBCachePool.java │ │ │ │ └── index/ │ │ │ │ └── Shard.java │ │ │ ├── catlets/ │ │ │ │ ├── Catlet.java │ │ │ │ ├── JoinParser.java │ │ │ │ ├── ShareJoin.java │ │ │ │ └── TableFilter.java │ │ │ ├── config/ │ │ │ │ ├── Alarms.java │ │ │ │ ├── Capabilities.java │ │ │ │ ├── ConfigInitializer.java │ │ │ │ ├── ErrorCode.java │ │ │ │ ├── Fields.java │ │ │ │ ├── Isolations.java │ │ │ │ ├── MycatCluster.java │ │ │ │ ├── MycatConfig.java │ │ │ │ ├── MycatNode.java │ │ │ │ ├── MycatPrivileges.java │ │ │ │ ├── Versions.java │ │ │ │ ├── Versions.template │ │ │ │ ├── classloader/ │ │ │ │ │ ├── DynaClassLoader.java │ │ │ │ │ └── JarLoader.java │ │ │ │ ├── loader/ │ │ │ │ │ ├── ConfigLoader.java │ │ │ │ │ ├── SchemaLoader.java │ │ │ │ │ ├── console/ │ │ │ │ │ │ └── ZookeeperPath.java │ │ │ │ │ ├── xml/ │ │ │ │ │ │ ├── XMLConfigLoader.java │ │ │ │ │ │ ├── XMLRuleLoader.java │ │ │ │ │ │ ├── XMLSchemaLoader.java │ │ │ │ │ │ └── XMLServerLoader.java │ │ │ │ │ └── zkprocess/ │ │ │ │ │ ├── comm/ │ │ │ │ │ │ ├── NotiflyService.java │ │ │ │ │ │ ├── ZkConfig.java │ │ │ │ │ │ ├── ZkParamCfg.java │ │ │ │ │ │ └── ZookeeperProcessListen.java │ │ │ │ │ ├── console/ │ │ │ │ │ │ ├── ParseParamEnum.java │ │ │ │ │ │ └── ZkNofiflyCfg.java │ │ │ │ │ ├── entity/ │ │ │ │ │ │ ├── Named.java │ │ │ │ │ │ ├── Propertied.java │ │ │ │ │ │ ├── Property.java │ │ │ │ │ │ ├── Rules.java │ │ │ │ │ │ ├── Schemas.java │ │ │ │ │ │ ├── Server.java │ │ │ │ │ │ ├── cache/ │ │ │ │ │ │ │ ├── CacheInfo.java │ │ │ │ │ │ │ └── Ehcache.java │ │ │ │ │ │ ├── package-info.java │ │ │ │ │ │ ├── rule/ │ │ │ │ │ │ │ ├── function/ │ │ │ │ │ │ │ │ └── Function.java │ │ │ │ │ │ │ └── tablerule/ │ │ │ │ │ │ │ ├── Rule.java │ │ │ │ │ │ │ └── TableRule.java │ │ │ │ │ │ ├── schema/ │ │ │ │ │ │ │ ├── datahost/ │ │ │ │ │ │ │ │ ├── DataHost.java │ │ │ │ │ │ │ │ ├── ReadHost.java │ │ │ │ │ │ │ │ └── WriteHost.java │ │ │ │ │ │ │ ├── datanode/ │ │ │ │ │ │ │ │ └── DataNode.java │ │ │ │ │ │ │ └── schema/ │ │ │ │ │ │ │ ├── ChildTable.java │ │ │ │ │ │ │ ├── Schema.java │ │ │ │ │ │ │ └── Table.java │ │ │ │ │ │ └── server/ │ │ │ │ │ │ ├── System.java │ │ │ │ │ │ └── user/ │ │ │ │ │ │ └── User.java │ │ │ │ │ ├── parse/ │ │ │ │ │ │ ├── JsonProcessBase.java │ │ │ │ │ │ ├── ParseJsonServiceInf.java │ │ │ │ │ │ ├── ParseXmlServiceInf.java │ │ │ │ │ │ ├── XmlProcessBase.java │ │ │ │ │ │ └── entryparse/ │ │ │ │ │ │ ├── cache/ │ │ │ │ │ │ │ ├── json/ │ │ │ │ │ │ │ │ └── EhcacheJsonParse.java │ │ │ │ │ │ │ └── xml/ │ │ │ │ │ │ │ └── EhcacheParseXmlImpl.java │ │ │ │ │ │ ├── rule/ │ │ │ │ │ │ │ ├── json/ │ │ │ │ │ │ │ │ ├── FunctionJsonParse.java │ │ │ │ │ │ │ │ └── TableRuleJsonParse.java │ │ │ │ │ │ │ └── xml/ │ │ │ │ │ │ │ └── RuleParseXmlImpl.java │ │ │ │ │ │ ├── schema/ │ │ │ │ │ │ │ ├── json/ │ │ │ │ │ │ │ │ ├── DataHostJsonParse.java │ │ │ │ │ │ │ │ ├── DataNodeJsonParse.java │ │ │ │ │ │ │ │ └── SchemaJsonParse.java │ │ │ │ │ │ │ └── xml/ │ │ │ │ │ │ │ └── SchemasParseXmlImpl.java │ │ │ │ │ │ └── server/ │ │ │ │ │ │ ├── json/ │ │ │ │ │ │ │ ├── SystemJsonParse.java │ │ │ │ │ │ │ └── UserJsonParse.java │ │ │ │ │ │ └── xml/ │ │ │ │ │ │ └── ServerParseXmlImpl.java │ │ │ │ │ ├── xmltozk/ │ │ │ │ │ │ ├── BindataToZK.java │ │ │ │ │ │ ├── XmltoZkMain.java │ │ │ │ │ │ └── listen/ │ │ │ │ │ │ ├── EcachesxmlTozkLoader.java │ │ │ │ │ │ ├── OthermsgTozkLoader.java │ │ │ │ │ │ ├── RulesxmlTozkLoader.java │ │ │ │ │ │ ├── SchemasxmlTozkLoader.java │ │ │ │ │ │ ├── SequenceTozkLoader.java │ │ │ │ │ │ └── ServerxmlTozkLoader.java │ │ │ │ │ ├── zktoxml/ │ │ │ │ │ │ ├── ZktoXmlMain.java │ │ │ │ │ │ ├── command/ │ │ │ │ │ │ │ └── CommandPathListener.java │ │ │ │ │ │ └── listen/ │ │ │ │ │ │ ├── BinDataPathChildrenCacheListener.java │ │ │ │ │ │ ├── EcacheszkToxmlLoader.java │ │ │ │ │ │ ├── RuleDataPathChildrenCacheListener.java │ │ │ │ │ │ ├── RuleFunctionCacheListener.java │ │ │ │ │ │ ├── RuleszkToxmlLoader.java │ │ │ │ │ │ ├── SchemaszkToxmlLoader.java │ │ │ │ │ │ ├── SequenceTopropertiesLoader.java │ │ │ │ │ │ └── ServerzkToxmlLoader.java │ │ │ │ │ └── zookeeper/ │ │ │ │ │ ├── ClusterInfo.java │ │ │ │ │ ├── DataInf.java │ │ │ │ │ ├── DiretoryInf.java │ │ │ │ │ └── process/ │ │ │ │ │ ├── ZkDataImpl.java │ │ │ │ │ ├── ZkDirectoryImpl.java │ │ │ │ │ └── ZkMultLoader.java │ │ │ │ ├── model/ │ │ │ │ │ ├── ClusterConfig.java │ │ │ │ │ ├── DBHostConfig.java │ │ │ │ │ ├── DataHostConfig.java │ │ │ │ │ ├── DataNodeConfig.java │ │ │ │ │ ├── FirewallConfig.java │ │ │ │ │ ├── MycatNodeConfig.java │ │ │ │ │ ├── SchemaConfig.java │ │ │ │ │ ├── SystemConfig.java │ │ │ │ │ ├── TableConfig.java │ │ │ │ │ ├── TableConfigMap.java │ │ │ │ │ ├── TableRuleConfig.java │ │ │ │ │ ├── UserConfig.java │ │ │ │ │ ├── UserPrivilegesConfig.java │ │ │ │ │ └── rule/ │ │ │ │ │ ├── RuleAlgorithm.java │ │ │ │ │ ├── RuleConfig.java │ │ │ │ │ └── TableRuleConfig.java │ │ │ │ ├── table/ │ │ │ │ │ └── structure/ │ │ │ │ │ ├── MySQLTableStructureDetector.java │ │ │ │ │ └── TableStructureProcessor.java │ │ │ │ └── util/ │ │ │ │ ├── BeanConfig.java │ │ │ │ ├── ConfigException.java │ │ │ │ ├── ConfigUtil.java │ │ │ │ ├── DnPropertyUtil.java │ │ │ │ ├── FieldDictionary.java │ │ │ │ ├── Initializable.java │ │ │ │ ├── JVMInfo.java │ │ │ │ ├── ObjectAccessException.java │ │ │ │ ├── OrderRetainingMap.java │ │ │ │ ├── ParameterMapping.java │ │ │ │ ├── ReflectionProvider.java │ │ │ │ └── Visitor.java │ │ │ ├── manager/ │ │ │ │ ├── ManagerConnection.java │ │ │ │ ├── ManagerConnectionFactory.java │ │ │ │ ├── ManagerQueryHandler.java │ │ │ │ ├── handler/ │ │ │ │ │ ├── ClearHandler.java │ │ │ │ │ ├── ConfFileHandler.java │ │ │ │ │ ├── ReloadHandler.java │ │ │ │ │ ├── RollbackHandler.java │ │ │ │ │ ├── SelectHandler.java │ │ │ │ │ ├── ShowHandler.java │ │ │ │ │ ├── ShowServerLog.java │ │ │ │ │ ├── StopHandler.java │ │ │ │ │ ├── SwitchHandler.java │ │ │ │ │ └── ZKHandler.java │ │ │ │ └── response/ │ │ │ │ ├── CheckGlobalTable.java │ │ │ │ ├── ClearSlow.java │ │ │ │ ├── KillConnection.java │ │ │ │ ├── Offline.java │ │ │ │ ├── Online.java │ │ │ │ ├── ReloadConfig.java │ │ │ │ ├── ReloadQueryCf.java │ │ │ │ ├── ReloadSqlSlowTime.java │ │ │ │ ├── ReloadSqlStat.java │ │ │ │ ├── ReloadUser.java │ │ │ │ ├── ReloadUserStat.java │ │ │ │ ├── ReloadZktoXml.java │ │ │ │ ├── RollbackConfig.java │ │ │ │ ├── RollbackUser.java │ │ │ │ ├── SelectSessionAutoIncrement.java │ │ │ │ ├── SelectSessionTxReadOnly.java │ │ │ │ ├── SelectVersionComment.java │ │ │ │ ├── ShowBackend.java │ │ │ │ ├── ShowBackendOld.java │ │ │ │ ├── ShowCollation.java │ │ │ │ ├── ShowCommand.java │ │ │ │ ├── ShowConnection.java │ │ │ │ ├── ShowConnectionSQL.java │ │ │ │ ├── ShowDataNode.java │ │ │ │ ├── ShowDataSource.java │ │ │ │ ├── ShowDatabase.java │ │ │ │ ├── ShowDatasourceCluster.java │ │ │ │ ├── ShowDatasourceSyn.java │ │ │ │ ├── ShowDatasourceSynDetail.java │ │ │ │ ├── ShowDirectMemory.java │ │ │ │ ├── ShowHeartbeat.java │ │ │ │ ├── ShowHeartbeatDetail.java │ │ │ │ ├── ShowHelp.java │ │ │ │ ├── ShowParser.java │ │ │ │ ├── ShowProcessor.java │ │ │ │ ├── ShowRouter.java │ │ │ │ ├── ShowSQL.java │ │ │ │ ├── ShowSQLCondition.java │ │ │ │ ├── ShowSQLDetail.java │ │ │ │ ├── ShowSQLExecute.java │ │ │ │ ├── ShowSQLHigh.java │ │ │ │ ├── ShowSQLLarge.java │ │ │ │ ├── ShowSQLSlow.java │ │ │ │ ├── ShowSQLSumTable.java │ │ │ │ ├── ShowSQLSumUser.java │ │ │ │ ├── ShowServer.java │ │ │ │ ├── ShowSession.java │ │ │ │ ├── ShowSqlResultSet.java │ │ │ │ ├── ShowSysLog.java │ │ │ │ ├── ShowSysParam.java │ │ │ │ ├── ShowThreadPool.java │ │ │ │ ├── ShowTime.java │ │ │ │ ├── ShowVariables.java │ │ │ │ ├── ShowVersion.java │ │ │ │ ├── ShowWhiteHost.java │ │ │ │ ├── StopHeartbeat.java │ │ │ │ └── SwitchDataSource.java │ │ │ ├── memory/ │ │ │ │ ├── MyCatMemory.java │ │ │ │ ├── environment/ │ │ │ │ │ ├── EnvironmentInformation.java │ │ │ │ │ ├── Hardware.java │ │ │ │ │ ├── HardwareDescription.java │ │ │ │ │ └── OperatingSystem.java │ │ │ │ └── unsafe/ │ │ │ │ ├── KVIterator.java │ │ │ │ ├── Platform.java │ │ │ │ ├── array/ │ │ │ │ │ ├── ByteArrayMethods.java │ │ │ │ │ ├── CharArray.java │ │ │ │ │ └── LongArray.java │ │ │ │ ├── bitset/ │ │ │ │ │ └── BitSetMethods.java │ │ │ │ ├── hash/ │ │ │ │ │ └── Murmur3_x86_32.java │ │ │ │ ├── map/ │ │ │ │ │ ├── BytesToBytesMap.java │ │ │ │ │ ├── HashMapGrowthStrategy.java │ │ │ │ │ └── UnsafeFixedWidthAggregationMap.java │ │ │ │ ├── memory/ │ │ │ │ │ ├── HeapMemoryAllocator.java │ │ │ │ │ ├── MemoryAllocator.java │ │ │ │ │ ├── MemoryBlock.java │ │ │ │ │ ├── MemoryLocation.java │ │ │ │ │ ├── UnsafeMemoryAllocator.java │ │ │ │ │ └── mm/ │ │ │ │ │ ├── DataNodeMemoryManager.java │ │ │ │ │ ├── MemoryConsumer.java │ │ │ │ │ ├── MemoryManager.java │ │ │ │ │ ├── MemoryMode.java │ │ │ │ │ ├── MemoryPool.java │ │ │ │ │ ├── ResultMergeMemoryManager.java │ │ │ │ │ └── ResultSetMemoryPool.java │ │ │ │ ├── ringbuffer/ │ │ │ │ │ ├── RingBuffer.java │ │ │ │ │ ├── common/ │ │ │ │ │ │ ├── Cursored.java │ │ │ │ │ │ ├── DataProvider.java │ │ │ │ │ │ ├── Sequenced.java │ │ │ │ │ │ ├── barrier/ │ │ │ │ │ │ │ └── SequenceBarrier.java │ │ │ │ │ │ ├── event/ │ │ │ │ │ │ │ ├── EventFactory.java │ │ │ │ │ │ │ ├── EventSequencer.java │ │ │ │ │ │ │ ├── EventSink.java │ │ │ │ │ │ │ ├── EventTranslator.java │ │ │ │ │ │ │ ├── EventTranslatorOneArg.java │ │ │ │ │ │ │ ├── EventTranslatorThreeArg.java │ │ │ │ │ │ │ ├── EventTranslatorTwoArg.java │ │ │ │ │ │ │ └── EventTranslatorVararg.java │ │ │ │ │ │ ├── sequence/ │ │ │ │ │ │ │ ├── Sequence.java │ │ │ │ │ │ │ └── SequenceGroups.java │ │ │ │ │ │ └── waitStrategy/ │ │ │ │ │ │ ├── WaitStrategy.java │ │ │ │ │ │ └── impl/ │ │ │ │ │ │ ├── BlockingWaitStrategy.java │ │ │ │ │ │ ├── BusySpinWaitStrategy.java │ │ │ │ │ │ └── SleepingWaitStrategy.java │ │ │ │ │ ├── exception/ │ │ │ │ │ │ ├── AlertException.java │ │ │ │ │ │ ├── InsufficientCapacityException.java │ │ │ │ │ │ └── TimeoutException.java │ │ │ │ │ ├── producer/ │ │ │ │ │ │ ├── AbstractSequencer.java │ │ │ │ │ │ ├── MultiProducerSequencer.java │ │ │ │ │ │ ├── Sequencer.java │ │ │ │ │ │ └── SingleProducerSequencer.java │ │ │ │ │ └── utils/ │ │ │ │ │ └── Util.java │ │ │ │ ├── row/ │ │ │ │ │ ├── BufferHolder.java │ │ │ │ │ ├── StructType.java │ │ │ │ │ ├── UnsafeRow.java │ │ │ │ │ └── UnsafeRowWriter.java │ │ │ │ ├── storage/ │ │ │ │ │ ├── ConnectionId.java │ │ │ │ │ ├── DataNodeDiskManager.java │ │ │ │ │ ├── DataNodeFileManager.java │ │ │ │ │ ├── DeserializationStream.java │ │ │ │ │ ├── DiskRowWriter.java │ │ │ │ │ ├── DummySerializerInstance.java │ │ │ │ │ ├── SerializationStream.java │ │ │ │ │ ├── SerializerInstance.java │ │ │ │ │ ├── SerializerManager.java │ │ │ │ │ ├── TempDataNodeId.java │ │ │ │ │ └── TimeTrackingOutputStream.java │ │ │ │ ├── types/ │ │ │ │ │ ├── ByteArray.java │ │ │ │ │ ├── CalendarInterval.java │ │ │ │ │ └── UTF8String.java │ │ │ │ └── utils/ │ │ │ │ ├── ByteUnit.java │ │ │ │ ├── BytesTools.java │ │ │ │ ├── JavaUtils.java │ │ │ │ ├── MycatPropertyConf.java │ │ │ │ └── sort/ │ │ │ │ ├── AbstractScalaRowIterator.java │ │ │ │ ├── PrefixComparator.java │ │ │ │ ├── PrefixComparators.java │ │ │ │ ├── RadixSort.java │ │ │ │ ├── RecordComparator.java │ │ │ │ ├── RecordPointerAndKeyPrefix.java │ │ │ │ ├── RowPrefixComputer.java │ │ │ │ ├── SortDataFormat.java │ │ │ │ ├── SortPrefixUtils.java │ │ │ │ ├── Sorter.java │ │ │ │ ├── TestSorter.java │ │ │ │ ├── TimSort.java │ │ │ │ ├── UnsafeExternalRowSorter.java │ │ │ │ ├── UnsafeExternalSorter.java │ │ │ │ ├── UnsafeInMemorySorter.java │ │ │ │ ├── UnsafeKVExternalSorter.java │ │ │ │ ├── UnsafeKeyValueSorter.java │ │ │ │ ├── UnsafeRowsMerger.java │ │ │ │ ├── UnsafeSortDataFormat.java │ │ │ │ ├── UnsafeSorterIterator.java │ │ │ │ ├── UnsafeSorterSpillMerger.java │ │ │ │ ├── UnsafeSorterSpillReader.java │ │ │ │ └── UnsafeSorterSpillWriter.java │ │ │ ├── migrate/ │ │ │ │ ├── BinlogIdleCheck.java │ │ │ │ ├── BinlogStream.java │ │ │ │ ├── BinlogStreamHoder.java │ │ │ │ ├── MigrateDumpRunner.java │ │ │ │ ├── MigrateMainRunner.java │ │ │ │ ├── MigrateTask.java │ │ │ │ ├── MigrateTaskWatch.java │ │ │ │ ├── MigrateUtils.java │ │ │ │ ├── SqlExecuteListener.java │ │ │ │ ├── SwitchCleanListener.java │ │ │ │ ├── SwitchCommitListener.java │ │ │ │ ├── SwitchPrepareCheckRunner.java │ │ │ │ ├── SwitchPrepareListener.java │ │ │ │ ├── TaskNode.java │ │ │ │ └── TaskStatus.java │ │ │ ├── net/ │ │ │ │ ├── AIOAcceptor.java │ │ │ │ ├── AIOConnector.java │ │ │ │ ├── AIOSocketWR.java │ │ │ │ ├── AbstractConnection.java │ │ │ │ ├── BIOConnection.java │ │ │ │ ├── BackendAIOConnection.java │ │ │ │ ├── ClosableConnection.java │ │ │ │ ├── ConnectionException.java │ │ │ │ ├── FrontendConnection.java │ │ │ │ ├── NIOAcceptor.java │ │ │ │ ├── NIOConnection.java │ │ │ │ ├── NIOConnector.java │ │ │ │ ├── NIOHandler.java │ │ │ │ ├── NIOProcessor.java │ │ │ │ ├── NIOReactor.java │ │ │ │ ├── NIOReactorPool.java │ │ │ │ ├── NIOSocketWR.java │ │ │ │ ├── SocketAcceptor.java │ │ │ │ ├── SocketConnector.java │ │ │ │ ├── SocketWR.java │ │ │ │ ├── WriteEventCheckRunner.java │ │ │ │ ├── factory/ │ │ │ │ │ ├── BackendConnectionFactory.java │ │ │ │ │ └── FrontendConnectionFactory.java │ │ │ │ ├── handler/ │ │ │ │ │ ├── BackendAsyncHandler.java │ │ │ │ │ ├── FrontendAuthenticator.java │ │ │ │ │ ├── FrontendCommandHandler.java │ │ │ │ │ ├── FrontendPrepareHandler.java │ │ │ │ │ ├── FrontendPrivileges.java │ │ │ │ │ ├── FrontendQueryHandler.java │ │ │ │ │ └── LoadDataInfileHandler.java │ │ │ │ ├── mysql/ │ │ │ │ │ ├── AuthPacket.java │ │ │ │ │ ├── AuthSwitchPacket.java │ │ │ │ │ ├── BinaryPacket.java │ │ │ │ │ ├── BinaryRowDataPacket.java │ │ │ │ │ ├── CommandPacket.java │ │ │ │ │ ├── EOFPacket.java │ │ │ │ │ ├── EmptyPacket.java │ │ │ │ │ ├── ErrorPacket.java │ │ │ │ │ ├── ExecutePacket.java │ │ │ │ │ ├── FieldPacket.java │ │ │ │ │ ├── HandshakePacket.java │ │ │ │ │ ├── HandshakeV10Packet.java │ │ │ │ │ ├── HeartbeatPacket.java │ │ │ │ │ ├── LongDataPacket.java │ │ │ │ │ ├── MySQLPacket.java │ │ │ │ │ ├── OkPacket.java │ │ │ │ │ ├── PingPacket.java │ │ │ │ │ ├── PreparedOkPacket.java │ │ │ │ │ ├── QuitPacket.java │ │ │ │ │ ├── Reply323Packet.java │ │ │ │ │ ├── RequestFilePacket.java │ │ │ │ │ ├── ResetPacket.java │ │ │ │ │ ├── ResultSetHeaderPacket.java │ │ │ │ │ ├── RowDataPacket.java │ │ │ │ │ └── StatusFlags.java │ │ │ │ └── postgres/ │ │ │ │ ├── AuthenticationCleartextPassword.java │ │ │ │ ├── AuthenticationGSS.java │ │ │ │ ├── AuthenticationGSSContinue.java │ │ │ │ ├── AuthenticationKerberosV5.java │ │ │ │ ├── AuthenticationMD5Password.java │ │ │ │ ├── AuthenticationOk.java │ │ │ │ ├── AuthenticationSCMCredential.java │ │ │ │ ├── AuthenticationSSPI.java │ │ │ │ ├── BackendKeyData.java │ │ │ │ ├── Bind.java │ │ │ │ ├── BindComplete.java │ │ │ │ ├── CancelRequest.java │ │ │ │ ├── Close.java │ │ │ │ ├── CloseComplete.java │ │ │ │ ├── CommandComplete.java │ │ │ │ ├── CopyBothResponse.java │ │ │ │ ├── CopyData.java │ │ │ │ ├── CopyDone.java │ │ │ │ ├── CopyFail.java │ │ │ │ ├── CopyInResponse.java │ │ │ │ ├── CopyOutResponse.java │ │ │ │ ├── DataRow.java │ │ │ │ ├── Describe.java │ │ │ │ ├── EmptyQueryResponse.java │ │ │ │ ├── ErrorResponse.java │ │ │ │ ├── Execute.java │ │ │ │ ├── Flush.java │ │ │ │ ├── FunctionCall.java │ │ │ │ ├── FunctionCallResponse.java │ │ │ │ ├── NoData.java │ │ │ │ ├── NoticeResponse.java │ │ │ │ ├── NotificationResponse.java │ │ │ │ ├── ParameterDescription.java │ │ │ │ ├── ParameterStatus.java │ │ │ │ ├── Parse.java │ │ │ │ ├── ParseComplete.java │ │ │ │ ├── PasswordMessage.java │ │ │ │ ├── PortalSuspended.java │ │ │ │ ├── PostgresPacket.java │ │ │ │ ├── Query.java │ │ │ │ ├── ReadyForQuery.java │ │ │ │ ├── RowDescription.java │ │ │ │ ├── SSLRequest.java │ │ │ │ ├── StartupMessage.java │ │ │ │ ├── Sync.java │ │ │ │ └── Terminate.java │ │ │ ├── route/ │ │ │ │ ├── MyCATSequnceProcessor.java │ │ │ │ ├── Procedure.java │ │ │ │ ├── ProcedureParameter.java │ │ │ │ ├── RouteCheckRule.java │ │ │ │ ├── RouteResultset.java │ │ │ │ ├── RouteResultsetNode.java │ │ │ │ ├── RouteService.java │ │ │ │ ├── RouteStrategy.java │ │ │ │ ├── SQLMerge.java │ │ │ │ ├── SessionSQLPair.java │ │ │ │ ├── factory/ │ │ │ │ │ └── RouteStrategyFactory.java │ │ │ │ ├── function/ │ │ │ │ │ ├── AbstractPartitionAlgorithm.java │ │ │ │ │ ├── AutoPartitionByLong.java │ │ │ │ │ ├── LatestMonthPartion.java │ │ │ │ │ ├── NumberParseUtil.java │ │ │ │ │ ├── PartitionByCRC32PreSlot.java │ │ │ │ │ ├── PartitionByDate.java │ │ │ │ │ ├── PartitionByFileMap.java │ │ │ │ │ ├── PartitionByHashMod.java │ │ │ │ │ ├── PartitionByHotDate.java │ │ │ │ │ ├── PartitionByJumpConsistentHash.java │ │ │ │ │ ├── PartitionByLong.java │ │ │ │ │ ├── PartitionByMod.java │ │ │ │ │ ├── PartitionByMonth.java │ │ │ │ │ ├── PartitionByMonthAndHistory.java │ │ │ │ │ ├── PartitionByMurmurHash.java │ │ │ │ │ ├── PartitionByPattern.java │ │ │ │ │ ├── PartitionByPrefixPattern.java │ │ │ │ │ ├── PartitionByRangeDateHash.java │ │ │ │ │ ├── PartitionByRangeMod.java │ │ │ │ │ ├── PartitionByString.java │ │ │ │ │ ├── PartitionDirectBySubString.java │ │ │ │ │ ├── PureJavaCrc32.java │ │ │ │ │ ├── ReloadFunction.java │ │ │ │ │ ├── SlotFunction.java │ │ │ │ │ └── TableRuleAware.java │ │ │ │ ├── handler/ │ │ │ │ │ ├── HintCatletHandler.java │ │ │ │ │ ├── HintDataNodeHandler.java │ │ │ │ │ ├── HintHandler.java │ │ │ │ │ ├── HintHandlerFactory.java │ │ │ │ │ ├── HintMasterDBHandler.java │ │ │ │ │ ├── HintSQLHandler.java │ │ │ │ │ └── HintSchemaHandler.java │ │ │ │ ├── impl/ │ │ │ │ │ ├── AbstractRouteStrategy.java │ │ │ │ │ ├── DruidMycatRouteStrategy.java │ │ │ │ │ └── middlerResultStrategy/ │ │ │ │ │ ├── BinaryOpResultHandler.java │ │ │ │ │ ├── InSubQueryResultHandler.java │ │ │ │ │ ├── RouteMiddlerReaultHandler.java │ │ │ │ │ ├── SQLAllResultHandler.java │ │ │ │ │ ├── SQLExistsResultHandler.java │ │ │ │ │ └── SQLQueryResultHandler.java │ │ │ │ ├── parser/ │ │ │ │ │ ├── ManagerParse.java │ │ │ │ │ ├── ManagerParseClear.java │ │ │ │ │ ├── ManagerParseHeartbeat.java │ │ │ │ │ ├── ManagerParseKill.java │ │ │ │ │ ├── ManagerParseReload.java │ │ │ │ │ ├── ManagerParseRollback.java │ │ │ │ │ ├── ManagerParseSelect.java │ │ │ │ │ ├── ManagerParseShow.java │ │ │ │ │ ├── ManagerParseStop.java │ │ │ │ │ ├── ManagerParseSwitch.java │ │ │ │ │ ├── druid/ │ │ │ │ │ │ ├── DruidParser.java │ │ │ │ │ │ ├── DruidParserFactory.java │ │ │ │ │ │ ├── DruidSequenceHandler.java │ │ │ │ │ │ ├── DruidShardingParseInfo.java │ │ │ │ │ │ ├── LoadDataOutputVisitor.java │ │ │ │ │ │ ├── LoadDataStatement.java │ │ │ │ │ │ ├── MycatExprParser.java │ │ │ │ │ │ ├── MycatLexer.java │ │ │ │ │ │ ├── MycatSchemaStatVisitor.java │ │ │ │ │ │ ├── MycatSelectParser.java │ │ │ │ │ │ ├── MycatStatementParser.java │ │ │ │ │ │ ├── MycatSubQueryVisitor.java │ │ │ │ │ │ ├── RouteCalculateUnit.java │ │ │ │ │ │ ├── SchemaStatVisitorFactory.java │ │ │ │ │ │ ├── SqlMethodInvocationHandler.java │ │ │ │ │ │ ├── SqlMethodInvocationHandlerFactory.java │ │ │ │ │ │ ├── WhereUnit.java │ │ │ │ │ │ └── impl/ │ │ │ │ │ │ ├── DefaultDruidParser.java │ │ │ │ │ │ ├── DruidAlterTableParser.java │ │ │ │ │ │ ├── DruidCreateTableParser.java │ │ │ │ │ │ ├── DruidDeleteParser.java │ │ │ │ │ │ ├── DruidInsertParser.java │ │ │ │ │ │ ├── DruidLockTableParser.java │ │ │ │ │ │ ├── DruidSelectDb2Parser.java │ │ │ │ │ │ ├── DruidSelectOracleParser.java │ │ │ │ │ │ ├── DruidSelectParser.java │ │ │ │ │ │ ├── DruidSelectPostgresqlParser.java │ │ │ │ │ │ ├── DruidSelectSqlServerParser.java │ │ │ │ │ │ ├── DruidUpdateParser.java │ │ │ │ │ │ ├── MysqlMethodInvocationHandler.java │ │ │ │ │ │ ├── OracleMethodInvocationHandler.java │ │ │ │ │ │ └── PgsqlMethodInvocationHandler.java │ │ │ │ │ ├── primitive/ │ │ │ │ │ │ ├── FunctionParser.java │ │ │ │ │ │ └── Model/ │ │ │ │ │ │ ├── Commons.java │ │ │ │ │ │ ├── Field.java │ │ │ │ │ │ ├── Function.java │ │ │ │ │ │ └── Identifier.java │ │ │ │ │ └── util/ │ │ │ │ │ ├── ArrayUtil.java │ │ │ │ │ ├── CharTypes.java │ │ │ │ │ ├── PageSQLUtil.java │ │ │ │ │ ├── Pair.java │ │ │ │ │ ├── PairUtil.java │ │ │ │ │ ├── ParseString.java │ │ │ │ │ ├── ParseUtil.java │ │ │ │ │ ├── SQLParserUtils.java │ │ │ │ │ └── WildcardUtil.java │ │ │ │ ├── sequence/ │ │ │ │ │ ├── BatchInsertSequence.java │ │ │ │ │ └── handler/ │ │ │ │ │ ├── DistributedSequenceHandler.java │ │ │ │ │ ├── HttpIncrSequenceHandler.java │ │ │ │ │ ├── IncrSequenceBDBHandler.java │ │ │ │ │ ├── IncrSequenceHandler.java │ │ │ │ │ ├── IncrSequenceMySQLHandler.java │ │ │ │ │ ├── IncrSequencePropHandler.java │ │ │ │ │ ├── IncrSequenceTimeHandler.java │ │ │ │ │ ├── IncrSequenceZKHandler.java │ │ │ │ │ ├── SequenceHandler.java │ │ │ │ │ ├── SequenceVal.java │ │ │ │ │ ├── SnowflakeIdSequenceHandler.java │ │ │ │ │ └── ThirftClientSequenceHandler.java │ │ │ │ └── util/ │ │ │ │ ├── CacheUtil.java │ │ │ │ ├── PartitionUtil.java │ │ │ │ ├── PropertiesUtil.java │ │ │ │ └── RouterUtil.java │ │ │ ├── server/ │ │ │ │ ├── NonBlockingSession.java │ │ │ │ ├── ServerConnection.java │ │ │ │ ├── ServerConnectionFactory.java │ │ │ │ ├── ServerQueryHandler.java │ │ │ │ ├── Session.java │ │ │ │ ├── handler/ │ │ │ │ │ ├── BeginHandler.java │ │ │ │ │ ├── CommandHandler.java │ │ │ │ │ ├── Explain2Handler.java │ │ │ │ │ ├── ExplainHandler.java │ │ │ │ │ ├── KillHandler.java │ │ │ │ │ ├── MigrateHandler.java │ │ │ │ │ ├── MysqlInformationSchemaHandler.java │ │ │ │ │ ├── MysqlProcHandler.java │ │ │ │ │ ├── SavepointHandler.java │ │ │ │ │ ├── SelectHandler.java │ │ │ │ │ ├── ServerLoadDataInfileHandler.java │ │ │ │ │ ├── ServerPrepareHandler.java │ │ │ │ │ ├── SetHandler.java │ │ │ │ │ ├── ShowCache.java │ │ │ │ │ ├── ShowHandler.java │ │ │ │ │ ├── StartHandler.java │ │ │ │ │ └── UseHandler.java │ │ │ │ ├── interceptor/ │ │ │ │ │ ├── SQLInterceptor.java │ │ │ │ │ └── impl/ │ │ │ │ │ ├── DefaultSqlInterceptor.java │ │ │ │ │ ├── GlobalTableUtil.java │ │ │ │ │ ├── StatSqlInterceptor.java │ │ │ │ │ └── StatisticsSqlInterceptor.java │ │ │ │ ├── parser/ │ │ │ │ │ ├── ServerParse.java │ │ │ │ │ ├── ServerParseSelect.java │ │ │ │ │ ├── ServerParseSet.java │ │ │ │ │ ├── ServerParseShow.java │ │ │ │ │ └── ServerParseStart.java │ │ │ │ ├── response/ │ │ │ │ │ ├── CharacterSet.java │ │ │ │ │ ├── ClientHeartbeatResponse.java │ │ │ │ │ ├── Heartbeat.java │ │ │ │ │ ├── InformationSchemaProfiling.java │ │ │ │ │ ├── InformationSchemaProfilingSqlyog.java │ │ │ │ │ ├── Ping.java │ │ │ │ │ ├── PreparedStmtResponse.java │ │ │ │ │ ├── SelectConnnectID.java │ │ │ │ │ ├── SelectDatabase.java │ │ │ │ │ ├── SelectIdentity.java │ │ │ │ │ ├── SelectLastInsertId.java │ │ │ │ │ ├── SelectTxReadOnly.java │ │ │ │ │ ├── SelectUser.java │ │ │ │ │ ├── SelectVariables.java │ │ │ │ │ ├── SelectVersion.java │ │ │ │ │ ├── SelectVersionComment.java │ │ │ │ │ ├── SessionIncrement.java │ │ │ │ │ ├── SessionIsolation.java │ │ │ │ │ ├── ShowCobarCluster.java │ │ │ │ │ ├── ShowCobarStatus.java │ │ │ │ │ ├── ShowDatabases.java │ │ │ │ │ ├── ShowFullTables.java │ │ │ │ │ ├── ShowMyCATCluster.java │ │ │ │ │ ├── ShowMyCatStatus.java │ │ │ │ │ └── ShowTables.java │ │ │ │ ├── sqlcmd/ │ │ │ │ │ ├── CommitCommand.java │ │ │ │ │ ├── SQLCmdConstant.java │ │ │ │ │ └── SQLCtrlCommand.java │ │ │ │ └── util/ │ │ │ │ └── SchemaUtil.java │ │ │ ├── sqlengine/ │ │ │ │ ├── AllJobFinishedListener.java │ │ │ │ ├── BatchSQLJob.java │ │ │ │ ├── EngineCtx.java │ │ │ │ ├── MultiRowSQLQueryResultHandler.java │ │ │ │ ├── OneRawSQLQueryResultHandler.java │ │ │ │ ├── SQLJob.java │ │ │ │ ├── SQLJobHandler.java │ │ │ │ ├── SQLQueryResult.java │ │ │ │ ├── SQLQueryResultListener.java │ │ │ │ └── mpp/ │ │ │ │ ├── AbstractDataNodeMerge.java │ │ │ │ ├── ColMeta.java │ │ │ │ ├── ColumnRoutePair.java │ │ │ │ ├── DataMergeService.java │ │ │ │ ├── DataNodeMergeManager.java │ │ │ │ ├── HavingCols.java │ │ │ │ ├── LoadData.java │ │ │ │ ├── MergeCol.java │ │ │ │ ├── OrderCol.java │ │ │ │ ├── PackWraper.java │ │ │ │ ├── RangRowDataPacketSorter.java │ │ │ │ ├── RangeValue.java │ │ │ │ ├── RowDataPacketGrouper.java │ │ │ │ ├── RowDataPacketSorter.java │ │ │ │ ├── UnsafeRowGrouper.java │ │ │ │ └── model/ │ │ │ │ ├── NodeRowDataPacket.java │ │ │ │ └── RangRowDataPacket.java │ │ │ ├── statistic/ │ │ │ │ ├── CommandCount.java │ │ │ │ ├── DataSourceSyncRecorder.java │ │ │ │ ├── HeartbeatRecorder.java │ │ │ │ ├── SQLRecord.java │ │ │ │ ├── SQLRecorder.java │ │ │ │ └── stat/ │ │ │ │ ├── Histogram.java │ │ │ │ ├── HostStatAnalyzer.java │ │ │ │ ├── QueryConditionAnalyzer.java │ │ │ │ ├── QueryResult.java │ │ │ │ ├── QueryResultDispatcher.java │ │ │ │ ├── QueryResultListener.java │ │ │ │ ├── SqlFrequency.java │ │ │ │ ├── SqlResultSet.java │ │ │ │ ├── SqlResultSizeRecorder.java │ │ │ │ ├── TableStat.java │ │ │ │ ├── TableStatAnalyzer.java │ │ │ │ ├── UserSqlHighStat.java │ │ │ │ ├── UserSqlLargeStat.java │ │ │ │ ├── UserSqlLastStat.java │ │ │ │ ├── UserSqlRWStat.java │ │ │ │ ├── UserStat.java │ │ │ │ └── UserStatAnalyzer.java │ │ │ └── util/ │ │ │ ├── ByteBufferUtil.java │ │ │ ├── ByteUtil.java │ │ │ ├── CircularArrayList.java │ │ │ ├── CollectionUtil.java │ │ │ ├── CompareUtil.java │ │ │ ├── CompressUtil.java │ │ │ ├── DateUtil.java │ │ │ ├── DecryptUtil.java │ │ │ ├── ExecutorUtil.java │ │ │ ├── FastByteOperations.java │ │ │ ├── FormatUtil.java │ │ │ ├── HexFormatUtil.java │ │ │ ├── IntegerUtil.java │ │ │ ├── LogUtil.java │ │ │ ├── LongUtil.java │ │ │ ├── MysqlDefs.java │ │ │ ├── NameableExecutor.java │ │ │ ├── NameableThreadFactory.java │ │ │ ├── ObjectUtil.java │ │ │ ├── ProcessUtil.java │ │ │ ├── RandomUtil.java │ │ │ ├── ResultSetUtil.java │ │ │ ├── SelectorUtil.java │ │ │ ├── SetIgnoreUtil.java │ │ │ ├── SmallSet.java │ │ │ ├── SplitUtil.java │ │ │ ├── StreamGobble.java │ │ │ ├── StringUtil.java │ │ │ ├── TimeUtil.java │ │ │ ├── ZKUtils.java │ │ │ ├── cmd/ │ │ │ │ └── CmdArgs.java │ │ │ ├── dataMigrator/ │ │ │ │ ├── ConfigComparer.java │ │ │ │ ├── DataClearRunner.java │ │ │ │ ├── DataIO.java │ │ │ │ ├── DataIOFactory.java │ │ │ │ ├── DataMigrateRunner.java │ │ │ │ ├── DataMigrator.java │ │ │ │ ├── DataMigratorArgs.java │ │ │ │ ├── DataMigratorUtil.java │ │ │ │ ├── DataNode.java │ │ │ │ ├── DataNodeClearGroup.java │ │ │ │ ├── DataNodeMigrateInfo.java │ │ │ │ ├── MigratorConditonFilesMaker.java │ │ │ │ ├── TableMigrateInfo.java │ │ │ │ └── dataIOImpl/ │ │ │ │ └── MysqlDataIO.java │ │ │ ├── exception/ │ │ │ │ ├── DataMigratorException.java │ │ │ │ ├── ErrorPacketException.java │ │ │ │ ├── HeartbeatException.java │ │ │ │ ├── MurmurHashException.java │ │ │ │ ├── RehashException.java │ │ │ │ ├── UnknownCharsetException.java │ │ │ │ ├── UnknownDataNodeException.java │ │ │ │ ├── UnknownPacketException.java │ │ │ │ └── UnknownTxIsolationException.java │ │ │ └── rehasher/ │ │ │ ├── HashType.java │ │ │ ├── RehashCmdArgs.java │ │ │ └── RehashLauncher.java │ │ └── resources/ │ │ ├── auto-sharding-long.txt │ │ ├── auto-sharding-rang-mod.txt │ │ ├── autopartition-long.txt │ │ ├── cacheservice.properties │ │ ├── dbseq - utf8mb4.sql │ │ ├── dbseq.sql │ │ ├── ehcache.xml │ │ ├── index_to_charset.properties │ │ ├── log4j2.xml │ │ ├── migrateTables.properties │ │ ├── myid.properties │ │ ├── partition-hash-int.txt │ │ ├── partition-range-mod.txt │ │ ├── rule.dtd │ │ ├── rule.xml │ │ ├── schema.dtd │ │ ├── schema.xml │ │ ├── sequence_conf.properties │ │ ├── sequence_db_conf.properties │ │ ├── sequence_distributed_conf.properties │ │ ├── sequence_http_conf.properties │ │ ├── sequence_time_conf.properties │ │ ├── server.dtd │ │ ├── server.xml │ │ ├── sharding-by-enum.txt │ │ ├── zkconf/ │ │ │ ├── auto-sharding-long.txt │ │ │ ├── auto-sharding-rang-mod.txt │ │ │ ├── autopartition-long.txt │ │ │ ├── cacheservice.properties │ │ │ ├── ehcache.xml │ │ │ ├── index_to_charset.properties │ │ │ ├── partition-hash-int.txt │ │ │ ├── partition-range-mod.txt │ │ │ ├── rule.xml │ │ │ ├── schema.xml │ │ │ ├── sequence_conf.properties │ │ │ ├── sequence_db_conf.properties │ │ │ ├── sequence_distributed_conf-mycat_fz_01.properties │ │ │ ├── sequence_distributed_conf.properties │ │ │ ├── sequence_time_conf-mycat_fz_01.properties │ │ │ ├── sequence_time_conf.properties │ │ │ ├── server-mycat_fz_01.xml │ │ │ ├── server.xml │ │ │ └── sharding-by-enum.txt │ │ └── zkdownload/ │ │ └── auto-sharding-long.txt │ └── test/ │ ├── java/ │ │ ├── demo/ │ │ │ ├── catlets/ │ │ │ │ └── MyHellowJoin.java │ │ │ └── test/ │ │ │ ├── ByteArrayToHexArray.java │ │ │ └── TestClass1.java │ │ └── io/ │ │ └── mycat/ │ │ ├── BufferPerformanceMain.java │ │ ├── ConfigInitializerTest.java │ │ ├── EchoBioServer.java │ │ ├── ExecutorTestMain.java │ │ ├── SimpleCachePool.java │ │ ├── VolatileTest.java │ │ ├── backend/ │ │ │ └── jdbc/ │ │ │ └── mongodb/ │ │ │ ├── MongoClientPropertyHelperTest.java │ │ │ └── MongoEmbeddedObjectProcessorTest.java │ │ ├── buffer/ │ │ │ ├── TestByteBufferArena.java │ │ │ ├── TestDirectByteBufferPool.java │ │ │ └── TestMycatMemoryAlloctor.java │ │ ├── cache/ │ │ │ ├── DefaultLayedCachePoolTest.java │ │ │ ├── EnCachePoolTest.java │ │ │ └── TestCachePoolPerformance.java │ │ ├── classload/ │ │ │ └── TestDynClassLoad.java │ │ ├── config/ │ │ │ └── ConfigTest.java │ │ ├── heartbeat/ │ │ │ ├── HeartbeatConfigForTest.java │ │ │ ├── HeartbeatContext.java │ │ │ └── HeartbeatStartup.java │ │ ├── memory/ │ │ │ └── unsafe/ │ │ │ ├── PlatformUtilSuite.java │ │ │ ├── array/ │ │ │ │ └── LongArraySuite.java │ │ │ ├── hash/ │ │ │ │ └── Murmur3_x86_32Suite.java │ │ │ ├── map/ │ │ │ │ ├── AbstractBytesToBytesMapSuite.java │ │ │ │ ├── BytesToBytesMapOffHeapSuite.java │ │ │ │ ├── BytesToBytesMapOnHeapSuite.java │ │ │ │ ├── MapSorterByValueTest.java │ │ │ │ └── UnsafeFixedWidthAggregationMapSuite.java │ │ │ ├── memory/ │ │ │ │ ├── MemoryManagerSuite.java │ │ │ │ ├── MycatMemoryTest.java │ │ │ │ ├── TaskMemoryManagerSuite.java │ │ │ │ ├── TestMemoryConsumer.java │ │ │ │ └── TestMemoryManager.java │ │ │ ├── row/ │ │ │ │ ├── UnsafeRowListTest.java │ │ │ │ └── UnsafeRowSuite.java │ │ │ ├── sort/ │ │ │ │ ├── HashPartitioner.java │ │ │ │ ├── TestTimSort.java │ │ │ │ ├── UnsafeExternalRowSorterTest.java │ │ │ │ ├── UnsafeExternalSorterRadixSortSuite.java │ │ │ │ ├── UnsafeExternalSorterSuite.java │ │ │ │ ├── UnsafeInMemorySorterRadixSortSuite.java │ │ │ │ └── UnsafeInMemorySorterSuite.java │ │ │ ├── storage/ │ │ │ │ ├── BlockManagerTest.java │ │ │ │ └── SerializerManagerTest.java │ │ │ └── types/ │ │ │ ├── CalendarIntervalSuite.java │ │ │ └── UTF8StringSuite.java │ │ ├── migrate/ │ │ │ └── MigrateUtilsTest.java │ │ ├── model/ │ │ │ ├── M1.java │ │ │ ├── M1Main.java │ │ │ ├── M2.java │ │ │ ├── M2Main.java │ │ │ └── TransferObject.java │ │ ├── mpp/ │ │ │ └── TestSorter.java │ │ ├── mysql/ │ │ │ ├── MySQLMessageTest.java │ │ │ └── ResultSetPacketParse.java │ │ ├── parser/ │ │ │ ├── ManagerParserTest.java │ │ │ ├── ManagerParserTestPerf.java │ │ │ ├── Performance.java │ │ │ ├── ServerParseTest.java │ │ │ ├── ServerParserTest.java │ │ │ ├── ServerParserTestPerf.java │ │ │ ├── TestEscapeProcess.java │ │ │ ├── druid/ │ │ │ │ ├── DruidSelectParserTest.java │ │ │ │ ├── DruidSequenceHandlerTest.java │ │ │ │ ├── DruidUpdateParserTest.java │ │ │ │ └── MycatSchemaStatVisitorTest.java │ │ │ ├── primitive/ │ │ │ │ └── TestFunctionParser.java │ │ │ └── util/ │ │ │ └── PairUtilTest.java │ │ ├── performance/ │ │ │ ├── AbstractMultiTreadBatchTester.java │ │ │ ├── GoodsInsertJob.java │ │ │ ├── RandomDataValueUtil.java │ │ │ ├── SimpleConPool.java │ │ │ ├── TestGlobalTableInsertPerf.java │ │ │ ├── TestInsertGlobalSeqPerf.java │ │ │ ├── TestInsertPerf.java │ │ │ ├── TestMaxConnection.java │ │ │ ├── TestMergeSelectPerf.java │ │ │ ├── TestMergeSorter.java │ │ │ ├── TestRandomDataUtil.java │ │ │ ├── TestSelectPerf.java │ │ │ ├── TestUpdatePerf.java │ │ │ ├── TravelRecordGlobalSeqInsertJob.java │ │ │ ├── TravelRecordInsertJob.java │ │ │ ├── TravelRecordMergeJob.java │ │ │ ├── TravelRecordSelectJob.java │ │ │ ├── TravelRecordUpdateJob.java │ │ │ ├── UserTableInsertJob.java │ │ │ └── UserTableSelectJob.java │ │ ├── postgres/ │ │ │ └── PostgresTest.java │ │ ├── queue/ │ │ │ ├── FixedQueue.java │ │ │ ├── Queue.java │ │ │ ├── QueuePerfMain.java │ │ │ └── QueueSimpleMain.java │ │ ├── route/ │ │ │ ├── .gitignore │ │ │ ├── DDLRouteTest.java │ │ │ ├── DQLRouteTest.java │ │ │ ├── DeleteSqlParseTest.java │ │ │ ├── DruidDb2SqlParserTest.java │ │ │ ├── DruidMysqlCreateTableTest.java │ │ │ ├── DruidMysqlHavingTest.java │ │ │ ├── DruidMysqlRouteStrategyTest.java │ │ │ ├── DruidMysqlSqlParserTest.java │ │ │ ├── DruidMysqlSqlSubqueriesParserTest.java │ │ │ ├── DruidOracleSqlParserTest.java │ │ │ ├── DruidPostgresqlSqlParserTest.java │ │ │ ├── DruidSqlServerSqlParserTest.java │ │ │ ├── HintDBTypeTest.java │ │ │ ├── HintTest.java │ │ │ ├── TestSelectBetweenSqlParser.java │ │ │ ├── function/ │ │ │ │ ├── AutoPartitionByLongTest.java │ │ │ │ ├── PartitionByCRC32PreSlotTest.java │ │ │ │ ├── PartitionByDateTest.java │ │ │ │ ├── PartitionByHashModTest.java │ │ │ │ ├── PartitionByHotDateTest.java │ │ │ │ ├── PartitionByJumpConsistentHashTest.java │ │ │ │ ├── PartitionByMonthTest.java │ │ │ │ ├── PartitionByPatternTest.java │ │ │ │ ├── PartitionByPrefixPatternTest.java │ │ │ │ ├── PartitionByRangeDateHashTest.java │ │ │ │ ├── PartitionByRangeModTest.java │ │ │ │ ├── PartitionByStringTest.java │ │ │ │ ├── RuleFunctionSuitTableTest.java │ │ │ │ ├── TestLatestMonthPartion.java │ │ │ │ └── TestNumberParseUtil.java │ │ │ ├── parser/ │ │ │ │ └── druid/ │ │ │ │ └── impl/ │ │ │ │ └── DefaultDruidParserTest.java │ │ │ ├── perf/ │ │ │ │ ├── NoShardingSpace.java │ │ │ │ ├── ShardingDefaultSpace.java │ │ │ │ └── ShardingMultiTableSpace.java │ │ │ └── util/ │ │ │ ├── PartitionForSingle.java │ │ │ ├── PartitionUtilTest.java │ │ │ └── RouterUtilTest.java │ │ ├── sequence/ │ │ │ ├── DistributedSequenceHandlerTest.java │ │ │ ├── IncrSequenceZKHandlerTest.java │ │ │ ├── SequenceHandlerTest.java │ │ │ └── SequenceTest.java │ │ ├── server/ │ │ │ ├── handler/ │ │ │ │ └── ServerHandlerTest.java │ │ │ └── interceptor/ │ │ │ └── impl/ │ │ │ └── GlobalTableUtilTest.java │ │ ├── sqlexecute/ │ │ │ ├── BaseSQLExeTest.java │ │ │ ├── MultiThreadSelectTest.java │ │ │ ├── MultiThreadSequnceTest.java │ │ │ ├── MycatMulitJdbcVersionTest.java │ │ │ ├── RollbackTest.java │ │ │ ├── ServerPrepareTest.java │ │ │ ├── StandBatchInsertTest.java │ │ │ ├── TestJdbc.java │ │ │ └── TestPrepareSql.java │ │ ├── statistic/ │ │ │ ├── SQLStatisticsMain.java │ │ │ └── TestConcurrentSafety.java │ │ └── util/ │ │ ├── ArrayPerformanceMain.java │ │ ├── BitTest.java │ │ ├── ConcurrentHashMapMain.java │ │ ├── HashMapMain.java │ │ ├── HexFormatUtilMain.java │ │ ├── HexFormatUtilTest.java │ │ ├── LockPerfMain.java │ │ ├── MapPerfMain.java │ │ ├── SchemaUtilTest.java │ │ ├── SmallSetTest.java │ │ ├── SplitUtilTest.java │ │ ├── StringHashPerfMain.java │ │ ├── StringUtilTest.java │ │ └── SyncPerfMain.java │ └── resources/ │ ├── autopartition-long-dupl.txt │ ├── autopartition-long.txt │ ├── autopartition-long2.txt │ ├── config/ │ │ ├── rule.xml │ │ └── schema.xml │ ├── ehcache.xml │ ├── log4j2.xml │ ├── partition-pattern.txt │ ├── partition-range-mod.txt │ ├── partition_prefix_pattern.txt │ ├── route/ │ │ ├── rule.xml │ │ └── schema.xml │ ├── rule.xml │ ├── schema.xml │ ├── sequence_conf.properties │ ├── server.xml │ ├── sharding.txt │ └── zk-create-test.yaml ├── test-output/ │ ├── Default suite/ │ │ ├── Default test.html │ │ └── Default test.xml │ ├── emailable-report.html │ ├── index.html │ ├── old/ │ │ ├── Default suite/ │ │ │ ├── Default test.properties │ │ │ ├── classes.html │ │ │ ├── groups.html │ │ │ ├── index.html │ │ │ ├── main.html │ │ │ ├── methods-alphabetical.html │ │ │ ├── methods-not-run.html │ │ │ ├── methods.html │ │ │ ├── reporter-output.html │ │ │ ├── testng.xml.html │ │ │ └── toc.html │ │ └── index.html │ ├── testng-reports.css │ ├── testng-reports.js │ ├── testng-results.xml │ └── testng.css ├── tmlogs/ │ └── tmlog-1.log ├── version.txt └── version.txt.template ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/ISSUE_TEMPLATE/bug-report.md ================================================ --- name: Bug report(默认) about: Create a report to help us improve title: '' labels: bug assignees: funnyAnt --- **1、bug描述** 简单清晰描述下bug现象. **2、版本号(非常重要)** v 1.6.*.* **3、相关表的配置信息** schema.xml (需包含表的配置信息，mysql的连接驱动是JDBC还是native方式) rule.xml (涉及到的路由函数) server.xml(可选) **4、操作步骤** 1. '...' 2. '....' 3. ... **5、期望结果** A clear and concise description of what you expected to happen. **6、实际结果** A clear and concise description of what actually happened. **7、额外信息** Add any other context about the problem here. ================================================ FILE: .github/ISSUE_TEMPLATE/new_feture.md ================================================ --- name: 功能建议 about: Suggest an idea for this project title: '' labels: enhancement assignees: funnyAnt, junwen12221 --- **问题描述** A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] **描述您想要的结果及解决方案** A clear and concise description of what you want to happen. **额外内容** Add any other context or screenshots about the feature request here. ================================================ FILE: .github/ISSUE_TEMPLATE/others.md ================================================ --- name: 其他（如使用方法，配置参数等） about: Describe this issue template's purpose here. title: '' labels: help wanted assignees: '' --- **描述你使用过程中遇到的问题: ** ================================================ FILE: .gitignore ================================================ ### Eclipse template *.pydevproject .metadata .gradle bin/ tmp/ *.tmp *.bak *.swp *~.nib local.properties .settings/ .loadpath # Eclipse Core .project # External tool builders .externalToolBuilders/ # Locally stored "Eclipse launch configurations" *.launch # CDT-specific .cproject # JDT-specific (Eclipse Java Development Tools) .classpath # Java annotation processor (APT) .factorypath # PDT-specific .buildpath # sbteclipse plugin .target # TeXlipse plugin .texlipse ### JetBrains template # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion *.iml ## Directory-based project format: .idea/ # if you remove the above rule, at least ignore the following: # User-specific stuff: # .idea/workspace.xml # .idea/tasks.xml # .idea/dictionaries # Sensitive or high-churn files: # .idea/dataSources.ids # .idea/dataSources.xml # .idea/sqlDataSources.xml # .idea/dynamic.xml # .idea/uiDesigner.xml # Gradle: # .idea/gradle.xml # .idea/libraries # Mongo Explorer plugin: # .idea/mongoSettings.xml ## File-based project format: *.ipr *.iws ## Plugin-specific files: # IntelliJ /out/ # mpeltonen/sbt-idea plugin .idea_modules/ # JIRA plugin atlassian-ide-plugin.xml # Crashlytics plugin (for Android Studio and IntelliJ) com_crashlytics_export_strings.xml crashlytics.properties crashlytics-build.properties ### Java template *.class # Mobile Tools for Java (J2ME) .mtj.tmp/ # Package Files # *.jar *.war *.ear # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* /project/project/ /project/target/ /project/activator-* /logs/ /RUNNING_PID .DS_Store /target/ /conf/ ================================================ FILE: .travis.yml ================================================ language: java jdk: - openjdk7 - oraclejdk7 ================================================ FILE: .vscode/launch.json ================================================ { // Use IntelliSense to learn about possible attributes. // Hover to view descriptions of existing attributes. // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ { "type": "java", "name": "Debug (Launch)-MycatStartup", "request": "launch", "mainClass": "io.mycat.MycatStartup", "projectName": "Mycat-server", "vmArgs": "-server -Xms2G -Xmx2G -XX:MaxMetaspaceSize=64M -XX:+AggressiveOpts -XX:MaxDirectMemorySize=2G -DMYCAT_HOME=${workspaceFolder} ", "args": "$" }, ] } ================================================ FILE: .vscode/settings.json ================================================ { "java.format.settings.url": "${workspaceFolder}/eclipse-java-google-style.xml" } ================================================ FILE: Dockerfile ================================================ FROM docker.io/adoptopenjdk/openjdk8:latest ADD http://dl.mycat.io/1.6.6.1/Mycat-server-1.6.6.1-release-20180908155252-linux.tar.gz /usr/local RUN cd /usr/local && tar -zxvf Mycat-server-1.6.6.1-release-20180908155252-linux.tar.gz && ls -lna VOLUME /usr/local/mycat/conf VOLUME /usr/local/mycat/logs EXPOSE 8066 9066 CMD ["/usr/local/mycat/bin/mycat", "console"] ================================================ FILE: LICENSE ================================================ GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Lesser General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. {description} Copyright (C) {year} {fullname} This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. Also add information on how to contact you by electronic and paper mail. If the program is interactive, make it output a short notice like this when it starts in an interactive mode: Gnomovision version 69, Copyright (C) year name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. {signature of Ty Coon}, 1 April 1989 Ty Coon, President of Vice This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. ================================================ FILE: README.md ================================================ # Mycat1 官网: http://mycatone.top [gitee](https://gitee.com/MycatOne/Mycat-Server) [github](https://github.com/MyCATApache/Mycat-Server) 提交代码,可以开issue,写清楚代码改动或者联系qq:1019100252 mycat1.6权威指南 [yuque](https://www.yuque.com/books/share/0576de75-ffc4-4c34-8586-952ae4636944) [pdf](http://mycat.org.cn/document/mycat-definitive-guide.pdf) 客户端连接配置 [gitee](https://gitee.com/MycatOne/Mycat-Server/wiki/%E5%BC%80%E5%8F%91%E7%AF%87%EF%BC%9A1.0-%E5%AE%A2%E6%88%B7%E7%AB%AF%E8%BF%9E%E6%8E%A5%E9%85%8D%E7%BD%AE) [github](https://github.com/MyCATApache/Mycat-Server/wiki/%E5%BC%80%E5%8F%91%E7%AF%87%EF%BC%9A1.0-%E5%AE%A2%E6%88%B7%E7%AB%AF%E8%BF%9E%E6%8E%A5%E9%85%8D%E7%BD%AE) [![Stargazers over time](https://starchart.cc/MyCATApache/Mycat-Server.svg)](https://starchart.cc/MyCATApache/Mycat-Server) Mycat志愿者开发群:332702697,106088787 [![GitHub issues](https://img.shields.io/github/issues/MyCATApache/Mycat-Server.svg)](https://github.com/MyCATApache/Mycat-Server/issues) [![GitHub forks](https://img.shields.io/github/forks/MyCATApache/Mycat-Server.svg)](https://github.com/MyCATApache/Mycat-Server/network) [![GitHub stars](https://img.shields.io/github/stars/MyCATApache/Mycat-Server.svg)](https://github.com/MyCATApache/Mycat-Server/stargazers) [![MyCAT](https://img.shields.io/badge/MyCAT-%E2%9D%A4%EF%B8%8F-%23ff69b4.svg)](http://mycat.io/) MyCAT is an Open-Source software, “a large database cluster” oriented to enterprises. MyCAT is an enforced database which is a replacement for MySQL and supports transaction and ACID. Regarded as MySQL cluster of enterprise database, MyCAT can take the place of expensive Oracle cluster. MyCAT is also a new type of database, which seems like a SQL Server integrated with the memory cache technology, NoSQL technology and HDFS big data. And as a new modern enterprise database product, MyCAT is combined with the traditional database and new distributed data warehouse. In a word, MyCAT is a fresh new middleware of database. Mycat’s target is to smoothly migrate the current stand-alone database and applications to cloud side with low cost and to solve the bottleneck problem caused by the rapid growth of data storage and business scale. 2020年1月1日合拼了一个PR,优化PartionByLong的分片算法,数据不均衡的问题,所以该分片算法与此前的PartionByLong的数据分布不一致,即1.675之后与之前的版本不兼容 MyCAT1.6不支持一个SQL包含多个语句 旧Mycat升级fastjson，把pom.xml中fastjson的版本更改即可 1.6的bug: 批处理插入,多语句,堆外合拼,请大家要避开这些功能 全局序列号语法 ```sql INSERT INTO `travelrecord` (`id`,user_id) VALUES ('next value for MYCATSEQ_GLOBAL',"xxx"); ``` 更新Druid 1.1.10版本的分支独立维护在 [gitee](https://gitee.com/MycatOne/Mycat-Server/tree/1.6.6-druid) [github](https://github.com/MyCATApache/Mycat-Server/tree/1.6.6-druid) * Getting Started * [gitee](https://gitee.com/mirrors_MyCATApache/Mycat-doc/tree/master/en) * [github](https://github.com/MyCATApache/Mycat-doc/tree/master/en) * 尝试 MyCAT * [gitee](https://gitee.com/mirrors_MyCATApache/Mycat-doc/tree/master/%E5%85%A5%E9%97%A8%E6%8C%87%E5%8D%97) * [github](https://github.com/MyCATApache/Mycat-doc/tree/master/%E5%85%A5%E9%97%A8%E6%8C%87%E5%8D%97) ## Features * Supports SQL 92 standard * Supports MySQL cluster, used as a Proxy * Supports JDBC connection with ORACLE, DB2, SQL Server, simulated as normal MySQL Server connection * Supports MySQL cluster, percona cluster or mariadb cluster, providing high availability of data fragmentation clusters * Supports automatic failover and high availability * Supports separation of read and write, dual-master with multi-slave, single-master with multi-master of MySQL model * Supports global table, automatically fragment data into multiple nodes for efficient relational query * Supports the unique fragmentation strategy based on ER-relation for efficient relational query * Supports multiple platforms, easy deployment and implementation ## Advantage * Based on Alibaba's open-source project Cobar [github](https://github.com/alibaba/cobar),[gitee](https://gitee.com/mirrors_alibaba/cobar), whose stability, reliability, excellent architecture and performance, as well as many mature use-cases make MyCAT have a good starting. Standing on the shoulders of giants, MyCAT feels confident enough to go farther. * Extensively drawing on the best open-source projects and innovative ideas, which are integrated into the Mycat’s gene, make MyCAT be ahead of the other current similar open-source projects, even beyond some commercial products. * MyCAT behind a strong technical team whose participants are experienced more than five years including some senior software engineer, architect, DBA, etc. Excellent technical team to ensure the product quality of Mycat. * MyCAT does not rely on any commercial company. It’s unlike some open-source projects whose important features is enclosed in its commercial products and making open-source projects like a decoration. ## Roadmap * On the basis of MySQL’s support, MyCAT add more support of commercial open-source database, including native support of PostgreSQL, FireBird and other open-source databases, as well as indirect support via JDBC of other non-open-source databases such as Oracle, DB2, SQL Server etc. * More intelligent self-regulating properties, such as automatic statistical analysis of SQL, automatic creating and adjusting indexes. Based on the frequency of read and write, MyCAT automatically optimizes caching and backup strategies * Achieve a more comprehensive monitoring and management * Integrated with HDFS, provide SQL commands, load databases into HDFS for rapid analysis * Integrated excellent open-source reporting tools to make MyCAT have data analysis capability ## Download There are some compiled binary installation packages in Mycat-download project on github at [Mycat-download](http://dl.mycat.org.cn/1.6.7.6/). ## Document There are some documents in Mycat-doc project on github at [Mycat-doc] [gitee](https://gitee.com/mirrors_MyCATApache/Mycat-doc) [github](https://github.com/MyCATApache/Mycat-doc) Mycat 简单demo，具体参考Mycat权威指南 # Mycat前世今生 2013年阿里的Cobar在社区使用过程中发现存在一些比较严重的问题，及其使用限制，经过Mycat发起人第一次改良，第一代改良版——Mycat诞生。 Mycat开源以后，一些Cobar的用户参与了Mycat的开发，最终Mycat发展成为一个由众多软件公司的实力派架构师和资深开发人员维护的社区型开源软件。 2014年Mycat首次在上海的《中华架构师》大会上对外宣讲，更多的人参与进来，随后越来越多的项目采用了Mycat。 2015年5月，由核心参与者们一起编写的第一本官方权威指南《Mycat权威指南》电子版发布，累计超过500本，成为开源项目中的首创。 2015年10月为止，Mycat项目总共有16个Committer。 截至2015年11月，超过300个项目采用Mycat，涵盖银行、电信、电子商务、物流、移动应用、O2O的众多领域和公司。 截至2015年12月，超过4000名用户加群或研究讨论或测试或使用Mycat。 Mycat是基于开源cobar演变而来，我们对cobar的代码进行了彻底的重构，使用NIO重构了网络模块，并且优化了Buffer内核，增强了聚合，Join等基本特性，同时兼容绝大多数数据库成为通用的数据库中间件。1.4 版本以后 完全的脱离基本cobar内核，结合Mycat集群管理、自动扩容、智能优化，成为高性能的中间件。我们致力于开发高性能数据库中间而努力。永不收费，永不闭源，持续推动开源社区的发展。 Mycat吸引和聚集了一大批业内大数据和云计算方面的资深工程师，Mycat的发展壮大基于开源社区志愿者的持续努力，感谢社区志愿者的努力让Mycat更加强大，同时我们也欢迎社区更多的志愿者，特别是公司能够参与进来，参与Mycat的开发，一起推动社区的发展，为社区提供更好的开源中间件。 Mycat还不够强大，Mycat还有很多不足，欢迎社区志愿者的持续优化改进。 # 关键特性 支持SQL92标准 遵守Mysql原生协议，跨语言，跨平台，跨数据库的通用中间件代理。 基于心跳的自动故障切换，支持读写分离，支持MySQL主从，以及galera cluster集群。 支持Galera for MySQL集群，Percona Cluster或者MariaDB cluster 基于Nio实现，有效管理线程，高并发问题。 支持数据的多片自动路由与聚合，支持sum,count,max等常用的聚合函数。 支持单库内部任意join，支持跨库2表join，甚至基于caltlet的多表join。 支持通过全局表，ER关系的分片策略，实现了高效的多表join查询。 支持多租户方案。 支持分布式事务（弱xa）。 支持全局序列号，解决分布式下的主键生成问题。 分片规则丰富，插件化开发，易于扩展。 强大的web，命令行监控。 支持前端作为mysq通用代理，后端JDBC方式支持Oracle、DB2、SQL Server 、 mongodb 、巨杉。 支持密码加密 支持服务降级 支持IP白名单 支持SQL黑名单、sql注入攻击拦截 支持分表（1.6） 集群基于ZooKeeper管理，在线升级，扩容，智能优化，大数据处理（2.0开发版）。 # Mycat安装与使用 ## 下载： 具体下载哪个版本以发布为准，推荐1.67. ## 安装： 下载的文件直接解压即可。 ## 运行： ### linux： ./mycat start 启动 ./mycat stop 停止 ./mycat console 前台运行 ./mycat install 添加到系统自动启动（暂未实现） ./mycat remove 取消随系统自动启动（暂未实现） ./mycat restart 重启服务 ./mycat pause 暂停 ./mycat status 查看启动状态 ### win： 直接运行startup_nowrap.bat，如果出现闪退，在cmd 命令行运行，查看出错原因。 ## 内存配置： 启动前，一般需要修改JVM配置参数，打开conf/wrapper.conf文件，如下行的内容为2G和2048，可根据本机配置情况修改为512M或其它值。 以下配置跟jvm参数完全一致，可以根据自己的jvm参数调整。 Java Additional Parameters wrapper.java.additional.1= wrapper.java.additional.1=-DMYCAT_HOME=. wrapper.java.additional.2=-server #wrapper.java.additional.3=-XX:MaxPermSize=64M wrapper.java.additional.4=-XX:+AggressiveOpts wrapper.java.additional.5=-XX:MaxDirectMemorySize=100m wrapper.java.additional.6=-Dcom.sun.management.jmxremote wrapper.java.additional.7=-Dcom.sun.management.jmxremote.port=1984 wrapper.java.additional.8=-Dcom.sun.management.jmxremote.authenticate=false wrapper.java.additional.9=-Dcom.sun.management.jmxremote.ssl=false wrapper.java.additional.10=-Xmx100m wrapper.java.additional.11=-Xms100m wrapper.java.additional.12=-XX:+UseParNewGC wrapper.java.additional.13=-XX:+UseConcMarkSweepGC wrapper.java.additional.14=-XX:+UseCMSCompactAtFullCollection wrapper.java.additional.15=-XX:CMSFullGCsBeforeCompaction=0 wrapper.java.additional.16=-XX:CMSInitiatingOccupancyFraction=70 以下配置作废： wrapper.java.initmemory=3 wrapper.java.maxmemory=64 ### Mycat连接测试： 测试mycat与测试mysql完全一致，mysql怎么连接，mycat就怎么连接。 推荐先采用命令行测试： mysql -uroot -proot -P8066 -h127.0.0.1 如果采用工具连接，1.4,1.3目前部分工具无法连接，会提示database not selected，建议采用高版本，navicat测试。1.5已经修复了部分工具连接。 # Mycat配置入门 ## 配置： --bin 启动目录 --conf 配置目录存放配置文件： --server.xml：是Mycat服务器参数调整和用户授权的配置文件。 --schema.xml：是逻辑库定义和表以及分片定义的配置文件。 --rule.xml： 是分片规则的配置文件，分片规则的具体一些参数信息单独存放为文件，也在这个目录下，配置文件修改需要重启MyCAT。 --log4j.xml： 日志存放在logs/log中，每天一个文件，日志的配置是在conf/log4j.xml中，根据自己的需要可以调整输出级别为debug debug级别下，会输出更多的信息，方便排查问题。 --autopartition-long.txt,partition-hash-int.txt,sequence_conf.properties， sequence_db_conf.properties 分片相关的id分片规则配置文件 --lib MyCAT自身的jar包或依赖的jar包的存放目录。 --logs MyCAT日志的存放目录。日志存放在logs/log中，每天一个文件 下面图片描述了Mycat最重要的3大配置文件：

## 逻辑库配置： ### 配置server.xml 添加两个mycat逻辑库：user,pay system 参数是所有的mycat参数配置，比如添加解析器：defaultSqlParser，其他类推 user 是用户参数。 druidparser mycat user,pay ### 编辑schema.xml 修改dataHost和schema对应的连接信息，user,pay 垂直切分后的配置如下所示： schema 是实际逻辑库的配置，user，pay分别对应两个逻辑库，多个schema代表多个逻辑库。 dataNode是逻辑库对应的分片，如果配置多个分片只需要多个dataNode即可。 dataHost是实际的物理库配置地址，可以配置多主主从等其他配置，多个dataHost代表分片对应的物理库地址，下面的writeHost、readHost代表该分片是否配置多写，主从，读写分离等高级特性。 以下例子配置了两个writeHost为主从。 select 1 ​ # Mycat逻辑库、系统参数配置 ## 配置Mycat环境参数 druidparser 如例子中配置的所有的Mycat参数变量都是配置在server.xml 文件中，system标签下配置所有的参数，如果需要配置某个变量添加相应的配置即可，例如添加启动端口8066，默认为8066： 8066 其他所有变量类似。 ## 配置Mycat逻辑库与用户 mycat TESTDB 如例子中配置的所有的Mycat连接的用户与逻辑库映射都是配置在server.xml 文件中，user标签下配置所有的参数，例如例子中配置了一个mycat用户供应用连接到mycat，同时mycat 在schema.xml中配置后了一个逻辑库TESTDB，配置好逻辑库与用户的映射关系。 # 逻辑库、表分片配置 ## 配置逻辑库（schema） Mycat作为一个中间件，实现mysql协议，那么对前端应用连接来说就是一个数据库，也就有数据库的配置，mycat的数据库配置是在schema.xml中配置，配置好后映射到server.xml里面的用户就可以了。
show status like 'wsrep%' 上面例子配置了一个逻辑库TESTDB，同时配置了t_user，ht_jy_login_log两个分片表。 ### 逻辑表配置
table 标签 是逻辑表的配置 其中 name代表表名， dataNode代表表对应的分片， Mycat默认采用分库方式，也就是一个表映射到不同的库上， rule代表表要采用的数据切分方式，名称对应到rule.xml中的对应配置，如果要分片必须配置。 ## 配置分片（dataNode） 表切分后需要配置映射到哪几个数据库中，Mycat的分片实际上就是库的别名，例如上面例子配置了两个分片dn1，dn2 分别对应到物理机映射dataHost localhost1 的两个库上。 ## 配置物理库分片映射（dataHost） show status like 'wsrep%' Mycat作为数据库代理需要逻辑库，逻辑用户，表切分后需要配置分片，分片也就需要映射到真实的物理主机上，至于是映射到一台还是一台的多个实例上，Mycat并不关心，只需要配置好映射即可，例如例子中： 配置了一个名为localhost1的物理主机（dataHost）映射。 heartbeat 标签代表Mycat需要对物理库心跳检测的语句，正常情况下生产案例可能配置主从，或者多写 或者单库，无论哪种情况Mycat都需要维持到数据库的数据源连接，因此需要定时检查后端连接可以性，心跳语句就是来作为心跳检测。 writeHost 此标签代表 一个逻辑主机（dataHost）对应的后端的物理主机映射，例如例子中写库hostM1 映射到127.0.0.1:3306。如果后端需要做读写分离或者多写 或者主从则通过配置 多个writeHost 或者readHost即可。 dataHost 标签中的 writeType balance 等标签则是不同的策略，具体参考指南。 # Mycat 表切分规则配置 ## 表切分规则 createTime sharding-by-hour 24 数据切分中作为表切分规则中最重要的配置，表的切分方式决定了数据切分后的性能好坏，因此也是最重要的配置。 如上面例子配置了一个切分规则，名为sharding-by-hour 对应的切分方式（function ）是按日期切分，该配置中： ### tableRule name 为schema.xml 中table 标签中对应的 rule="sharding-by-hour" ,也就是配置表的分片规则， columns 是表的切分字段： createTime 创建日期。 algorithm 是规则对应的切分规则：映射到function 的name。 ### function function 配置是分片规则的配置。 name 为切分规则的名称，名字任意取，但是需要与tableRule 中匹配。 class 是切分规则对应的切分类，写死，需要哪种规则则配置哪种，例如本例子是按小时分片：org.opencloudb.route.function.LatestMonthPartion property 标签是切分规则对应的不同属性，不同的切分规则配置不同。 ================================================ FILE: README_Chinese.md ================================================ ### Mycat介绍 ### 官网:[http://www.mycat.org.cn](http://www.mycat.org.cn) ### github:[https://github.com/MyCATApache](https://github.com/MyCATApache) ##### 入门: [zh-CN: https://github.com/MyCATApache/Mycat-doc/blob/master/history/MyCat_In_Action_%E4%B8%AD%E6%96%87%E7%89%88.doc] [English:https://github.com/MyCATApache/Mycat-doc/tree/master/en] 什么是Mycat？简单的说，Mycat就是： * 一个彻底开源的，面向企业应用开发的“大数据库集群” * 支持事务、ACID、可以替代MySQL的加强版数据库 * 一个可以视为“MySQL”集群的企业级数据库，用来替代昂贵的Oracle集群 * 一个融合内存缓存技术、Nosql技术、HDFS大数据的新型SQL Server * 结合传统数据库和新型分布式数据仓库的新一代企业级数据库产品 * 一个新颖的数据库中间件产品 ##### Mycat的目标是： 低成本的将现有的单机数据库和应用平滑迁移到“云”端，解决数据存储和业务规模迅速增长情况下的数据瓶颈问题。 ##### Mycat的关键特性： * 支持 SQL 92标准 * 支持MySQL集群，可以作为Proxy使用 * 支持JDBC连接ORACLE、DB2、SQL Server，将其模拟为MySQL Server使用 * 支持galera for MySQL集群，percona-cluster或者mariadb cluster，提供高可用性数据分片集群 * 自动故障切换，高可用性 * 支持读写分离，支持MySQL双主多从，以及一主多从的模式 * 支持全局表，数据自动分片到多个节点，用于高效表关联查询 * 支持独有的基于E-R 关系的分片策略，实现了高效的表关联查询 * 多平台支持，部署和实施简单 ##### Mycat的优势： * 基于阿里开源的Cobar产品而研发，Cobar的稳定性、可靠性、优秀的架构和性能，以及众多成熟的使用案例使得Mycat一开始就拥有一个很好的起点，站在巨人的肩膀上，我们能看到更远。 * 广泛吸取业界优秀的开源项目和创新思路，将其融入到Mycat的基因中，使得Mycat在很多方面都领先于目前其他一些同类的开源项目，甚至超越某些商业产品。 * Mycat背后有一只强大的技术团队，其参与者都是5年以上资深软件工程师、架构师、DBA等，优秀的技术团队保证了Mycat的产品质量。 * Mycat并不依托于任何一个商业公司，因此不像某些开源项目，将一些重要的特性封闭在其商业产品中，使得开源项目成了一个摆设。 ##### Mycat的长期路线规划： * 在支持MySQL的基础上，后端增加更多的开源数据库和商业数据库的支持，包括原生支持PosteSQL、FireBird等开源数据库，以及通过JDBC等方式间接支持其他非开源的数据库如Oracle、DB2、SQL Server等 * 实现更为智能的自我调节特性，如自动统计分析SQL，自动创建和调整索引，根据数据表的读写频率，自动优化缓存和备份策略等 * 实现更全面的监控管理功能 * 与HDFS集成，提供SQL命令，将数据库装入HDFS中并能够快速分析 * 集成优秀的开源报表工具，使之具备一定的数据分析的能力 ##### 下载： github上面的Mycat-download项目是编译好的二进制安装包 [https://github.com/MyCATApache/Mycat-download](https://github.com/MyCATApache/Mycat-download) ##### 文档： github上面的Mycat-doc项目是相关文档 [https://github.com/MyCATApache/Mycat-doc](https://github.com/MyCATApache/Mycat-doc) ================================================ FILE: catlet/readme.txt ================================================ put your customer Catlet class files in this dir ================================================ FILE: eclipse-java-google-style.xml ================================================ ================================================ FILE: intellij-java-google-style.xml ================================================ ================================================ FILE: pom.xml ================================================ 4.0.0 io.mycat Mycat-server 1.6.7.6-release jar Mycat-server The project of Mycat-server http://io.mycat UTF-8 yyyy-MM-dd HH:mm:ss version.txt.template version.txt scm:git:ssh://apachemycat@github.com/MyCATApache/Mycat-Server.git scm:git:ssh://apachemycat@github.com/MyCATApache/Mycat-Server.git scm:git:ssh://apachemycat@github.com/MyCATApache/Mycat-Server.git org.mongodb mongo-java-driver 3.11.0 org.iq80.leveldb leveldb 0.7 org.iq80.leveldb leveldb-api 0.7 com.google.guava guava 28.2-jre com.alibaba druid 1.2.6 mysql mysql-connector-java 5.1.35 net.sf.ehcache ehcache-core 2.6.11 compile org.mapdb mapdb 1.0.7 junit junit 4.4 provided org.apache.velocity velocity 1.7 org.codehaus.jsr166-mirror jsr166y 1.7.0 test com.lmax disruptor 3.3.4 org.apache.logging.log4j log4j-slf4j-impl 2.17.0 org.apache.logging.log4j log4j-core 2.17.1 org.apache.logging.log4j log4j-1.2-api 2.17.0 com.univocity univocity-parsers 2.8.4 jar com.sequoiadb sequoiadb-driver 1.12 dom4j dom4j 1.6.1 xml-apis xml-apis org.apache.curator curator-framework 4.0.1 org.apache.curator curator-recipes 4.0.1 org.apache.curator curator-test 4.0.1 test log4j log4j com.alibaba fastjson 1.2.68 joda-time joda-time 2.9.3 com.github.shyiko mysql-binlog-connector-java 0.16.1 org.mockito mockito-all 1.8.5 test com.google.code.findbugs jsr305 2.0.3 com.esotericsoftware.kryo kryo 2.10 org.apache.commons commons-csv 1.8 org.hamcrest hamcrest-library 1.3 commons-lang commons-lang 2.6 io.netty netty-buffer 4.1.9.Final com.squareup.okhttp3 okhttp 4.2.2 JIRA http://io.mycat src/main/resources **/.svn/** ${basedir} ${version.file} src/test/resources **/.svn/** org.eclipse.m2e lifecycle-mapping 1.0.0 com.google.code.maven-replacer-plugin replacer [1.0.0,) replace com.google.code.maven-replacer-plugin replacer 1.5.3 version process-sources replace ${project.basedir}/${version.template.file} ${project.basedir}/${version.file} @buildnumber@ ${buildNumber} @buildtime@ ${maven.build.timestamp} @pomversion@ ${project.version} @giturl@ https://github.com/MyCATApache/Mycat-Server.git @mycatsite@ http://www.mycat.org.cn @qqgroup@ 106088787 version2 process-sources replace ${project.basedir}/src/main/java/io/mycat/config/Versions.template ${project.basedir}/src/main/java/io/mycat/config/Versions.java @server-version@ 5.6.29-mycat-${project.version}-${timestamp} org.apache.maven.plugins maven-compiler-plugin 1.8 1.8 ${app.encoding} org.apache.maven.plugins maven-source-plugin 2.1.2 ${app.encoding} true attach-sources jar-no-fork org.apache.maven.plugins maven-jar-plugin **/.svn/** test-jar org.apache.maven.plugins maven-eclipse-plugin **/.svn/** true classes .settings/org.eclipse.core.resources.prefs =${app.encoding}${line.separator}]]> org.codehaus.mojo appassembler-maven-plugin 1.8 conf true flat true mycat io.mycat.MycatStartup start jsw 128 MYCAT_HOME=. -server -XX:+AggressiveOpts -XX:MaxDirectMemorySize=2G -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=1984 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Xmx4G -Xms1G jsw aix-ppc-32 aix-ppc-64 hpux-parisc-64 linux-x86-32 linux-x86-64 linux-ppc-64 macosx-ppc-32 macosx-x86-universal-32 macosx-universal-32 macosx-universal-64 solaris-sparc-32 solaris-sparc-64 solaris-x86-32 windows-x86-32 windows-x86-64 configuration.directory.in.classpath.first conf wrapper.ping.timeout 120 set.default.REPO_DIR lib wrapper.logfile.maxsize 512m wrapper.logfile.maxfiles 30 wrapper.logfile logs/wrapper.log generate-jsw package generate-daemons maven-assembly-plugin src/main/assembly/assembly-win.xml src/main/assembly/assembly-linux.xml src/main/assembly/assembly-mac.xml src/main/assembly/assembly-solaris.xml src/main/assembly/assembly-unix.xml src/main/assembly/assembly-testtool.xml make-assembly package single org.apache.maven.plugins maven-scm-plugin 1.11.2 jgit org.apache.maven.scm maven-scm-provider-jgit 1.11.2 org.codehaus.mojo buildnumber-maven-plugin 1.4 validate create {1} timestamp scmVersion false false {0,date,yyyyMMddHHmmss} git ================================================ FILE: src/main/assembly/assembly-linux.xml ================================================ ${timestamp}-linux false tar.gz target/generated-resources/appassembler/jsw/mycat/bin mycat/bin 0755 mycat wrapper-linux* target/generated-resources/appassembler/jsw/mycat/lib mycat/lib *.jar libwrapper-linux* target/generated-resources/appassembler/jsw/mycat/conf mycat/conf * src/main/resources mycat/conf *.dtd log4j* ${basedir} mycat/ version.txt src/main/assembly/conf mycat/conf src/main/assembly/bin mycat/bin 0755 *.sh ${basedir}/logs mycat/logs **/* ${basedir}/catlet mycat/catlet **/* ================================================ FILE: src/main/assembly/assembly-mac.xml ================================================ ${timestamp}-mac false tar.gz target/generated-resources/appassembler/jsw/mycat/bin mycat/bin 0755 mycat wrapper-macosx* target/generated-resources/appassembler/jsw/mycat/lib mycat/lib *.jar libwrapper-macosx* target/generated-resources/appassembler/jsw/mycat/conf mycat/conf * src/main/resources mycat/conf *.dtd log4j* src/main/assembly/conf mycat/conf ${basedir} mycat/ version.txt src/main/assembly/bin mycat/bin 0755 *.sh ${basedir}/logs mycat/logs **/* ${basedir}/catlet mycat/catlet **/* ================================================ FILE: src/main/assembly/assembly-solaris.xml ================================================ ${timestamp}-solaris false tar.gz target/generated-resources/appassembler/jsw/mycat/bin mycat/bin 0755 mycat wrapper-solaris* target/generated-resources/appassembler/jsw/mycat/lib mycat/lib *.jar libwrapper-solaris* target/generated-resources/appassembler/jsw/mycat/conf mycat/conf * src/main/resources mycat/conf *.dtd log4j* src/main/assembly/conf mycat/conf ${basedir} mycat/ version.txt src/main/assembly/bin mycat/bin 0755 *.sh ${basedir}/logs mycat/logs **/* ${basedir}/catlet mycat/catlet **/* ================================================ FILE: src/main/assembly/assembly-testtool.xml ================================================ false ${timestamp}-testtool tar.gz src/main/assembly/testtool mycat/bin **/* 0755 ${basedir} mycat/ version.txt target mycat/lib **tests.jar test mycat/lib mysql-con* junit* ================================================ FILE: src/main/assembly/assembly-unix.xml ================================================ ${timestamp}-unix false tar.gz target/generated-resources/appassembler/jsw/mycat/bin mycat/bin 0755 mycat wrapper-aix* wrapper-hpux* target/generated-resources/appassembler/jsw/mycat/lib mycat/lib *.jar libwrapper-linux* target/generated-resources/appassembler/jsw/mycat/conf mycat/conf * src/main/resources mycat/conf *.dtd log4j* src/main/assembly/conf mycat/conf ${basedir} mycat/ version.txt src/main/assembly/bin mycat/bin 0755 *.sh ${basedir}/logs mycat/logs **/* ${basedir}/catlet mycat/catlet **/* ================================================ FILE: src/main/assembly/assembly-win.xml ================================================ ${timestamp}-win false tar.gz target/generated-resources/appassembler/jsw/mycat/bin mycat/bin 0755 mycat.bat wrapper-windows* target/generated-resources/appassembler/jsw/mycat/lib mycat/lib *.jar wrapper-windows* target/generated-resources/appassembler/jsw/mycat/conf mycat/conf * src/main/resources mycat/conf *.dtd log4j* src/main/assembly/conf mycat/conf ${basedir} mycat/ version.txt src/main/assembly/bin mycat/bin 0755 *.bat ${basedir}/logs mycat/logs **/* ${basedir}/catlet mycat/catlet **/* ================================================ FILE: src/main/assembly/conf/log4j2.xml ================================================ %d{yyyy-MM-dd HH:mm:ss.SSS} %5p [%t] (%l) - %m%n ================================================ FILE: src/main/assembly/testtool/test_globalseq_insert_perf.bat ================================================ REM check JAVA_HOME & java set "JAVA_CMD="%JAVA_HOME%/bin/java"" if "%JAVA_HOME%" == "" goto noJavaHome if exist "%JAVA_HOME%\bin\java.exe" goto mainEntry :noJavaHome echo --------------------------------------------------- echo WARN: JAVA_HOME environment variable is not set. echo --------------------------------------------------- set "JAVA_CMD=java" :mainEntry REM set HOME_DIR set "CURR_DIR=%cd%" cd .. set "MYCAT_HOME=%cd%" cd %CURR_DIR% "%JAVA_CMD%" -Xms256M -Xmx1G -XX:MaxPermSize=64M -DMYCAT_HOME=%MYCAT_HOME% -cp "..\conf;..\lib\*" io.mycat.performance.TestInsertPerf %1 %2 %3 %4 %5 ================================================ FILE: src/main/assembly/testtool/test_globaltable_insert_perf.bat ================================================ REM check JAVA_HOME & java set "JAVA_CMD=%JAVA_HOME%/bin/java" if "%JAVA_HOME%" == "" goto noJavaHome if exist "%JAVA_HOME%\bin\java.exe" goto mainEntry :noJavaHome echo --------------------------------------------------- echo WARN: JAVA_HOME environment variable is not set. echo --------------------------------------------------- set "JAVA_CMD=java" :mainEntry REM set HOME_DIR set "CURR_DIR=%cd%" cd .. set "MYCAT_HOME=%cd%" cd %CURR_DIR% "%JAVA_CMD%" -Xms256M -Xmx1G -XX:MaxPermSize=64M -DMYCAT_HOME=%MYCAT_HOME% -cp "..\conf;..\lib\*" io.mycat.performance.TestGlobalTableInsertPerf %1 %2 %3 %4 %5 ================================================ FILE: src/main/assembly/testtool/test_globaltable_insert_perf.sh ================================================ #!/bin/bash echo "check JAVA_HOME & java" JAVA_CMD=$JAVA_HOME/bin/java MAIN_CLASS=io.mycat.performance.TestGlobalTableInsertPerf if [ ! -d "$JAVA_HOME" ]; then echo --------------------------------------------------- echo WARN: JAVA_HOME environment variable is not set. echo --------------------------------------------------- JAVA_CMD=java fi echo "---------set HOME_DIR------------" CURR_DIR=`pwd` cd .. MYCAT_HOME=`pwd` cd $CURR_DIR $JAVA_CMD -Xms256M -Xmx1G -XX:MaxPermSize=64M -DMYCAT_HOME=$MYCAT_HOME -cp "$MYCAT_HOME/conf:$MYCAT_HOME/lib/*" $MAIN_CLASS $1 $2 $3 $4 $5 ================================================ FILE: src/main/assembly/testtool/test_stand_insert_perf.bat ================================================ REM check JAVA_HOME & java set "JAVA_CMD=%JAVA_HOME%/bin/java" if "%JAVA_HOME%" == "" goto noJavaHome if exist "%JAVA_HOME%\bin\java.exe" goto mainEntry :noJavaHome echo --------------------------------------------------- echo WARN: JAVA_HOME environment variable is not set. echo --------------------------------------------------- set "JAVA_CMD=java" :mainEntry REM set HOME_DIR set "CURR_DIR=%cd%" cd .. set "MYCAT_HOME=%cd%" cd %CURR_DIR% "%JAVA_CMD%" -Xms256M -Xmx1G -XX:MaxPermSize=64M -DMYCAT_HOME=%MYCAT_HOME% -cp "..\conf;..\lib\*" io.mycat.performance.TestInsertPerf %1 %2 %3 %4 %5 ================================================ FILE: src/main/assembly/testtool/test_stand_insert_perf.sh ================================================ #!/bin/bash echo "check JAVA_HOME & java" JAVA_CMD=$JAVA_HOME/bin/java MAIN_CLASS=io.mycat.performance.TestInsertPerf if [ ! -d "$JAVA_HOME" ]; then echo --------------------------------------------------- echo WARN: JAVA_HOME environment variable is not set. echo --------------------------------------------------- JAVA_CMD=java fi echo "---------set HOME_DIR------------" CURR_DIR=`pwd` cd .. MYCAT_HOME=`pwd` cd $CURR_DIR $JAVA_CMD -Xms256M -Xmx1G -XX:MaxPermSize=64M -DMYCAT_HOME=$MYCAT_HOME -cp "$MYCAT_HOME/conf:$MYCAT_HOME/lib/*" $MAIN_CLASS $1 $2 $3 $4 $5 ================================================ FILE: src/main/assembly/testtool/test_stand_merge_sel_perf.bat ================================================ REM check JAVA_HOME & java set "JAVA_CMD=%JAVA_HOME%/bin/java" if "%JAVA_HOME%" == "" goto noJavaHome if exist "%JAVA_HOME%\bin\java.exe" goto mainEntry :noJavaHome echo --------------------------------------------------- echo WARN: JAVA_HOME environment variable is not set. echo --------------------------------------------------- set "JAVA_CMD=java" :mainEntry REM set HOME_DIR set "CURR_DIR=%cd%" cd .. set "MYCAT_HOME=%cd%" cd %CURR_DIR% "%JAVA_CMD%" -Xms256M -Xmx1G -XX:MaxPermSize=64M -DMYCAT_HOME=%MYCAT_HOME% -cp "..\conf;..\lib\*" io.mycat.performance.TestMergeSelectPerf %1 %2 %3 %4 %5 %6 %7 ================================================ FILE: src/main/assembly/testtool/test_stand_merge_sel_perf.sh ================================================ #!/bin/bash echo "check JAVA_HOME & java" JAVA_CMD=$JAVA_HOME/bin/java MAIN_CLASS=io.mycat.performance.TestMergeSelectPerf if [ ! -d "$JAVA_HOME" ]; then echo --------------------------------------------------- echo WARN: JAVA_HOME environment variable is not set. echo --------------------------------------------------- JAVA_CMD=java fi echo "---------set HOME_DIR------------" CURR_DIR=`pwd` cd .. MYCAT_HOME=`pwd` cd $CURR_DIR $JAVA_CMD -Xms256M -Xmx1G -XX:MaxPermSize=64M -DMYCAT_HOME=$MYCAT_HOME -cp "$MYCAT_HOME/conf:$MYCAT_HOME/lib/*" $MAIN_CLASS $1 $2 $3 $4 $5 $6 $7 ================================================ FILE: src/main/assembly/testtool/test_stand_select_perf.bat ================================================ REM check JAVA_HOME & java set "JAVA_CMD=%JAVA_HOME%/bin/java" if "%JAVA_HOME%" == "" goto noJavaHome if exist "%JAVA_HOME%\bin\java.exe" goto mainEntry :noJavaHome echo --------------------------------------------------- echo WARN: JAVA_HOME environment variable is not set. echo --------------------------------------------------- set "JAVA_CMD=java" :mainEntry REM set HOME_DIR set "CURR_DIR=%cd%" cd .. set "MYCAT_HOME=%cd%" cd %CURR_DIR% "%JAVA_CMD%" -Xms256M -Xmx1G -XX:MaxPermSize=64M -DMYCAT_HOME=%MYCAT_HOME% -cp "..\conf;..\lib\*" io.mycat.performance.TestSelectPerf %1 %2 %3 %4 %5 %6 %7 %8 %9 ================================================ FILE: src/main/assembly/testtool/test_stand_select_perf.sh ================================================ #!/bin/bash echo "check JAVA_HOME & java" JAVA_CMD=$JAVA_HOME/bin/java MAIN_CLASS=io.mycat.performance.TestSelectPerf if [ ! -d "$JAVA_HOME" ]; then echo --------------------------------------------------- echo WARN: JAVA_HOME environment variable is not set. echo --------------------------------------------------- JAVA_CMD=java fi echo "---------set HOME_DIR------------" CURR_DIR=`pwd` cd .. MYCAT_HOME=`pwd` cd $CURR_DIR $JAVA_CMD -Xms256M -Xmx1G -XX:MaxPermSize=64M -DMYCAT_HOME=$MYCAT_HOME -cp "$MYCAT_HOME/conf:$MYCAT_HOME/lib/*" $MAIN_CLASS $1 $2 $3 $4 $5 $6 $7 $8 $9 ================================================ FILE: src/main/assembly/testtool/test_stand_update_perf.bat ================================================ REM check JAVA_HOME & java set "JAVA_CMD=%JAVA_HOME%/bin/java" if "%JAVA_HOME%" == "" goto noJavaHome if exist "%JAVA_HOME%\bin\java.exe" goto mainEntry :noJavaHome echo --------------------------------------------------- echo WARN: JAVA_HOME environment variable is not set. echo --------------------------------------------------- set "JAVA_CMD=java" :mainEntry REM set HOME_DIR set "CURR_DIR=%cd%" cd .. set "MYCAT_HOME=%cd%" cd %CURR_DIR% "%JAVA_CMD%" -Xms256M -Xmx1G -XX:MaxPermSize=64M -DMYCAT_HOME=%MYCAT_HOME% -cp "..\conf;..\lib\*" io.mycat.performance.TestUpdatePerf %1 %2 %3 %4 %5 %6 %7 %8 %9 ================================================ FILE: src/main/assembly/testtool/test_stand_update_perf.sh ================================================ #!/bin/bash echo "check JAVA_HOME & java" JAVA_CMD=$JAVA_HOME/bin/java MAIN_CLASS=io.mycat.performance.TestUpdatePerf if [ ! -d "$JAVA_HOME" ]; then echo --------------------------------------------------- echo WARN: JAVA_HOME environment variable is not set. echo --------------------------------------------------- JAVA_CMD=java fi echo "---------set HOME_DIR------------" CURR_DIR=`pwd` cd .. MYCAT_HOME=`pwd` cd $CURR_DIR $JAVA_CMD -Xms256M -Xmx1G -XX:MaxPermSize=64M -DMYCAT_HOME=$MYCAT_HOME -cp "$MYCAT_HOME/conf:$MYCAT_HOME/lib/*" $MAIN_CLASS $1 $2 $3 $4 $5 $6 $7 $8 $9 ================================================ FILE: src/main/java/io/mycat/MycatServer.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.nio.channels.AsynchronousChannelGroup; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.UUID; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.io.Files; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import io.mycat.backend.BackendConnection; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.heartbeat.zkprocess.MycatLeaderLatch; import io.mycat.backend.mysql.nio.handler.MultiNodeCoordinator; import io.mycat.backend.mysql.xa.CoordinatorLogEntry; import io.mycat.backend.mysql.xa.ParticipantLogEntry; import io.mycat.backend.mysql.xa.TxState; import io.mycat.backend.mysql.xa.XACommitCallback; import io.mycat.backend.mysql.xa.XARollbackCallback; import io.mycat.backend.mysql.xa.recovery.Repository; import io.mycat.backend.mysql.xa.recovery.impl.FileSystemRepository; import io.mycat.buffer.BufferPool; import io.mycat.buffer.DirectByteBufferPool; import io.mycat.buffer.NettyBufferPool; import io.mycat.cache.CacheService; import io.mycat.config.MycatConfig; import io.mycat.config.classloader.DynaClassLoader; import io.mycat.config.loader.zkprocess.comm.ZkConfig; import io.mycat.config.loader.zkprocess.comm.ZkParamCfg; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.config.model.TableConfig; import io.mycat.config.table.structure.MySQLTableStructureDetector; import io.mycat.manager.ManagerConnectionFactory; import io.mycat.memory.MyCatMemory; import io.mycat.net.AIOAcceptor; import io.mycat.net.AIOConnector; import io.mycat.net.NIOAcceptor; import io.mycat.net.NIOConnector; import io.mycat.net.NIOProcessor; import io.mycat.net.NIOReactorPool; import io.mycat.net.SocketAcceptor; import io.mycat.net.SocketConnector; import io.mycat.route.MyCATSequnceProcessor; import io.mycat.route.RouteService; import io.mycat.route.factory.RouteStrategyFactory; import io.mycat.route.sequence.handler.SequenceHandler; import io.mycat.server.ServerConnectionFactory; import io.mycat.server.interceptor.SQLInterceptor; import io.mycat.server.interceptor.impl.GlobalTableUtil; import io.mycat.sqlengine.OneRawSQLQueryResultHandler; import io.mycat.sqlengine.SQLJob; import io.mycat.statistic.SQLRecorder; import io.mycat.statistic.stat.SqlResultSizeRecorder; import io.mycat.statistic.stat.UserStat; import io.mycat.statistic.stat.UserStatAnalyzer; import io.mycat.util.ExecutorUtil; import io.mycat.util.NameableExecutor; import io.mycat.util.TimeUtil; import io.mycat.util.ZKUtils; /** * @author mycat */ public class MycatServer { public static final String NAME = "MyCat"; private static final long LOG_WATCH_DELAY = 60000L; private static final long TIME_UPDATE_PERIOD = 20L; private static final long DEFAULT_SQL_STAT_RECYCLE_PERIOD = 5 * 1000L; private static final long DEFAULT_OLD_CONNECTION_CLEAR_PERIOD = 5 * 1000L; private static final long DEFAULT_DATANODE_CALC_ACTIVECOUNT = 1000L; private static final MycatServer INSTANCE = new MycatServer(); private static final Logger LOGGER = LoggerFactory.getLogger("MycatServer"); private static final Repository fileRepository = new FileSystemRepository(); private final RouteService routerService; private final CacheService cacheService; private Properties dnIndexProperties; //AIO连接群组 private AsynchronousChannelGroup[] asyncChannelGroups; private volatile int channelIndex = 0; //全局序列号 // private final MyCATSequnceProcessor sequnceProcessor = new MyCATSequnceProcessor(); private final DynaClassLoader catletClassLoader; private final SQLInterceptor sqlInterceptor; private volatile int nextProcessor; // System Buffer Pool Instance private BufferPool bufferPool; private boolean aio = false; //XA事务全局ID生成 private final AtomicLong xaIDInc = new AtomicLong(); //sequence处理对象 private SequenceHandler sequenceHandler; /** * Mycat 内存管理类 */ private MyCatMemory myCatMemory = null; public static final MycatServer getInstance() { return INSTANCE; } private final MycatConfig config; private final ScheduledExecutorService scheduler; private final ScheduledExecutorService heartbeatScheduler; private final SQLRecorder sqlRecorder; private final AtomicBoolean isOnline; private final long startupTime; private NIOProcessor[] processors; private SocketConnector connector; private NameableExecutor businessExecutor; private NameableExecutor sequenceExecutor; private NameableExecutor timerExecutor; private ListeningExecutorService listeningExecutorService; private InterProcessMutex dnindexLock; private long totalNetWorkBufferSize = 0; private volatile MycatLeaderLatch leaderLatch; private final AtomicBoolean startup = new AtomicBoolean(false); private ScheduledFuture recycleSqlStatFuture = null; private MycatServer() { //读取文件配置 this.config = new MycatConfig(); //定时线程池，单线程线程池 scheduler = Executors.newSingleThreadScheduledExecutor(); //心跳调度独立出来，避免被其他任务影响 heartbeatScheduler = Executors.newSingleThreadScheduledExecutor(); //SQL记录器 this.sqlRecorder = new SQLRecorder(config.getSystem().getSqlRecordCount()); /** * 是否在线，MyCat manager中有命令控制 * | offline | Change MyCat status to OFF | * | online | Change MyCat status to ON | */ this.isOnline = new AtomicBoolean(true); //缓存服务初始化 cacheService = new CacheService(); //路由计算初始化 routerService = new RouteService(cacheService); // load datanode active index from properties dnIndexProperties = loadDnIndexProps(); try { //SQL解析器 sqlInterceptor = (SQLInterceptor) Class.forName( config.getSystem().getSqlInterceptor()).newInstance(); } catch (Exception e) { throw new RuntimeException(e); } //catlet加载器 catletClassLoader = new DynaClassLoader(SystemConfig.getHomePath() + File.separator + "catlet", config.getSystem().getCatletClassCheckSeconds()); //记录启动时间 this.startupTime = TimeUtil.currentTimeMillis(); if (isUseZkSwitch()) { String path = ZKUtils.getZKBasePath() + "lock/dnindex.lock"; dnindexLock = new InterProcessMutex(ZKUtils.getConnection(), path); } } public AtomicBoolean getStartup() { return startup; } public long getTotalNetWorkBufferSize() { return totalNetWorkBufferSize; } public BufferPool getBufferPool() { return bufferPool; } public NameableExecutor getTimerExecutor() { return timerExecutor; } public DynaClassLoader getCatletClassLoader() { return catletClassLoader; } public MyCATSequnceProcessor getSequnceProcessor() { return MyCATSequnceProcessor.getInstance(); } public SQLInterceptor getSqlInterceptor() { return sqlInterceptor; } public ScheduledExecutorService getScheduler() { return scheduler; } public String genXATXID() { long seq = this.xaIDInc.incrementAndGet(); if (seq < 0) { synchronized (xaIDInc) { if (xaIDInc.get() < 0) { xaIDInc.set(0); } seq = xaIDInc.incrementAndGet(); } } return "'Mycat." + this.getConfig().getSystem().getMycatNodeId() + "." + seq + "'"; } public String getXATXIDGLOBAL() { return "'" + getUUID() + "'"; } public static String getUUID() { String s = UUID.randomUUID().toString(); //去掉“-”符号 return s.substring(0, 8) + s.substring(9, 13) + s.substring(14, 18) + s.substring(19, 23) + s.substring(24); } public MyCatMemory getMyCatMemory() { return myCatMemory; } /** * get next AsynchronousChannel ,first is exclude if multi * AsynchronousChannelGroups * * @return */ public AsynchronousChannelGroup getNextAsyncChannelGroup() { if (asyncChannelGroups.length == 1) { return asyncChannelGroups[0]; } else { int index = (++channelIndex) % asyncChannelGroups.length; if (index == 0) { ++channelIndex; return asyncChannelGroups[1]; } else { return asyncChannelGroups[index]; } } } public MycatConfig getConfig() { return config; } public void beforeStart() { String home = SystemConfig.getHomePath(); //ZkConfig.instance().initZk(); } public void startup() throws IOException { SystemConfig system = config.getSystem(); int processorCount = system.getProcessors(); //init RouteStrategyFactory first RouteStrategyFactory.init(); // server startup LOGGER.info(NAME + " is ready to startup ..."); String inf = "Startup processors ...,total processors:" + system.getProcessors() + ",aio thread pool size:" + system.getProcessorExecutor() + " \r\n each process allocated socket buffer pool " + " bytes ,a page size:" + system.getBufferPoolPageSize() + " a page's chunk number(PageSize/ChunkSize) is:" + (system.getBufferPoolPageSize() / system.getBufferPoolChunkSize()) + " buffer page's number is:" + system.getBufferPoolPageNumber(); LOGGER.info(inf); LOGGER.info("sysconfig params:" + system.toString()); // startup manager ManagerConnectionFactory mf = new ManagerConnectionFactory(); ServerConnectionFactory sf = new ServerConnectionFactory(); SocketAcceptor manager = null; SocketAcceptor server = null; aio = (system.getUsingAIO() == 1); // startup processors int threadPoolSize = system.getProcessorExecutor(); processors = new NIOProcessor[processorCount]; // a page size int bufferPoolPageSize = system.getBufferPoolPageSize(); // total page number short bufferPoolPageNumber = system.getBufferPoolPageNumber(); //minimum allocation unit short bufferPoolChunkSize = system.getBufferPoolChunkSize(); int socketBufferLocalPercent = system.getProcessorBufferLocalPercent(); int bufferPoolType = system.getProcessorBufferPoolType(); switch (bufferPoolType) { case 0: bufferPool = new DirectByteBufferPool(bufferPoolPageSize, bufferPoolChunkSize, bufferPoolPageNumber, system.getFrontSocketSoRcvbuf()); totalNetWorkBufferSize = bufferPoolPageSize * bufferPoolPageNumber; break; case 1: /** * todo 对应权威指南修改： * * bytebufferarena由6个bytebufferlist组成，这六个list有减少内存碎片的机制 * 每个bytebufferlist由多个bytebufferchunk组成，每个list也有减少内存碎片的机制 * 每个bytebufferchunk由多个page组成，平衡二叉树管理内存使用状态，计算灵活 * 设置的pagesize对应bytebufferarena里面的每个bytebufferlist的每个bytebufferchunk的buffer长度 * bufferPoolChunkSize对应每个bytebufferchunk的每个page的长度 * bufferPoolPageNumber对应每个bytebufferlist有多少个bytebufferchunk */ totalNetWorkBufferSize = 6 * bufferPoolPageSize * bufferPoolPageNumber; break; case 2: bufferPool = new NettyBufferPool(bufferPoolChunkSize); LOGGER.info("Use Netty Buffer Pool"); break; default: bufferPool = new DirectByteBufferPool(bufferPoolPageSize, bufferPoolChunkSize, bufferPoolPageNumber, system.getFrontSocketSoRcvbuf()); ; totalNetWorkBufferSize = bufferPoolPageSize * bufferPoolPageNumber; } /** * Off Heap For Merge/Order/Group/Limit 初始化 */ if (system.getUseOffHeapForMerge() == 1) { try { myCatMemory = new MyCatMemory(system, totalNetWorkBufferSize); } catch (NoSuchFieldException e) { LOGGER.error("NoSuchFieldException", e); } catch (IllegalAccessException e) { LOGGER.error("Error", e); } } businessExecutor = ExecutorUtil.create("BusinessExecutor", threadPoolSize); sequenceExecutor = ExecutorUtil.create("SequenceExecutor", threadPoolSize); timerExecutor = ExecutorUtil.create("Timer", system.getTimerExecutor()); listeningExecutorService = MoreExecutors.listeningDecorator(businessExecutor); for (int i = 0; i < processors.length; i++) { processors[i] = new NIOProcessor("Processor" + i, bufferPool, businessExecutor); } if (aio) { LOGGER.info("using aio network handler "); asyncChannelGroups = new AsynchronousChannelGroup[processorCount]; // startup connector connector = new AIOConnector(); for (int i = 0; i < processors.length; i++) { asyncChannelGroups[i] = AsynchronousChannelGroup.withFixedThreadPool(processorCount, new ThreadFactory() { private int inx = 1; @Override public Thread newThread(Runnable r) { Thread th = new Thread(r); //TODO th.setName(DirectByteBufferPool.LOCAL_BUF_THREAD_PREX + "AIO" + (inx++)); LOGGER.info("created new AIO thread " + th.getName()); return th; } } ); } manager = new AIOAcceptor(NAME + "Manager", system.getBindIp(), system.getManagerPort(), system.getServerBacklog(), mf, this.asyncChannelGroups[0]); // startup server server = new AIOAcceptor(NAME + "Server", system.getBindIp(), system.getServerPort(), system.getServerBacklog(), sf, this.asyncChannelGroups[0]); } else { LOGGER.info("using nio network handler "); NIOReactorPool reactorPool = new NIOReactorPool( DirectByteBufferPool.LOCAL_BUF_THREAD_PREX + "NIOREACTOR", processors.length); connector = new NIOConnector(DirectByteBufferPool.LOCAL_BUF_THREAD_PREX + "NIOConnector", reactorPool); ((NIOConnector) connector).start(); manager = new NIOAcceptor(DirectByteBufferPool.LOCAL_BUF_THREAD_PREX + NAME + "Manager", system.getBindIp(), system.getManagerPort(), system.getServerBacklog(), mf, reactorPool); server = new NIOAcceptor(DirectByteBufferPool.LOCAL_BUF_THREAD_PREX + NAME + "Server", system.getBindIp(), system.getServerPort(), system.getServerBacklog(), sf, reactorPool); } // manager start manager.start(); LOGGER.info(manager.getName() + " is started and listening on " + manager.getPort()); server.start(); // server started LOGGER.info(server.getName() + " is started and listening on " + server.getPort()); LOGGER.info("==============================================="); // init datahost Map dataHosts = config.getDataHosts(); LOGGER.info("Initialize dataHost ..."); for (PhysicalDBPool node : dataHosts.values()) { String index = dnIndexProperties.getProperty(node.getHostName(), "0"); if (!"0".equals(index)) { LOGGER.info("init datahost: " + node.getHostName() + " to use datasource index:" + index); } node.init(Integer.parseInt(index)); node.startHeartbeat(); } long dataNodeIldeCheckPeriod = system.getDataNodeIdleCheckPeriod(); heartbeatScheduler.scheduleAtFixedRate(updateTime(), 0L, TIME_UPDATE_PERIOD, TimeUnit.MILLISECONDS); heartbeatScheduler.scheduleAtFixedRate(processorCheck(), 0L, system.getProcessorCheckPeriod(), TimeUnit.MILLISECONDS); heartbeatScheduler.scheduleAtFixedRate(dataNodeConHeartBeatCheck(dataNodeIldeCheckPeriod), 0L, dataNodeIldeCheckPeriod, TimeUnit.MILLISECONDS); heartbeatScheduler.scheduleAtFixedRate(dataNodeHeartbeat(), 0L, system.getDataNodeHeartbeatPeriod(), TimeUnit.MILLISECONDS); heartbeatScheduler.scheduleAtFixedRate(dataSourceOldConsClear(), 0L, DEFAULT_OLD_CONNECTION_CLEAR_PERIOD, TimeUnit.MILLISECONDS); heartbeatScheduler.scheduleAtFixedRate(dataNodeCalcActiveCons(), 0L, DEFAULT_DATANODE_CALC_ACTIVECOUNT, TimeUnit.MILLISECONDS); // scheduler.schedule(catletClassClear(), 30000, TimeUnit.MILLISECONDS); if (system.getCheckTableConsistency() == 1) { scheduler.scheduleAtFixedRate(tableStructureCheck(), 0L, system.getCheckTableConsistencyPeriod(), TimeUnit.MILLISECONDS); } ensureSqlstatRecycleFuture(); if (system.getUseGlobleTableCheck() == 1) { // 全局表一致性检测是否开启 // scheduler.scheduleAtFixedRate(glableTableConsistencyCheck(), 0L, system.getGlableTableCheckPeriod(), TimeUnit.MILLISECONDS); } //定期清理结果集排行榜，控制拒绝策略 scheduler.scheduleAtFixedRate(resultSetMapClear(), 0L, system.getClearBigSqLResultSetMapMs(), TimeUnit.MILLISECONDS); //xa 事务定时检查 是否全部提交 或者部分提交 部分回滚, 进行补充提交补充回滚. scheduler.scheduleAtFixedRate(xaTaskCheck(), 0L, 10 * 1000, TimeUnit.MILLISECONDS); // new Thread(tableStructureCheck()).start(); //XA Init recovery Log LOGGER.info("==============================================="); LOGGER.info("Perform XA recovery log ..."); CoordinatorLogEntry[] coordinatorLogEntries = getCoordinatorLogEntries(); putXARecoveryLogToMemory(coordinatorLogEntries); performXARecoveryLog(coordinatorLogEntries); LOGGER.info("Perform XA recovery log end..."); if (isUseZkSwitch()) { //首次启动如果发现zk上dnindex为空，则将本地初始化上zk initZkDnindex(); leaderLatch = new MycatLeaderLatch("heartbeat/leader"); try { leaderLatch.start(); } catch (Exception e) { LOGGER.error(e.getMessage(), e); e.printStackTrace(); } } initRuleData(); startup.set(true); } public void ensureSqlstatRecycleFuture() { if (config.getSystem().getUseSqlStat() == 1) { if(recycleSqlStatFuture == null){ recycleSqlStatFuture = scheduler .scheduleAtFixedRate(recycleSqlStat(), 0L, DEFAULT_SQL_STAT_RECYCLE_PERIOD, TimeUnit.MILLISECONDS); } } else { if (recycleSqlStatFuture != null) { recycleSqlStatFuture.cancel(false); recycleSqlStatFuture = null; } } } public void initRuleData() { if (!isUseZk()) return; InterProcessMutex ruleDataLock = null; try { File file = new File(SystemConfig.getHomePath(), "conf" + File.separator + "ruledata"); if (!file.exists()) { file.mkdir(); } String path = ZKUtils.getZKBasePath() + "lock/ruledata.lock"; ruleDataLock = new InterProcessMutex(ZKUtils.getConnection(), path); ruleDataLock.acquire(30, TimeUnit.SECONDS); File[] childFiles = file.listFiles(); if (childFiles != null && childFiles.length > 0) { String basePath = ZKUtils.getZKBasePath() + "ruledata/"; for (File childFile : childFiles) { CuratorFramework zk = ZKUtils.getConnection(); if (zk.checkExists().forPath(basePath + childFile.getName()) == null) { zk.create().creatingParentsIfNeeded().forPath(basePath + childFile.getName(), Files.toByteArray(childFile)); } } } } catch (Exception e) { throw new RuntimeException(e); } finally { try { if (ruleDataLock != null) ruleDataLock.release(); } catch (Exception e) { throw new RuntimeException(e); } } } private void initZkDnindex() { try { File file = new File(SystemConfig.getHomePath(), "conf" + File.separator + "dnindex.properties"); dnindexLock.acquire(30, TimeUnit.SECONDS); String path = ZKUtils.getZKBasePath() + "bindata/dnindex.properties"; CuratorFramework zk = ZKUtils.getConnection(); if (zk.checkExists().forPath(path) == null) { zk.create().creatingParentsIfNeeded().forPath(path, Files.toByteArray(file)); } } catch (Exception e) { throw new RuntimeException(e); } finally { try { dnindexLock.release(); } catch (Exception e) { throw new RuntimeException(e); } } } public void reloadDnIndex() { if (MycatServer.getInstance().getProcessors() == null) return; // load datanode active index from properties dnIndexProperties = loadDnIndexProps(); // init datahost Map dataHosts = config.getDataHosts(); LOGGER.info("reInitialize dataHost ..."); for (PhysicalDBPool node : dataHosts.values()) { String index = dnIndexProperties.getProperty(node.getHostName(), "0"); if (!"0".equals(index)) { LOGGER.info("reinit datahost: " + node.getHostName() + " to use datasource index:" + index); } node.switchSource(Integer.parseInt(index), true, "reload dnindex"); } } private Runnable catletClassClear() { return new Runnable() { @Override public void run() { try { catletClassLoader.clearUnUsedClass(); } catch (Exception e) { LOGGER.warn("catletClassClear err " + e); } } ; }; } /** * 清理 reload @@config_all 后，老的 connection 连接 * * @return */ private Runnable dataSourceOldConsClear() { return new Runnable() { @Override public void run() { timerExecutor.execute(new Runnable() { @Override public void run() { long sqlTimeout = MycatServer.getInstance().getConfig().getSystem().getSqlExecuteTimeout() * 1000L; //根据 lastTime 确认事务的执行， 超过 sqlExecuteTimeout 阀值 close connection long currentTime = TimeUtil.currentTimeMillis(); Iterator iter = NIOProcessor.backends_old.iterator(); while (iter.hasNext()) { BackendConnection con = iter.next(); long lastTime = con.getLastTime(); if (currentTime - lastTime > sqlTimeout) { con.close("clear old backend connection ..."); iter.remove(); } } } }); } ; }; } /** * 在bufferpool使用率大于使用率阈值时不清理 * 在bufferpool使用率小于使用率阈值时清理大结果集清单内容 */ private Runnable resultSetMapClear() { return new Runnable() { @Override public void run() { try { BufferPool bufferPool = getBufferPool(); long bufferSize = bufferPool.size(); long bufferCapacity = bufferPool.capacity(); long bufferUsagePercent = (bufferCapacity - bufferSize) * 100 / bufferCapacity; if (bufferUsagePercent < config.getSystem().getBufferUsagePercent()) { Map map = UserStatAnalyzer.getInstance().getUserStatMap(); Set userSet = config.getUsers().keySet(); for (String user : userSet) { UserStat userStat = map.get(user); if (userStat != null) { SqlResultSizeRecorder recorder = userStat.getSqlResultSizeRecorder(); //System.out.println(recorder.getSqlResultSet().size()); recorder.clearSqlResultSet(); } } } } catch (Exception e) { LOGGER.warn("resultSetMapClear err " + e); } } ; }; } private Properties loadDnIndexProps() { Properties prop = new Properties(); File file = new File(SystemConfig.getHomePath(), "conf" + File.separator + "dnindex.properties"); if (!file.exists()) { return prop; } FileInputStream filein = null; try { filein = new FileInputStream(file); prop.load(filein); } catch (Exception e) { LOGGER.warn("load DataNodeIndex err:" + e); } finally { if (filein != null) { try { filein.close(); } catch (IOException e) { } } } return prop; } public synchronized boolean saveDataHostIndexToZk(String dataHost, int curIndex) { boolean result = false; try { try { dnindexLock.acquire(30, TimeUnit.SECONDS); String path = ZKUtils.getZKBasePath() + "bindata/dnindex.properties"; Map propertyMap = new HashMap<>(); propertyMap.put(dataHost, String.valueOf(curIndex)); result = ZKUtils.writeProperty(path, propertyMap); } finally { dnindexLock.release(); } } catch (Exception e) { LOGGER.warn("saveDataHostIndexToZk err:", e); } return result; } /** * save cur datanode index to properties file * * @param * @param curIndex */ public synchronized void saveDataHostIndex(String dataHost, int curIndex) { File file = new File(SystemConfig.getHomePath(), "conf" + File.separator + "dnindex.properties"); FileOutputStream fileOut = null; try { String oldIndex = dnIndexProperties.getProperty(dataHost); String newIndex = String.valueOf(curIndex); if (newIndex.equals(oldIndex)) { return; } dnIndexProperties.setProperty(dataHost, newIndex); LOGGER.info("save DataHost index " + dataHost + " cur index " + curIndex); File parent = file.getParentFile(); if (parent != null && !parent.exists()) { parent.mkdirs(); } fileOut = new FileOutputStream(file); dnIndexProperties.store(fileOut, "update"); // if(isUseZkSwitch()) { // // save to zk // try { // dnindexLock.acquire(30,TimeUnit.SECONDS) ; // String path = ZKUtils.getZKBasePath() + "bindata/dnindex.properties"; // CuratorFramework zk = ZKUtils.getConnection(); // if(zk.checkExists().forPath(path)==null) { // zk.create().creatingParentsIfNeeded().forPath(path, Files.toByteArray(file)); // } else{ // byte[] data= zk.getData().forPath(path); // ByteArrayOutputStream out=new ByteArrayOutputStream(); // Properties properties=new Properties(); // properties.load(new ByteArrayInputStream(data)); // if(!String.valueOf(curIndex).equals(properties.getProperty(dataHost))) { // properties.setProperty(dataHost, String.valueOf(curIndex)); // properties.store(out, "update"); // zk.setData().forPath(path, out.toByteArray()); // } // } // // }finally { // dnindexLock.release(); // } // } } catch (Exception e) { LOGGER.warn("saveDataNodeIndex err:", e); } finally { if (fileOut != null) { try { fileOut.close(); } catch (IOException e) { } } } } private boolean isUseZk() { String loadZk = ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_FLAG); return "true".equalsIgnoreCase(loadZk); } public boolean isUseZkSwitch() { MycatConfig mycatConfig = config; boolean isUseZkSwitch = mycatConfig.getSystem().isUseZKSwitch(); String loadZk = ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_FLAG); return (isUseZkSwitch && "true".equalsIgnoreCase(loadZk)); } public RouteService getRouterService() { return routerService; } public CacheService getCacheService() { return cacheService; } public NameableExecutor getBusinessExecutor() { return businessExecutor; } public RouteService getRouterservice() { return routerService; } public NIOProcessor nextProcessor() { int i = ++nextProcessor; if (i >= processors.length) { i = nextProcessor = 0; } return processors[i]; } public NIOProcessor[] getProcessors() { return processors; } public SocketConnector getConnector() { return connector; } public SQLRecorder getSqlRecorder() { return sqlRecorder; } public long getStartupTime() { return startupTime; } public boolean isOnline() { return isOnline.get(); } public void offline() { isOnline.set(false); } public void online() { isOnline.set(true); } // 系统时间定时更新任务 private Runnable updateTime() { return new Runnable() { @Override public void run() { TimeUtil.update(); } }; } // 处理器定时检查任务 private Runnable processorCheck() { return new Runnable() { @Override public void run() { timerExecutor.execute(new Runnable() { @Override public void run() { try { for (NIOProcessor p : processors) { p.checkBackendCons(); } } catch (Exception e) { LOGGER.warn("checkBackendCons caught err:" + e); } } }); timerExecutor.execute(new Runnable() { @Override public void run() { try { for (NIOProcessor p : processors) { p.checkFrontCons(); } } catch (Exception e) { LOGGER.warn("checkFrontCons caught err:" + e); } } }); } }; } // 数据节点定时连接空闲超时检查任务 private Runnable dataNodeConHeartBeatCheck(final long heartPeriod) { return new Runnable() { @Override public void run() { timerExecutor.execute(new Runnable() { @Override public void run() { Map nodes = config.getDataHosts(); for (PhysicalDBPool node : nodes.values()) { node.heartbeatCheck(heartPeriod); } /* Map _nodes = config.getBackupDataHosts(); if (_nodes != null) { for (PhysicalDBPool node : _nodes.values()) { node.heartbeatCheck(heartPeriod); } }*/ } }); } }; } // 数据节点定时心跳任务 private Runnable dataNodeHeartbeat() { return new Runnable() { @Override public void run() { timerExecutor.execute(new Runnable() { @Override public void run() { Map nodes = config.getDataHosts(); for (PhysicalDBPool node : nodes.values()) { node.doHeartbeat(); } } }); } }; } //by kaiz : 定时计算datanode active connection private Runnable dataNodeCalcActiveCons() { return new Runnable() { @Override public void run() { timerExecutor.execute(new Runnable() { @Override public void run() { Map nodes = config.getDataHosts(); for (PhysicalDBPool node : nodes.values()) { Collection dataSources = node.getAllDataSources(); for(PhysicalDatasource ds : dataSources) { ds.calcTotalCount(); } } } }); } }; } //定时清理保存SqlStat中的数据 private Runnable recycleSqlStat() { return new Runnable() { @Override public void run() { Map statMap = UserStatAnalyzer.getInstance().getUserStatMap(); for (UserStat userStat : statMap.values()) { userStat.getSqlLastStat().recycle(); userStat.getSqlRecorder().recycle(); userStat.getSqlHigh().recycle(); userStat.getSqlLargeRowStat().recycle(); } } }; } //定时清理xa任务 对超过阈值的xa任务回滚或者提交 private Runnable xaTaskCheck() { return new Runnable() { @Override public void run() { Collection coordinatorLogEntries = MultiNodeCoordinator.inMemoryRepository.getAllCoordinatorLogEntries(); long sqlTimeout = MycatServer.getInstance().getConfig().getSystem().getSqlExecuteTimeout() * 1000L; List CoordinatorLogEntryList = null; long currentTime = TimeUtil.currentTimeMillis(); for(CoordinatorLogEntry coordinatorLogEntry : coordinatorLogEntries) { //超过执行时间20秒 进行重试 if(currentTime > sqlTimeout + 20 * 1000 + coordinatorLogEntry.createTime){ if(CoordinatorLogEntryList == null) { CoordinatorLogEntryList = new ArrayList(); } CoordinatorLogEntryList.add(coordinatorLogEntry); } } if(CoordinatorLogEntryList != null) { performXARecoveryLog((CoordinatorLogEntry[])CoordinatorLogEntryList.toArray()); } } }; } //定时检查不同分片表结构一致性 private Runnable tableStructureCheck() { return new MySQLTableStructureDetector(); } // 全局表一致性检查任务 private Runnable glableTableConsistencyCheck() { return new Runnable() { @Override public void run() { timerExecutor.execute(new Runnable() { @Override public void run() { GlobalTableUtil.consistencyCheck(); } }); } }; } private void putXARecoveryLogToMemory(CoordinatorLogEntry[] coordinatorLogEntries) { //init into in memory cached for (int i = 0; i < coordinatorLogEntries.length; i++) { MultiNodeCoordinator.inMemoryRepository.put(coordinatorLogEntries[i].id, coordinatorLogEntries[i]); //discard the recovery log MultiNodeCoordinator.fileRepository.writeCheckpoint(coordinatorLogEntries[i].id, MultiNodeCoordinator.inMemoryRepository.getAllCoordinatorLogEntries()); } } //XA recovery log check private void performXARecoveryLog(CoordinatorLogEntry[] coordinatorLogEntries) { //fetch the recovery log for (int i = 0; i < coordinatorLogEntries.length; i++) { CoordinatorLogEntry coordinatorLogEntry = coordinatorLogEntries[i]; boolean needRollback = false; boolean hasCommit = false; //检查xa事务是否完成 ,处于部分commit 或者部分prepare中 for (int j = 0; j < coordinatorLogEntry.participants.length; j++) { ParticipantLogEntry participantLogEntry = coordinatorLogEntry.participants[j]; if (participantLogEntry.txState == TxState.TX_PREPARED_STATE || participantLogEntry.txState == TxState.TX_STARTED_STATE) { needRollback = true; } if (participantLogEntry.txState == TxState.TX_COMMITED_STATE) { hasCommit = true; } } //补充提交 prepare 状态的提交, xa commit or xa rollback if (needRollback) { //1 can rollback if(!hasCommit) { for (int j = 0; j < coordinatorLogEntry.participants.length; j++) { ParticipantLogEntry participantLogEntry = coordinatorLogEntry.participants[j]; if (participantLogEntry.txState == TxState.TX_COMMITED_STATE || participantLogEntry.txState == TxState.TX_ROLLBACKED_STATE) { continue; } //XA rollback String xacmd = "XA ROLLBACK " + coordinatorLogEntry.id +",'"+ participantLogEntry.resourceName+"'" + ';'; LOGGER.debug("send xaCmd : {}", xacmd); OneRawSQLQueryResultHandler resultHandler = new OneRawSQLQueryResultHandler(new String[0], new XARollbackCallback(coordinatorLogEntry.id, participantLogEntry )); //xa cmd send sendXaCmd(participantLogEntry, xacmd, resultHandler); } } else { LOGGER.debug( "some has commit in {}",coordinatorLogEntry); for (int j = 0; j < coordinatorLogEntry.participants.length; j++) { ParticipantLogEntry participantLogEntry = coordinatorLogEntry.participants[j]; if (participantLogEntry.txState == TxState.TX_COMMITED_STATE || participantLogEntry.txState == TxState.TX_ROLLBACKED_STATE) { continue; } //XA commit String xacmd = "XA COMMIT " + coordinatorLogEntry.id +",'"+ participantLogEntry.resourceName+"'" + ';'; LOGGER.debug("send xaCmd : {}", xacmd); OneRawSQLQueryResultHandler resultHandler = new OneRawSQLQueryResultHandler(new String[0], new XACommitCallback(coordinatorLogEntry.id, participantLogEntry )); //xa cmd send sendXaCmd(participantLogEntry, xacmd, resultHandler); } } } } } private void sendXaCmd(ParticipantLogEntry participantLogEntry, String xacmd, OneRawSQLQueryResultHandler resultHandler) { for (SchemaConfig schema : MycatServer.getInstance().getConfig().getSchemas().values()) { for (TableConfig table : schema.getTables().values()) { for (String dataNode : table.getDataNodes()) { PhysicalDBNode dn = MycatServer.getInstance().getConfig().getDataNodes().get(dataNode); if (dn.getDbPool().getSource().getConfig().getIp().equals(participantLogEntry.uri) && dn.getDatabase().equals(participantLogEntry.resourceName)) { //XA STATE ROLLBACK SQLJob sqlJob = new SQLJob(xacmd, dn.getDatabase(), resultHandler, dn.getDbPool().getSource()); sqlJob.run(); LOGGER.debug(String.format("[XA cmd] [%s] Host:[%s] schema:[%s]", xacmd, dn.getName(), dn.getDatabase())); // break outloop; return; } } } } } /** * covert the collection to array **/ private CoordinatorLogEntry[] getCoordinatorLogEntries() { Collection allCoordinatorLogEntries = fileRepository.getAllCoordinatorLogEntries(); if (allCoordinatorLogEntries == null) { return new CoordinatorLogEntry[0]; } if (allCoordinatorLogEntries.size() == 0) { return new CoordinatorLogEntry[0]; } return allCoordinatorLogEntries.toArray(new CoordinatorLogEntry[allCoordinatorLogEntries.size()]); } public NameableExecutor getSequenceExecutor() { return sequenceExecutor; } //huangyiming add public DirectByteBufferPool getDirectByteBufferPool() { return (DirectByteBufferPool) bufferPool; } public boolean isAIO() { return aio; } public ListeningExecutorService getListeningExecutorService() { return listeningExecutorService; } public ScheduledExecutorService getHeartbeatScheduler() { return heartbeatScheduler; } public MycatLeaderLatch getLeaderLatch() { return leaderLatch; } public static void main(String[] args) throws Exception { String path = ZKUtils.getZKBasePath() + "bindata"; CuratorFramework zk = ZKUtils.getConnection(); if (zk.checkExists().forPath(path) == null) ; byte[] data = zk.getData().forPath(path); System.out.println(data.length); } } ================================================ FILE: src/main/java/io/mycat/MycatShutdown.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat; import java.util.Date; /** * @author mycat */ public final class MycatShutdown { public static void main(String[] args) { System.out.println(new Date() + ",server shutdown!"); } } ================================================ FILE: src/main/java/io/mycat/MycatStartup.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat; import java.text.SimpleDateFormat; import java.util.Date; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.config.loader.zkprocess.comm.ZkConfig; import io.mycat.config.model.SystemConfig; /** * @author mycat */ public final class MycatStartup { private static final String dateFormat = "yyyy-MM-dd HH:mm:ss"; private static final Logger LOGGER = LoggerFactory.getLogger(MycatStartup.class); public static void main(String[] args) { //use zk ? ZkConfig.getInstance().initZk(); try { String home = SystemConfig.getHomePath(); if (home == null) { System.out.println(SystemConfig.SYS_HOME + " is not set."); System.exit(-1); } // init MycatServer server = MycatServer.getInstance(); //这个方法执行的代码，上面SystemConfig.getHomePath()已经执行过了，建议注释掉。 //server.beforeStart(); // startup server.startup(); System.out.println("MyCAT Server startup successfully. see logs in logs/mycat.log"); } catch (Exception e) { SimpleDateFormat sdf = new SimpleDateFormat(dateFormat); LOGGER.error(sdf.format(new Date()) + " startup error", e); System.exit(-1); } } } ================================================ FILE: src/main/java/io/mycat/backend/BackendConnection.java ================================================ package io.mycat.backend; import java.io.IOException; import java.io.UnsupportedEncodingException; import io.mycat.backend.mysql.nio.handler.ResponseHandler; import io.mycat.net.ClosableConnection; import io.mycat.route.RouteResultsetNode; import io.mycat.server.ServerConnection; public interface BackendConnection extends ClosableConnection { public boolean isModifiedSQLExecuted(); public boolean isFromSlaveDB(); public String getSchema(); public void setSchema(String newSchema); public long getLastTime(); public boolean isClosedOrQuit(); public void setAttachment(Object attachment); public void quit(); public void setLastTime(long currentTimeMillis); public void release(); public boolean setResponseHandler(ResponseHandler commandHandler); public void commit(); public void query(String sql) throws UnsupportedEncodingException; public Object getAttachment(); public void closeWithoutRsp(String reason); // public long getThreadId(); public void execute(RouteResultsetNode node, ServerConnection source, boolean autocommit) throws IOException; public void recordSql(String host, String schema, String statement); public boolean syncAndExcute(); public void rollback(); public boolean isBorrowed(); public void setBorrowed(boolean borrowed); public int getTxIsolation(); public boolean isAutocommit(); public boolean isTxReadonly(); public int getSqlSelectLimit(); public long getId(); public void discardClose(String reason); public void query(String sql, int charsetIndex); public boolean checkAlive(); public void disableRead(); public void enableRead(); } ================================================ FILE: src/main/java/io/mycat/backend/ConMap.java ================================================ package io.mycat.backend; import java.util.Collection; import java.util.Iterator; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.jdbc.JDBCConnection; import io.mycat.backend.mysql.nio.MySQLConnection; import io.mycat.net.NIOProcessor; public class ConMap { // key -schema private final ConcurrentHashMap items = new ConcurrentHashMap(); public ConQueue getSchemaConQueue(String schema) { ConQueue queue = items.get(schema); if (queue == null) { ConQueue newQueue = new ConQueue(); queue = items.putIfAbsent(schema, newQueue); return (queue == null) ? newQueue : queue; } return queue; } public BackendConnection tryTakeCon(final String schema, boolean autoCommit) { final ConQueue queue = items.get(schema); BackendConnection con = tryTakeCon(queue, autoCommit); if (con != null) { return con; } else { for (ConQueue queue2 : items.values()) { if (queue != queue2) { con = tryTakeCon(queue2, autoCommit); if (con != null) { return con; } } } } return null; } private BackendConnection tryTakeCon(ConQueue queue, boolean autoCommit) { BackendConnection con = null; if (queue != null && ((con = queue.takeIdleCon(autoCommit)) != null)) { return con; } else { return null; } } public Collection getAllConQueue() { return items.values(); } public int getActiveCountForSchema(String schema, PhysicalDatasource dataSouce) { int total = 0; for (NIOProcessor processor : MycatServer.getInstance().getProcessors()) { for (BackendConnection con : processor.getBackends().values()) { if (con instanceof MySQLConnection) { MySQLConnection mysqlCon = (MySQLConnection) con; if (mysqlCon.getSchema().equals(schema) && mysqlCon.getPool() == dataSouce && mysqlCon.isBorrowed()) { total++; } }else if (con instanceof JDBCConnection) { JDBCConnection jdbcCon = (JDBCConnection) con; if (jdbcCon.getSchema().equals(schema) && jdbcCon.getPool() == dataSouce && jdbcCon.isBorrowed()) { total++; } } } } return total; } public int getActiveCountForDs(PhysicalDatasource dataSouce) { int total = 0; for (NIOProcessor processor : MycatServer.getInstance().getProcessors()) { for (BackendConnection con : processor.getBackends().values()) { if (con instanceof MySQLConnection) { MySQLConnection mysqlCon = (MySQLConnection) con; if (mysqlCon.getPool() == dataSouce && mysqlCon.isBorrowed() && !mysqlCon.isClosed()) { total++; } } else if (con instanceof JDBCConnection) { JDBCConnection jdbcCon = (JDBCConnection) con; if (jdbcCon.getPool() == dataSouce && jdbcCon.isBorrowed() && !jdbcCon.isClosed()) { total++; } } } } return total; } public int getTotalCountForDs(PhysicalDatasource dataSouce) { int total = 0; for (NIOProcessor processor : MycatServer.getInstance().getProcessors()) { for (BackendConnection con : processor.getBackends().values()) { if (con instanceof MySQLConnection) { MySQLConnection mysqlCon = (MySQLConnection) con; if (mysqlCon.getPool() == dataSouce && !mysqlCon.isClosed()) { total++; } } else if (con instanceof JDBCConnection) { JDBCConnection jdbcCon = (JDBCConnection) con; if (jdbcCon.getPool() == dataSouce && !jdbcCon.isClosed()) { total++; } } } } return total; } public void clearConnections(String reason, PhysicalDatasource dataSouce) { for (NIOProcessor processor : MycatServer.getInstance().getProcessors()) { ConcurrentMap map = processor.getBackends(); Iterator> itor = map.entrySet().iterator(); while (itor.hasNext()) { Entry entry = itor.next(); BackendConnection con = entry.getValue(); if (con instanceof MySQLConnection) { if (((MySQLConnection) con).getPool() == dataSouce) { con.close(reason); itor.remove(); } } else if((con instanceof JDBCConnection) && (((JDBCConnection) con).getPool() == dataSouce)){ con.close(reason); itor.remove(); } } } items.clear(); } } ================================================ FILE: src/main/java/io/mycat/backend/ConQueue.java ================================================ package io.mycat.backend; import java.util.ArrayList; import java.util.concurrent.ConcurrentLinkedQueue; public class ConQueue { private final ConcurrentLinkedQueue autoCommitCons = new ConcurrentLinkedQueue(); private final ConcurrentLinkedQueue manCommitCons = new ConcurrentLinkedQueue(); private long executeCount; public BackendConnection takeIdleCon(boolean autoCommit) { ConcurrentLinkedQueue f1 = autoCommitCons; ConcurrentLinkedQueue f2 = manCommitCons; if (!autoCommit) { f1 = manCommitCons; f2 = autoCommitCons; } BackendConnection con = f1.poll(); while (con != null && !con.checkAlive()){ con.close("channel is closed"); con = f1.poll(); } if (con == null || con.isClosedOrQuit()) { con = f2.poll(); } if (con == null || con.isClosedOrQuit()) { return null; } else { return con; } } public long getExecuteCount() { return executeCount; } public void incExecuteCount() { this.executeCount++; } public boolean removeCon(BackendConnection con) { boolean removed = autoCommitCons.remove(con); if (!removed) { return manCommitCons.remove(con); } return removed; } public boolean isSameCon(BackendConnection con) { if (autoCommitCons.contains(con)) { return true; } else if (manCommitCons.contains(con)) { return true; } return false; } public ConcurrentLinkedQueue getAutoCommitCons() { return autoCommitCons; } public ConcurrentLinkedQueue getManCommitCons() { return manCommitCons; } public ArrayList getIdleConsToClose(int count) { ArrayList readyCloseCons = new ArrayList( count); while (!manCommitCons.isEmpty() && readyCloseCons.size() < count) { BackendConnection theCon = manCommitCons.poll(); if (theCon != null&&!theCon.isBorrowed()) { readyCloseCons.add(theCon); } } while (!autoCommitCons.isEmpty() && readyCloseCons.size() < count) { BackendConnection theCon = autoCommitCons.poll(); if (theCon != null&&!theCon.isBorrowed()) { readyCloseCons.add(theCon); } } return readyCloseCons; } } ================================================ FILE: src/main/java/io/mycat/backend/ConnectionMeta.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend; /** * connection metadata info * * @author wuzhih * */ public class ConnectionMeta { private final String schema; private final String charset; private final int txIsolation; private final boolean autocommit; public ConnectionMeta(String schema, String charset, int txIsolation, boolean autocommit) { super(); this.schema = schema; this.charset = charset; this.txIsolation = txIsolation; this.autocommit = autocommit; } public String getSchema() { return schema; } // public String getCharset() { // return charset; // } // // public int getTxIsolation() { // return txIsolation; // } // // public boolean isAutocommit() { // return autocommit; // } public boolean isSameSchema(BackendConnection theCon) { return theCon.getSchema().equals(schema); } /** * get metadata similarity * * @param theCon * @return */ public int getMetaSimilarity(BackendConnection theCon) { int result = 0; if (schema == null || schema.equals(theCon.getSchema())) { result++; } if (charset == null || charset.equals(theCon.getCharset())) { result++; } if (txIsolation == -1 || txIsolation == theCon.getTxIsolation()) { result++; } if (autocommit == theCon.isAutocommit()) { result++; } return result; } @Override public String toString() { return "ConnectionMeta [schema=" + schema + ", charset=" + charset + ", txIsolation=" + txIsolation + ", autocommit=" + autocommit + "]"; } } ================================================ FILE: src/main/java/io/mycat/backend/datasource/PhysicalDBNode.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.datasource; import io.mycat.MycatServer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.backend.BackendConnection; import io.mycat.backend.mysql.nio.handler.ResponseHandler; import io.mycat.route.RouteResultsetNode; public class PhysicalDBNode { protected static final Logger LOGGER = LoggerFactory .getLogger(PhysicalDBNode.class); protected final String name; protected final String database; protected final PhysicalDBPool dbPool; public PhysicalDBNode(String hostName, String database, PhysicalDBPool dbPool) { this.name = hostName; this.database = database; this.dbPool = dbPool; } public String getName() { return name; } public PhysicalDBPool getDbPool() { return dbPool; } public String getDatabase() { return database; } /** * get connection from the same datasource * * @param exitsCon * @throws Exception */ public void getConnectionFromSameSource(String schema,boolean autocommit, BackendConnection exitsCon, ResponseHandler handler, Object attachment) throws Exception { PhysicalDatasource ds = this.dbPool.findDatasouce(exitsCon); if (ds == null) { throw new RuntimeException( "can't find existing connection,maybe fininshed " + exitsCon); } else { ds.getConnection(schema,autocommit, handler, attachment); } } private void checkRequest(String schema){ if (schema != null && !schema.equals(this.database)) { throw new RuntimeException( "invalid param ,connection request db is :" + schema + " and datanode db is " + this.database); } if (!dbPool.isInitSuccess()) { dbPool.init(dbPool.activedIndex); } } public void getConnection(String schema,boolean autoCommit, RouteResultsetNode rrs, ResponseHandler handler, Object attachment) throws Exception { checkRequest(schema); boolean needMaster = !autoCommit && MycatServer.getInstance().getConfig().getSystem().isStrictTxIsolation(); if (needMaster && rrs.getRunOnSlave()==null){ rrs.setRunOnSlave(false);//#2305 } if (dbPool.isInitSuccess()) { LOGGER.debug("rrs.getRunOnSlave() " + rrs.getRunOnSlaveDebugInfo()); if(rrs.getRunOnSlave() != null){ // 带有 /*db_type=master/slave*/ 注解 // 强制走 slave if(rrs.getRunOnSlave()){ LOGGER.debug("rrs.isHasBlanceFlag() " + rrs.isHasBlanceFlag()); if (rrs.isHasBlanceFlag()) { // 带有 /*balance*/ 注解(目前好像只支持一个注解...) dbPool.getReadBanlanceCon(schema,autoCommit,handler, attachment, this.database); }else{ // 没有 /*balance*/ 注解 LOGGER.debug("rrs.isHasBlanceFlag()" + rrs.isHasBlanceFlag()); if(!dbPool.getReadCon(schema, autoCommit, handler, attachment, this.database)){ LOGGER.warn("Do not have slave connection to use, use master connection instead."); PhysicalDatasource writeSource=dbPool.getSource(); //记录写节点写负载值 writeSource.setWriteCount(); writeSource.getConnection(schema, autoCommit, handler, attachment); rrs.setRunOnSlave(false); rrs.setCanRunInReadDB(false); } } }else{ // 强制走 master // 默认获得的是 writeSource，也就是 走master LOGGER.debug("rrs.getRunOnSlave() " + rrs.getRunOnSlaveDebugInfo()); PhysicalDatasource writeSource=dbPool.getSource(); //记录写节点写负载值 writeSource.setWriteCount(); writeSource.getConnection(schema, autoCommit, handler, attachment); rrs.setCanRunInReadDB(false); } }else{ // 没有 /*db_type=master/slave*/ 注解，按照原来的处理方式 LOGGER.debug("rrs.getRunOnSlave() " + rrs.getRunOnSlaveDebugInfo()); // null if (rrs.canRunnINReadDB(autoCommit)) { dbPool.getRWBanlanceCon(schema,autoCommit, handler, attachment, this.database); } else { PhysicalDatasource writeSource =dbPool.getSource(); //记录写节点写负载值 writeSource.setWriteCount(); writeSource.getConnection(schema, autoCommit, handler, attachment); } } } else { throw new IllegalArgumentException("Invalid DataSource:" + dbPool.getActivedIndex()); } } // public void getConnection(String schema,boolean autoCommit, RouteResultsetNode rrs, // ResponseHandler handler, Object attachment) throws Exception { // checkRequest(schema); // if (dbPool.isInitSuccess()) { // if (rrs.canRunnINReadDB(autoCommit)) { // dbPool.getRWBanlanceCon(schema,autoCommit, handler, attachment, // this.database); // } else { // dbPool.getSource().getConnection(schema,autoCommit, handler, attachment); // } // // } else { // throw new IllegalArgumentException("Invalid DataSource:" // + dbPool.getActivedIndex()); // } // } } ================================================ FILE: src/main/java/io/mycat/backend/datasource/PhysicalDBPool.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.datasource; import io.mycat.MycatServer; import io.mycat.backend.BackendConnection; import io.mycat.backend.heartbeat.DBHeartbeat; import io.mycat.backend.heartbeat.zkprocess.SwitchStatueToZK; import io.mycat.backend.loadbalance.LeastActiveLoadBalance; import io.mycat.backend.loadbalance.LoadBalance; import io.mycat.backend.loadbalance.RandomLoadBalance; import io.mycat.backend.loadbalance.WeightedRoundRobinLoadBalance; import io.mycat.backend.mysql.nio.handler.GetConnectionHandler; import io.mycat.backend.mysql.nio.handler.ResponseHandler; import io.mycat.config.Alarms; import io.mycat.config.loader.zkprocess.comm.ZkConfig; import io.mycat.config.loader.zkprocess.comm.ZkParamCfg; import io.mycat.config.model.DataHostConfig; import io.mycat.util.LogUtil; import io.mycat.util.ZKUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; import java.util.LinkedList; import java.util.Map; import java.util.Random; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.locks.ReentrantLock; public class PhysicalDBPool { protected static final Logger LOGGER = LoggerFactory.getLogger(PhysicalDBPool.class); public static final int BALANCE_NONE = 0; public static final int BALANCE_ALL_BACK = 1; public static final int BALANCE_ALL = 2; public static final int BALANCE_ALL_READ = 3; public static final int RANDOM = 0; public static final int WEIGHTED_ROUND_ROBIN = 1; public static final int LEAST_ACTIVE = 2; public static final int WRITE_ONLYONE_NODE = 0; public static final int WRITE_RANDOM_NODE = 1; public static final int WRITE_ALL_NODE = 2; public static final long LONG_TIME = 300000; public static final int WEIGHT = 0; private final String hostName; protected PhysicalDatasource[] writeSources; protected Map readSources; protected volatile int activedIndex; protected volatile boolean initSuccess; protected final ReentrantLock switchLock = new ReentrantLock(); private final Collection allDs; private final int banlance; private final int writeType; private final Random random = new Random(); private final Random wnrandom = new Random(); private String[] schemas; private final DataHostConfig dataHostConfig; private String slaveIDs; private LoadBalance loadBalance; public PhysicalDBPool(String name, DataHostConfig conf, PhysicalDatasource[] writeSources, Map readSources, int balance, int writeType) { this.hostName = name; this.dataHostConfig = conf; this.writeSources = writeSources; this.banlance = balance; this.writeType = writeType; switch (dataHostConfig.getBalanceType()) { case WEIGHTED_ROUND_ROBIN: loadBalance = new WeightedRoundRobinLoadBalance(); break; case LEAST_ACTIVE: loadBalance = new LeastActiveLoadBalance(); break; default: loadBalance = new RandomLoadBalance(); break; } Iterator> entryItor = readSources.entrySet().iterator(); while (entryItor.hasNext()) { PhysicalDatasource[] values = entryItor.next().getValue(); if (values.length == 0) { entryItor.remove(); } } this.readSources = readSources; this.allDs = this.genAllDataSources(); LOGGER.info("total resources of dataHost " + this.hostName + " is :" + allDs.size()); setDataSourceProps(); } public int getWriteType() { return writeType; } private void setDataSourceProps() { for (PhysicalDatasource ds : this.allDs) { ds.setDbPool(this); } } public PhysicalDatasource findDatasouce(BackendConnection exitsCon) { for (PhysicalDatasource ds : this.allDs) { if ((ds.isReadNode() == exitsCon.isFromSlaveDB()) && ds.isMyConnection(exitsCon)) { return ds; } } LOGGER.warn("can't find connection in pool " + this.hostName + " con:" + exitsCon); return null; } public String getSlaveIDs() { return slaveIDs; } public void setSlaveIDs(String slaveIDs) { this.slaveIDs = slaveIDs; } public String getHostName() { return hostName; } /** * all write datanodes * @return */ public PhysicalDatasource[] getSources() { return writeSources; } public PhysicalDatasource getSource() { switch (writeType) { case WRITE_ONLYONE_NODE: { return writeSources[activedIndex]; } case WRITE_RANDOM_NODE: { int index = Math.abs(wnrandom.nextInt(Integer.MAX_VALUE)) % writeSources.length; PhysicalDatasource result = writeSources[index]; if (!this.isAlive(result)) { // find all live nodes ArrayList alives = new ArrayList(writeSources.length - 1); for (int i = 0; i < writeSources.length; i++) { if (i != index && this.isAlive(writeSources[i])) { alives.add(i); } } if (alives.isEmpty()) { result = writeSources[0]; } else { // random select one index = Math.abs(wnrandom.nextInt(Integer.MAX_VALUE)) % alives.size(); result = writeSources[alives.get(index)]; } } if (LOGGER.isDebugEnabled()) { LOGGER.debug("select write source " + result.getName() + " for dataHost:" + this.getHostName()); } return result; } default: { throw new java.lang.IllegalArgumentException("writeType is " + writeType + " ,so can't return one write datasource "); } } } public int getActivedIndex() { return activedIndex; } public boolean isInitSuccess() { return initSuccess; } public int next(int i) { if (checkIndex(i)) { return (++i == writeSources.length) ? 0 : i; } else { return 0; } } //进行投票选择的节点. private boolean switchSourceVoted(int newIndex, boolean isAlarm, String reason) { if (notSwitchSource(newIndex)) { return false; } final ReentrantLock lock = this.switchLock; if(MycatServer.getInstance().isUseZkSwitch()) { lock.lock(); try { final String myId = ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_MYID); String manageVotePath = ZKUtils.getZKBasePath() +"heartbeat/" + hostName +"/" + "voteInformation/" + myId; String data = String.format("%s=%d", myId,newIndex); ZKUtils.createPath(manageVotePath, data); LogUtil.writeDataSourceLog(String.format("[%s 發生投票: %s]", myId, this.getSources()[newIndex].getName())); } finally { lock.unlock(); } } return true; } //判断是进行zk投票还是直接切换读写 public boolean switchSourceOrVoted(int newIndex, boolean isAlarm, String reason) { if(MycatServer.getInstance().isUseZkSwitch()) { return switchSourceVoted( newIndex, isAlarm, reason); } else { return switchSource( newIndex, isAlarm, reason); } } public boolean notSwitchSource(int newIndex){ return this.writeType != PhysicalDBPool.WRITE_ONLYONE_NODE || !checkIndex(newIndex) ; } public boolean switchSource(int newIndex, boolean isAlarm, String reason) { LOGGER.warn("switchSource: active=" + activedIndex + " new=" + newIndex + " alarm=" + isAlarm + " reason=" + reason); if (notSwitchSource(newIndex)) { return false; } final ReentrantLock lock = this.switchLock; lock.lock(); try { int current = activedIndex; if (current != newIndex) { if(MycatServer.getInstance().isUseZkSwitch()){ LOGGER.info( ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_MYID) + "正在开始进行转换节点 " + hostName+ " = " + newIndex ); SwitchStatueToZK.startSwitch(hostName); } // switch index activedIndex = newIndex; initSuccess = false; // init again this.init(activedIndex, reason); // clear all connections this.getSources()[current].clearCons("switch datasource"); // write log String msg = switchMessage(current, newIndex, false, reason); LOGGER.warn(msg); LogUtil.writeDataSourceLog(msg); if(MycatServer.getInstance().isUseZkSwitch()) { LOGGER.warn(switchMessage(current, newIndex, false, reason)); current = activedIndex; if(!isInitSuccess() || current != newIndex) { LOGGER.error(String.format("%s switch to index %d error ! now index is to switch %d but %d", hostName, newIndex ,newIndex, current)); //报错 然后程序直接挂掉 System.exit(-1); } SwitchStatueToZK.endSwitch(hostName); } return true; } else { if(MycatServer.getInstance().isUseZkSwitch()) { SwitchStatueToZK.startSwitch(hostName); SwitchStatueToZK.endSwitch(hostName); } } } finally { lock.unlock(); } return false; } private String switchMessage(int current, int newIndex, boolean alarm, String reason) { StringBuilder s = new StringBuilder(); if (alarm) { s.append(Alarms.DATANODE_SWITCH); } s.append("[Host=").append(hostName).append(",result=[").append(this.getSources()[current].getName()).append("->"); s.append(this.getSources()[newIndex].getName()).append("],reason=").append(reason).append(']'); return s.toString(); } private int loop(int i) { return i < writeSources.length ? i : (i - writeSources.length); } public void init(int index) { init(index, ""); } public void init(int index, String reason) { if (!checkIndex(index)) { index = 0; } int active = -1; for (int i = 0; i < writeSources.length; i++) { int j = loop(i + index); if ( initSource(j, writeSources[j]) ) { //不切换-1时，如果主写挂了 不允许切换过去 boolean isNotSwitchDs = ( dataHostConfig.getSwitchType() == DataHostConfig.NOT_SWITCH_DS )&& !"MANAGER".equals(reason); if ( isNotSwitchDs && j > 0 ) { break; } active = j; activedIndex = active; initSuccess = true; LOGGER.info(getMessage(active, " init success")); if (this.writeType == WRITE_ONLYONE_NODE) { // only init one write datasource MycatServer.getInstance().saveDataHostIndex(hostName, activedIndex); break; } } } if (!checkIndex(active)) { initSuccess = false; StringBuilder s = new StringBuilder(); s.append(Alarms.DEFAULT).append(hostName).append(" init failure"); LOGGER.error(s.toString()); } } private boolean checkIndex(int i) { return i >= 0 && i < writeSources.length; } private String getMessage(int index, String info) { return new StringBuilder().append(hostName).append(" index:").append(index).append(info).toString(); } private boolean initSource(int index, PhysicalDatasource ds) { int initSize = ds.getConfig().getMinCon(); LOGGER.info("init backend mysql source ,create connections total " + initSize + " for " + ds.getName() + " index :" + index); CopyOnWriteArrayList list = new CopyOnWriteArrayList(); GetConnectionHandler getConHandler = new GetConnectionHandler(list, initSize); // long start = System.currentTimeMillis(); // long timeOut = start + 5000 * 1000L; for (int i = 0; i < initSize; i++) { try { ds.getConnection(this.schemas[i % schemas.length], true, getConHandler, null); } catch (Exception e) { LOGGER.warn(getMessage(index, " init connection error."), e); } } long timeOut = System.currentTimeMillis() + 60 * 1000; // waiting for finish while (!getConHandler.finished() && (System.currentTimeMillis() < timeOut)) { try { Thread.sleep(100); } catch (InterruptedException e) { LOGGER.error("initError", e); } } LOGGER.info("init result :" + getConHandler.getStatusInfo()); // for (BackendConnection c : list) { // c.release(); // } return !list.isEmpty(); } public void doHeartbeat() { if (writeSources == null || writeSources.length == 0) { return; } for (PhysicalDatasource source : this.allDs) { if (source != null) { source.doHeartbeat(); } else { StringBuilder s = new StringBuilder(); s.append(Alarms.DEFAULT).append(hostName).append(" current dataSource is null!"); LOGGER.error(s.toString()); } } } /** * back physical connection heartbeat check */ public void heartbeatCheck(long ildCheckPeriod) { for (PhysicalDatasource ds : allDs) { // only readnode or all write node or writetype=WRITE_ONLYONE_NODE // and current write node will check if (ds != null && (ds.getHeartbeat().getStatus() == DBHeartbeat.OK_STATUS) && (ds.isReadNode() || (this.writeType != WRITE_ONLYONE_NODE) || (this.writeType == WRITE_ONLYONE_NODE && ds == this.getSource()))) { ds.heatBeatCheck(ds.getConfig().getIdleTimeout(), ildCheckPeriod); } } } public void startHeartbeat() { for (PhysicalDatasource source : this.allDs) { source.startHeartbeat(); } } public void stopHeartbeat() { for (PhysicalDatasource source : this.allDs) { source.stopHeartbeat(); } } /** * 强制清除 dataSources * @param reason */ public void clearDataSources(String reason) { LOGGER.info("clear datasource of pool " + this.hostName); for (PhysicalDatasource source : this.allDs) { LOGGER.info("clear datasource of pool " + this.hostName + " ds:" + source.getConfig()); source.clearCons(reason); source.stopHeartbeat(); } } public Collection genAllDataSources() { LinkedList allSources = new LinkedList(); for (PhysicalDatasource ds : writeSources) { if (ds != null) { allSources.add(ds); } } for (PhysicalDatasource[] dataSources : this.readSources.values()) { for (PhysicalDatasource ds : dataSources) { if (ds != null) { allSources.add(ds); } } } return allSources; } public Collection getAllDataSources() { return this.allDs; } /** * return connection for read balance * * @param handler * @param attachment * @param database * @throws Exception */ public void getRWBanlanceCon(String schema, boolean autocommit, ResponseHandler handler, Object attachment, String database) throws Exception { PhysicalDatasource theNode = null; ArrayList okSources = null; switch (banlance) { case BALANCE_ALL_BACK: { // all read nodes and the standard by masters okSources = getAllActiveRWSources(true, false, checkSlaveSynStatus()); if (okSources.isEmpty()) { theNode = this.getSource(); } else { theNode = randomSelect(okSources); } break; } case BALANCE_ALL: { okSources = getAllActiveRWSources(true, true, checkSlaveSynStatus()); theNode = randomSelect(okSources); break; } case BALANCE_ALL_READ: { okSources = getAllActiveRWSources(false, false, checkSlaveSynStatus()); theNode = randomSelect(okSources); break; } case BALANCE_NONE: default: // return default write data source theNode = this.getSource(); } if (LOGGER.isDebugEnabled()) { LOGGER.debug("select read source " + theNode.getName() + " for dataHost:" + this.getHostName()); } //统计节点读操作次数 theNode.setReadCount(); theNode.getConnection(schema, autocommit, handler, attachment); } /** * slave 读负载均衡，也就是 readSource 之间实现负载均衡 * @param schema * @param autocommit * @param handler * @param attachment * @param database * @throws Exception */ public void getReadBanlanceCon(String schema, boolean autocommit, ResponseHandler handler, Object attachment, String database)throws Exception { PhysicalDatasource theNode = null; ArrayList okSources = null; okSources = getAllActiveRWSources(false, false, checkSlaveSynStatus()); theNode = randomSelect(okSources); //统计节点读操作次数 theNode.setReadCount(); theNode.getConnection(schema, autocommit, handler, attachment); } /** * 从 writeHost 下面的 readHost中随机获取一个 connection, 用于slave注解 * @param schema * @param autocommit * @param handler * @param attachment * @param database * @return * @throws Exception */ public boolean getReadCon(String schema, boolean autocommit, ResponseHandler handler, Object attachment, String database)throws Exception { PhysicalDatasource theNode = null; LOGGER.debug("!readSources.isEmpty() " + !readSources.isEmpty()); if (!readSources.isEmpty()) { int index = Math.abs(random.nextInt(Integer.MAX_VALUE)) % readSources.size(); PhysicalDatasource[] allSlaves = this.readSources.get(index); // System.out.println("allSlaves.length " + allSlaves.length); if (allSlaves != null) { index = Math.abs(random.nextInt(Integer.MAX_VALUE)) % readSources.size(); PhysicalDatasource slave = allSlaves[index]; for (int i=0; i okSources) { if (okSources.isEmpty()) { return this.getSource(); } else { return loadBalance.doSelect(hostName, okSources); // int length = okSources.size(); // 总个数 // int totalWeight = 0; // 总权重 // boolean sameWeight = true; // 权重是否都一样 // for (int i = 0; i < length; i++) { // int weight = okSources.get(i).getConfig().getWeight(); // totalWeight += weight; // 累计总权重 // if (sameWeight && i > 0 // && weight != okSources.get(i-1).getConfig().getWeight() ) { // 计算所有权重是否一样 // sameWeight = false; // } // } // // if (totalWeight > 0 && !sameWeight ) { // // // 如果权重不相同且权重大于0则按总权重数随机 // int offset = random.nextInt(totalWeight); // // // 并确定随机值落在哪个片断上 // for (int i = 0; i < length; i++) { // offset -= okSources.get(i).getConfig().getWeight(); // if (offset < 0) { // return okSources.get(i); // } // } // } // // // 如果权重相同或权重为0则均等随机 // return okSources.get( random.nextInt(length) ); // // //int index = Math.abs(random.nextInt()) % okSources.size(); // //return okSources.get(index); } } // public int getBalance() { return banlance; } private boolean isAlive(PhysicalDatasource theSource) { return (theSource.getHeartbeat().getStatus() == DBHeartbeat.OK_STATUS); } private boolean canSelectAsReadNode(PhysicalDatasource theSource) { Integer slaveBehindMaster = theSource.getHeartbeat().getSlaveBehindMaster(); int dbSynStatus = theSource.getHeartbeat().getDbSynStatus(); if ( slaveBehindMaster == null || dbSynStatus == DBHeartbeat.DB_SYN_ERROR) { return false; } boolean isSync = dbSynStatus == DBHeartbeat.DB_SYN_NORMAL; boolean isNotDelay = slaveBehindMaster < this.dataHostConfig.getSlaveThreshold(); return isSync && isNotDelay; } /** * return all backup write sources * * @param includeWriteNode if include write nodes * @param includeCurWriteNode if include current active write node. invalid when includeWriteNode is false * @param filterWithSlaveThreshold * * @return */ private ArrayList getAllActiveRWSources( boolean includeWriteNode, boolean includeCurWriteNode, boolean filterWithSlaveThreshold) { int curActive = activedIndex; ArrayList okSources = new ArrayList(this.allDs.size()); for (int i = 0; i < this.writeSources.length; i++) { PhysicalDatasource theSource = writeSources[i]; if (isAlive(theSource)) {// write node is active if (includeWriteNode) { boolean isCurWriteNode = ( i == curActive ); if ( isCurWriteNode && includeCurWriteNode == false) { // not include cur active source } else if (filterWithSlaveThreshold && theSource.isSalveOrRead() ) { boolean selected = canSelectAsReadNode(theSource); if ( selected ) { okSources.add(theSource); } else { continue; } } else { okSources.add(theSource); } } if (!readSources.isEmpty()) { // check all slave nodes PhysicalDatasource[] allSlaves = this.readSources.get(i); if (allSlaves != null) { for (PhysicalDatasource slave : allSlaves) { if (isAlive(slave)) { if (filterWithSlaveThreshold) { boolean selected = canSelectAsReadNode(slave); if ( selected ) { okSources.add(slave); } else { continue; } } else { okSources.add(slave); } } } } } } else { // TODO : add by zhuam // 如果写节点不OK, 也要保证临时的读服务正常 if ( this.dataHostConfig.isTempReadHostAvailable() && !readSources.isEmpty()) { // check all slave nodes PhysicalDatasource[] allSlaves = this.readSources.get(i); if (allSlaves != null) { for (PhysicalDatasource slave : allSlaves) { if (isAlive(slave)) { if (filterWithSlaveThreshold) { if (canSelectAsReadNode(slave)) { okSources.add(slave); } else { continue; } } else { okSources.add(slave); } } } } } } } return okSources; } public String[] getSchemas() { return schemas; } public void setSchemas(String[] mySchemas) { this.schemas = mySchemas; } } ================================================ FILE: src/main/java/io/mycat/backend/datasource/PhysicalDatasource.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.datasource; import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.LongAdder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.backend.BackendConnection; import io.mycat.backend.ConMap; import io.mycat.backend.ConQueue; import io.mycat.backend.heartbeat.DBHeartbeat; import io.mycat.backend.mysql.nio.MySQLConnection; import io.mycat.backend.mysql.nio.handler.ConnectionHeartBeatHandler; import io.mycat.backend.mysql.nio.handler.DelegateResponseHandler; import io.mycat.backend.mysql.nio.handler.NewConnectionRespHandler; import io.mycat.backend.mysql.nio.handler.ResponseHandler; import io.mycat.config.Alarms; import io.mycat.config.model.DBHostConfig; import io.mycat.config.model.DataHostConfig; import io.mycat.util.TimeUtil; public abstract class PhysicalDatasource { private static final Logger LOGGER = LoggerFactory.getLogger(PhysicalDatasource.class); private final String name; private final int size; private final DBHostConfig config; private final ConMap conMap = new ConMap(); private DBHeartbeat heartbeat; private final boolean readNode; private volatile long heartbeatRecoveryTime; private final DataHostConfig hostConfig; private final ConnectionHeartBeatHandler conHeartBeatHanler = new ConnectionHeartBeatHandler(); private PhysicalDBPool dbPool; private volatile long totalConnectionCount = 0L; //判断是否需要同步 increamentCount private LongAdder increamentCount = new LongAdder(); private long preIncrementCount = 0; // 添加DataSource读计数 private AtomicLong readCount = new AtomicLong(0); // 添加DataSource写计数 private AtomicLong writeCount = new AtomicLong(0); /** * edit by dingw at 2017.06.08 * @see https://github.com/MyCATApache/Mycat-Server/issues/1524 * */ // 当前活动连接 //private volatile AtomicInteger activeCount = new AtomicInteger(0); // 当前存活的总连接数,为什么不直接使用activeCount,主要是因为连接的创建是异步完成的 //private volatile AtomicInteger totalConnection = new AtomicInteger(0); /** * 由于在Mycat中，returnCon被多次调用（与takeCon并没有成对调用）导致activeCount、totalConnection容易出现负数 */ //private static final String TAKE_CONNECTION_FLAG = "1"; //private ConcurrentMap takeConnectionContext = new ConcurrentHashMap<>(); public PhysicalDatasource(DBHostConfig config, DataHostConfig hostConfig, boolean isReadNode) { this.size = config.getMaxCon(); this.config = config; this.name = config.getHostName(); this.hostConfig = hostConfig; heartbeat = this.createHeartBeat(); this.readNode = isReadNode; } public boolean isMyConnection(BackendConnection con) { if (con instanceof MySQLConnection) { return ((MySQLConnection) con).getPool() == this; } else { return false; } } public long getReadCount() { return readCount.get(); } public void setReadCount() { readCount.addAndGet(1); } public long getWriteCount() { return writeCount.get(); } public void setWriteCount() { writeCount.addAndGet(1); } public DataHostConfig getHostConfig() { return hostConfig; } public boolean isReadNode() { return readNode; } public int getSize() { return size; } public void setDbPool(PhysicalDBPool dbPool) { this.dbPool = dbPool; } public PhysicalDBPool getDbPool() { return dbPool; } public abstract DBHeartbeat createHeartBeat(); public String getName() { return name; } public long getExecuteCount() { long executeCount = 0; for (ConQueue queue : conMap.getAllConQueue()) { executeCount += queue.getExecuteCount(); } return executeCount; } public long getExecuteCountForSchema(String schema) { return conMap.getSchemaConQueue(schema).getExecuteCount(); } public int getActiveCountForSchema(String schema) { return conMap.getActiveCountForSchema(schema, this); } public int getIdleCountForSchema(String schema) { ConQueue queue = conMap.getSchemaConQueue(schema); int total = 0; total += queue.getAutoCommitCons().size() + queue.getManCommitCons().size(); return total; } public DBHeartbeat getHeartbeat() { return heartbeat; } public int getIdleCount() { int total = 0; for (ConQueue queue : conMap.getAllConQueue()) { total += queue.getAutoCommitCons().size() + queue.getManCommitCons().size(); } return total; } /** * 该方法也不是非常精确，因为该操作也不是一个原子操作,相对getIdleCount高效与准确一些 * @return */ // public int getIdleCountSafe() { // return getTotalConnectionsSafe() - getActiveCountSafe(); // } /** * 是否需要继续关闭空闲连接 * @return */ // private boolean needCloseIdleConnection() { // return getIdleCountSafe() > hostConfig.getMinCon(); // } private boolean validSchema(String schema) { String theSchema = schema; return theSchema != null && !"".equals(theSchema) && !"snyn...".equals(theSchema); } private void checkIfNeedHeartBeat( LinkedList heartBeatCons, ConQueue queue, ConcurrentLinkedQueue checkLis, long hearBeatTime, long hearBeatTime2) { int maxConsInOneCheck = 10; Iterator checkListItor = checkLis.iterator(); while (checkListItor.hasNext()) { BackendConnection con = checkListItor.next(); if (con.isClosedOrQuit()) { checkListItor.remove(); continue; } if (validSchema(con.getSchema())) { if (con.getLastTime() < hearBeatTime && heartBeatCons.size() < maxConsInOneCheck) { if(checkLis.remove(con)) { //如果移除成功，则放入到心跳连接中，如果移除失败，说明该连接已经被其他线程使用，忽略本次心跳检测 con.setBorrowed(true); heartBeatCons.add(con); } } } else if (con.getLastTime() < hearBeatTime2) { // not valid schema conntion should close for idle // exceed 2*conHeartBeatPeriod // 同样，这里也需要先移除，避免被业务连接 if(checkLis.remove(con)) { con.close(" heart beate idle "); } } } } public int getIndex() { int currentIndex = 0; for (int i = 0; i < dbPool.getSources().length; i++) { PhysicalDatasource writeHostDatasource = dbPool.getSources()[i]; if (writeHostDatasource.getName().equals(getName())) { currentIndex = i; break; } } return currentIndex; } public boolean isSalveOrRead() { int currentIndex = getIndex(); if (currentIndex != dbPool.activedIndex || this.readNode) { return true; } return false; } public void heatBeatCheck(long timeout, long conHeartBeatPeriod) { // int ildeCloseCount = hostConfig.getMinCon() * 3; int maxConsInOneCheck = 5; LinkedList heartBeatCons = new LinkedList(); long hearBeatTime = TimeUtil.currentTimeMillis() - conHeartBeatPeriod; long hearBeatTime2 = TimeUtil.currentTimeMillis() - 2 * conHeartBeatPeriod; for (ConQueue queue : conMap.getAllConQueue()) { checkIfNeedHeartBeat(heartBeatCons, queue, queue.getAutoCommitCons(), hearBeatTime, hearBeatTime2); if (heartBeatCons.size() < maxConsInOneCheck) { checkIfNeedHeartBeat(heartBeatCons, queue, queue.getManCommitCons(), hearBeatTime, hearBeatTime2); } else if (heartBeatCons.size() >= maxConsInOneCheck) { break; } } if (!heartBeatCons.isEmpty()) { for (BackendConnection con : heartBeatCons) { conHeartBeatHanler .doHeartBeat(con, hostConfig.getHearbeatSQL()); } } // check if there has timeouted heatbeat cons conHeartBeatHanler.abandTimeOuttedConns(); int idleCons = getIdleCount(); int activeCons = this.getActiveCount(); int createCount = (hostConfig.getMinCon() - idleCons) / 3; // create if idle too little if ((createCount > 0) && (idleCons + activeCons < size) && (idleCons < hostConfig.getMinCon())) { createByIdleLitte(idleCons, createCount); } else if (idleCons > hostConfig.getMinCon()) { closeByIdleMany(idleCons - hostConfig.getMinCon()); } else { int activeCount = this.getActiveCount(); if (activeCount > size) { StringBuilder s = new StringBuilder(); s.append(Alarms.DEFAULT).append("DATASOURCE EXCEED [name=") .append(name).append(",active="); s.append(activeCount).append(",size=").append(size).append(']'); LOGGER.warn(s.toString()); } } } /** * * @param ildeCloseCount * 首先，从已创建的连接中选择本次心跳需要关闭的空闲连接数（由当前连接连接数-减去配置的最小连接数。 * 然后依次关闭这些连接。由于连接空闲心跳检测与业务是同时并发的，在心跳关闭阶段，可能有连接被使用，导致需要关闭的空闲连接数减少. * * 所以每次关闭新连接时，先判断当前空闲连接数是否大于配置的最少空闲连接，如果为否，则结束本次关闭空闲连接操作。 * 该方法修改之前： * 首先从ConnMap中获取 ildeCloseCount 个连接，然后关闭；在关闭中，可能又有连接被使用，导致可能多关闭一些链接， * 导致相对频繁的创建新连接和关闭连接 * * 该方法修改之后： * ildeCloseCount 为预期要关闭的连接 * 使用循环操作，首先在关闭之前，先再一次判断是否需要关闭连接，然后每次从ConnMap中获取一个空闲连接，然后进行关闭 * edit by dingw at 2017.06.16 */ private void closeByIdleMany(int ildeCloseCount) { LOGGER.info("too many ilde cons ,close some for datasouce " + name); List readyCloseCons = new ArrayList( ildeCloseCount); for (ConQueue queue : conMap.getAllConQueue()) { readyCloseCons.addAll(queue.getIdleConsToClose(ildeCloseCount)); if (readyCloseCons.size() >= ildeCloseCount) { break; } } for (BackendConnection idleCon : readyCloseCons) { if (idleCon.isBorrowed()) { LOGGER.warn("find idle con is using " + idleCon); } idleCon.close("too many idle con"); } // LOGGER.info("too many ilde cons ,close some for datasouce " + name); // // Iterator conQueueIt = conMap.getAllConQueue().iterator(); // ConQueue queue = null; // if(conQueueIt.hasNext()) { // queue = conQueueIt.next(); // } // // for(int i = 0; i < ildeCloseCount; i ++ ) { // // if(!needCloseIdleConnection() || queue == null) { // break; //如果当时空闲连接数没有超过最小配置连接数，则结束本次连接关闭 // } // // LOGGER.info("cur conns:" + getTotalConnectionsSafe() ); // // BackendConnection idleCon = queue.takeIdleCon(false); // // while(idleCon == null && conQueueIt.hasNext()) { // queue = conQueueIt.next(); // idleCon = queue.takeIdleCon(false); // } // // if(idleCon == null) { // break; // } // // if (idleCon.isBorrowed() ) { // LOGGER.warn("find idle con is using " + idleCon); // } // idleCon.close("too many idle con"); // // } } private void createByIdleLitte(int idleCons, int createCount) { LOGGER.info("create connections ,because idle connection not enough ,cur is " + idleCons + ", minCon is " + hostConfig.getMinCon() + " for " + name); NewConnectionRespHandler simpleHandler = new NewConnectionRespHandler(); final String[] schemas = dbPool.getSchemas(); for (int i = 0; i < createCount; i++) { if (this.getActiveCount() + this.getIdleCount() >= size) { break; } try { // creat new connection this.createNewConnection(simpleHandler, null, schemas[i % schemas.length]); } catch (IOException e) { LOGGER.warn("create connection err " + e); } } } public int getActiveCount() { return this.conMap.getActiveCountForDs(this); } public long getTotalCount() { return totalConnectionCount + increamentCount.intValue(); } public void calcTotalCount() { //当连接数增量开始变化的时候，先直接用increamentCount记录连接数，当一秒钟内不再有新增了之后，开始同步 totalConnectionCount if (preIncrementCount == increamentCount.longValue()) { long total = this.conMap.getTotalCountForDs(this); long inc = increamentCount.sumThenReset() - preIncrementCount; totalConnectionCount = total + inc; preIncrementCount = 0; } else { preIncrementCount = increamentCount.longValue(); } } public void clearCons(String reason) { this.conMap.clearConnections(reason, this); } public void startHeartbeat() { heartbeat.start(); } public void stopHeartbeat() { heartbeat.stop(); } public void doHeartbeat() { // 未到预定恢复时间，不执行心跳检测。 if (TimeUtil.currentTimeMillis() < heartbeatRecoveryTime) { return; } if (!heartbeat.isStop()) { try { heartbeat.heartbeat(); } catch (Exception e) { LOGGER.error(name + " heartbeat error.", e); } } } private BackendConnection takeCon(BackendConnection conn, final ResponseHandler handler, final Object attachment, String schema) { conn.setBorrowed(true); // if(takeConnectionContext.putIfAbsent(conn.getId(), TAKE_CONNECTION_FLAG) == null) { // incrementActiveCountSafe(); // } if (!conn.getSchema().equals(schema)) { // need do schema syn in before sql send conn.setSchema(schema); } ConQueue queue = conMap.getSchemaConQueue(schema); queue.incExecuteCount(); conn.setAttachment(attachment); conn.setLastTime(System.currentTimeMillis()); // 每次取连接的时候，更新下lasttime，防止在前端连接检查的时候，关闭连接，导致sql执行失败 handler.connectionAcquired(conn); return conn; } private void createNewConnection(final ResponseHandler handler, final Object attachment, final String schema) throws IOException { // aysn create connection final AtomicBoolean hasError = new AtomicBoolean(false); MycatServer.getInstance().getBusinessExecutor().execute(new Runnable() { public void run() { try { createNewConnection(new DelegateResponseHandler(handler) { @Override public void connectionError(Throwable e, BackendConnection conn) { if(hasError.compareAndSet(false, true)) { handler.connectionError(e, conn); } else { LOGGER.info("connection connectionError "); } } @Override public void connectionAcquired(BackendConnection conn) { LOGGER.info("connection id is "+conn.getId()); takeCon(conn, handler, attachment, schema); } }, schema); } catch (IOException e) { if(hasError.compareAndSet(false, true)) { handler.connectionError(e, null); } else { LOGGER.info("connection connectionError "); } } } }); } public void getConnection(String schema, boolean autocommit, final ResponseHandler handler, final Object attachment) throws IOException { // 从当前连接map中拿取已建立好的后端连接 BackendConnection con = this.conMap.tryTakeCon(schema, autocommit); if (con != null) { //如果不为空，则绑定对应前端请求的handler takeCon(con, handler, attachment, schema); return; } else { // this.getActiveCount并不是线程安全的（严格上说该方法获取数量不准确）， // int curTotalConnection = this.totalConnection.get(); // while(curTotalConnection + 1 <= size) { // // if (this.totalConnection.compareAndSet(curTotalConnection, curTotalConnection + 1)) { // LOGGER.info("no ilde connection in pool,create new connection for " + this.name + " of schema " + schema); // createNewConnection(handler, attachment, schema); // return; // } // // curTotalConnection = this.totalConnection.get(); //CAS更新失败，则重新判断当前连接是否超过最大连接数 // // } // // // 如果后端连接不足，立即失败,故直接抛出连接数超过最大连接异常 // LOGGER.error("the max activeConnnections size can not be max than maxconnections:" + curTotalConnection); // throw new IOException("the max activeConnnections size can not be max than maxconnections:" + curTotalConnection); // 当前最大连接 long activeCons = increamentCount.longValue()+totalConnectionCount; if (activeCons < size) {// 下一个连接大于最大连接数 //提前increamentCount的操作 increamentCount.increment(); LOGGER.info("no ilde connection in pool "+System.identityHashCode(this)+" ,create new connection for " + this.name + " of schema " + schema + " totalConnectionCount: " + totalConnectionCount + " increamentCount: "+increamentCount); createNewConnection(handler, attachment, schema); } else { // create connection LOGGER.error("the max activeConnnections size can not be max than maxconnections"); throw new IOException("the max activeConnnections size can not be max than maxconnections"); } } } /** * 是否超过最大连接数 * @return */ // private boolean exceedMaxConnections() { // return this.totalConnection.get() + 1 > size; // } // // public int decrementActiveCountSafe() { // return this.activeCount.decrementAndGet(); // } // // public int incrementActiveCountSafe() { // return this.activeCount.incrementAndGet(); // } // // public int getActiveCountSafe() { // return this.activeCount.get(); // } // // public int getTotalConnectionsSafe() { // return this.totalConnection.get(); // } // // public int decrementTotalConnectionsSafe() { // return this.totalConnection.decrementAndGet(); // } // // public int incrementTotalConnectionSafe() { // return this.totalConnection.incrementAndGet(); // } private void returnCon(BackendConnection c) { c.setAttachment(null); c.setBorrowed(false); c.setLastTime(TimeUtil.currentTimeMillis()); ConQueue queue = this.conMap.getSchemaConQueue(c.getSchema()); boolean ok = false; if (c.isAutocommit()) { ok = queue.getAutoCommitCons().offer(c); } else { ok = queue.getManCommitCons().offer(c); } // if(c.getId() > 0 && takeConnectionContext.remove(c.getId(), TAKE_CONNECTION_FLAG) ) { // decrementActiveCountSafe(); // } if(!ok) { LOGGER.warn("can't return to pool ,so close con " + c); c.close("can't return to pool "); } } public void releaseChannel(BackendConnection c) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("release channel " + c); } // release connection returnCon(c); } public void connectionClosed(BackendConnection conn) { ConQueue queue = this.conMap.getSchemaConQueue(conn.getSchema()); if (queue != null ) { queue.removeCon(conn); } // decrementTotalConnectionsSafe(); } /** * 创建新连接 */ public abstract void createNewConnection(ResponseHandler handler, String schema) throws IOException; /** * 测试连接，用于初始化及热更新配置检测 */ public abstract boolean testConnection(String schema) throws IOException; public long getHeartbeatRecoveryTime() { return heartbeatRecoveryTime; } public void setHeartbeatRecoveryTime(long heartbeatRecoveryTime) { this.heartbeatRecoveryTime = heartbeatRecoveryTime; } public DBHostConfig getConfig() { return config; } public boolean isAlive() { return getHeartbeat().getStatus() == DBHeartbeat.OK_STATUS; } } ================================================ FILE: src/main/java/io/mycat/backend/heartbeat/ConsistenCollectHandler.java ================================================ package io.mycat.backend.heartbeat; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.mysql.nio.MySQLDataSource; import io.mycat.config.ErrorCode; import io.mycat.config.MycatConfig; import io.mycat.config.model.TableConfig; import io.mycat.manager.ManagerConnection; import io.mycat.manager.response.CheckGlobalTable; import io.mycat.server.interceptor.impl.GlobalTableUtil; import io.mycat.sqlengine.SQLQueryResult; public class ConsistenCollectHandler { public static final Logger LOGGER = LoggerFactory.getLogger(ConsistenCollectHandler.class); private AtomicBoolean isStop =new AtomicBoolean(false); //任务是否停止 private final int retryTime ; //校验次数 private final int dnCount ; //节点数量 private final long intervalTime ; //间隔时间 private final String tableName; //表名 private final String schemaName; //dataBase private final AtomicInteger successTime; //成功次数 private final ManagerConnection con; //session public ConsistenCollectHandler(ManagerConnection c, String tableName, String schemaName, int dnCount, int retryTime, long intervalTime) { this.tableName = tableName; this.schemaName = schemaName; this.retryTime = retryTime; this.intervalTime = intervalTime; successTime = new AtomicInteger(0); this.con = c; this.dnCount = dnCount; } private volatile ScheduledFuture task; //定时发送校验的任务 //定时器不断的检测 public void startDetector(){ task = MycatServer.getInstance().getScheduler() .scheduleAtFixedRate(new ConsisterThread(tableName, schemaName, this), 0, intervalTime, TimeUnit.MILLISECONDS); } private ReentrantLock lock = new ReentrantLock(); Map>>> resultMap = new HashMap<>(); public void onSuccess(SQLQueryResult> result) { if(isStop.get() == true){ return ; } lock.lock(); try{ if(!resultMap.containsKey(result.getDataNode())) { resultMap.put(result.getDataNode(),new ArrayList>>()); } resultMap.get(result.getDataNode()).add(result); } finally { lock.unlock(); } int count = successTime.incrementAndGet(); LOGGER.info(count + " :{}", JSON.toJSONString(result)); if (count == retryTime * dnCount) { cancelTask(); if(LOGGER.isDebugEnabled()){ String str = ""; for(List>> list : resultMap.values()) { str += JSONObject.toJSONString(list) + "\n"; } LOGGER.debug(str); } ///数据的校验 查询最大修改日期和修改时间有交集的节点。 List>> unionResult = resultMap.remove(result.getDataNode()); List>> tempResult = null; for(List>> list : resultMap.values()) { tempResult = new ArrayList<>(); for(SQLQueryResult> r1 : list) { Map md1 = r1.getResult(); String md1_max_column = md1.get(GlobalTableUtil.MAX_COLUMN); for(int i = 0 ; i < unionResult.size(); i++) { Map md2 = unionResult.get(i).getResult(); String md2_max_column = md2.get(GlobalTableUtil.MAX_COLUMN); if(md1.get(GlobalTableUtil.COUNT_COLUMN).equals(md2.get(GlobalTableUtil.COUNT_COLUMN)) && (md1_max_column==null && null == md2_max_column ) || ( md1_max_column != null && md1_max_column.equals(md2_max_column)) ) { tempResult.add(r1); unionResult.remove(unionResult.get(i)); break; } } } unionResult = tempResult; if(unionResult.size() == 0){ break; } } if(unionResult.size() == 0) { LOGGER.debug("check table " +tableName +" not consistence , get info" ); CheckGlobalTable.response(con, tableName, "No"); } else { LOGGER.debug("check table " +tableName +" consistence , consistence info" + JSONObject.toJSONString(unionResult)); CheckGlobalTable.response(con, tableName, "Yes"); } } } //定时器任务的取消。 public void cancelTask() { final ScheduledFuture t = task; if(t != null){ t.cancel(false); task = null; } } public void onError(String msg) { this.cancelTask(); ///将错误消息写回mysql端 if(isStop.compareAndSet(false, true)) { con.writeErrMessage(ErrorCode.ERR_HANDLE_DATA, msg); } } public int getRetryTime() { return retryTime; } } class ConsisterThread implements Runnable{ private final String tableName; private final String schemaName; private final ConsistenCollectHandler handler; private final AtomicInteger sendTime; // public ConsisterThread(String tableName, String schemaName,ConsistenCollectHandler handler) { this.tableName = tableName; this.schemaName = schemaName; this.handler = handler; sendTime = new AtomicInteger(0); } public void run() { int count = sendTime.incrementAndGet(); //所有的校验最大的时间的任务发送完成 if (count > handler.getRetryTime()) { handler.cancelTask(); ConsistenCollectHandler.LOGGER.info(" table check consistence job send finish"); return; } try { MycatConfig config = MycatServer.getInstance().getConfig(); TableConfig table = config.getSchemas().get(schemaName).getTables().get(tableName.toUpperCase()); List dataNodeList = table.getDataNodes(); // 记录本次已经执行的datanode // 多个 datanode 对应到同一个 PhysicalDatasource 只执行一次 for (String nodeName : dataNodeList) { Map map = config.getDataNodes(); for (String k2 : map.keySet()) { // PhysicalDBNode dBnode = map.get(k2); if (nodeName.equals(dBnode.getName())) { // dn1,dn2,dn3 PhysicalDBPool pool = dBnode.getDbPool(); // dataHost Collection allDS = pool.genAllDataSources(); for (PhysicalDatasource pds : allDS) { // if (pds instanceof MySQLDataSource) { MySQLDataSource mds = (MySQLDataSource) pds; MySQLConsistencyChecker checker = new MySQLConsistencyCheckerHandler(dBnode, mds, table.getName(), handler); checker.checkMaxTimeStamp(); break; } } } } } } catch (Exception e) { ConsistenCollectHandler.LOGGER.error("check consisten err: {}", e); handler.onError("ConsisterThread err:" + e.getMessage()); } } } ================================================ FILE: src/main/java/io/mycat/backend/heartbeat/DBHeartbeat.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.heartbeat; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import io.mycat.statistic.DataSourceSyncRecorder; import io.mycat.statistic.HeartbeatRecorder; import io.mycat.util.LogUtil; public abstract class DBHeartbeat { public static final int DB_SYN_ERROR = -1; public static final int DB_SYN_NORMAL = 1; public static final int OK_STATUS = 1; public static final int ERROR_STATUS = -1; public static final int TIMEOUT_STATUS = -2; public static final int INIT_STATUS = 0; private static final long DEFAULT_HEARTBEAT_TIMEOUT = 30 * 1000L; private static final int DEFAULT_HEARTBEAT_RETRY = 10; // heartbeat config protected long heartbeatTimeout = DEFAULT_HEARTBEAT_TIMEOUT; // 心跳超时时间 protected int heartbeatRetry = DEFAULT_HEARTBEAT_RETRY; // 检查连接发生异常到切换，重试次数 protected String heartbeatSQL;// 静态心跳语句 protected final AtomicBoolean isStop = new AtomicBoolean(true); protected final AtomicBoolean isChecking = new AtomicBoolean(false); protected AtomicInteger errorCount = new AtomicInteger(0); protected volatile int status; protected final HeartbeatRecorder recorder = new HeartbeatRecorder(); protected final DataSourceSyncRecorder asynRecorder = new DataSourceSyncRecorder(); private volatile Integer slaveBehindMaster; private volatile int dbSynStatus = DB_SYN_NORMAL; public Integer getSlaveBehindMaster() { return slaveBehindMaster; } public int getDbSynStatus() { return dbSynStatus; } public void setDbSynStatus(int dbSynStatus) { this.dbSynStatus = dbSynStatus; } public void setSlaveBehindMaster(Integer slaveBehindMaster) { this.slaveBehindMaster = slaveBehindMaster; } public int getStatus() { return status; } public boolean isChecking() { return isChecking.get(); } public abstract void start(); public abstract void stop(); public boolean isStop() { return isStop.get(); } public int getErrorCount() { return errorCount.get(); } public HeartbeatRecorder getRecorder() { return recorder; } public abstract String getLastActiveTime(); public abstract long getTimeout(); public abstract void heartbeat(); public long getHeartbeatTimeout() { return heartbeatTimeout; } public void setHeartbeatTimeout(long heartbeatTimeout) { this.heartbeatTimeout = heartbeatTimeout; } public int getHeartbeatRetry() { return heartbeatRetry; } public void setHeartbeatRetry(int heartbeatRetry) { this.heartbeatRetry = heartbeatRetry; } public String getHeartbeatSQL() { return heartbeatSQL; } public void setHeartbeatSQL(String heartbeatSQL) { this.heartbeatSQL = heartbeatSQL; } public boolean isNeedHeartbeat() { return heartbeatSQL != null; } public DataSourceSyncRecorder getAsynRecorder() { return this.asynRecorder; } /* * * @desc 將心跳的狀態寫入到日誌中 * */ protected void writeStatusMsg(String dataHost, String dataSourceName,int nextstatus) { if(status != nextstatus) { StringBuilder msg = new StringBuilder(""); msg.append("[dataHost=").append(dataHost).append(", dataSource=").append(dataSourceName) .append(",statue=").append(getMsg(status)).append(" -> ").append(getMsg(nextstatus)).append("]"); LogUtil.writeDataSourceLog(msg.toString()); } } /* * * @return 獲取對應狀態的字符串狀態 * */ protected String getMsg(int status) { switch (status) { case DBHeartbeat.INIT_STATUS: return "init status"; case DBHeartbeat.TIMEOUT_STATUS: return "timeout status"; case DBHeartbeat.OK_STATUS: return "ok status"; case DBHeartbeat.ERROR_STATUS: return "error status"; default: return "unknown status"; } } } ================================================ FILE: src/main/java/io/mycat/backend/heartbeat/MySQLConsistencyChecker.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.heartbeat; import io.mycat.backend.mysql.nio.MySQLDataSource; import io.mycat.server.interceptor.impl.GlobalTableUtil; import io.mycat.sqlengine.OneRawSQLQueryResultHandler; import io.mycat.sqlengine.SQLJob; import io.mycat.sqlengine.SQLQueryResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.fastjson.JSON; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; /** * @author digdeep@126.com */ public class MySQLConsistencyChecker{ public static final Logger LOGGER = LoggerFactory.getLogger(MySQLConsistencyChecker.class); protected final MySQLDataSource source; protected final ReentrantLock lock; protected AtomicInteger jobCount = new AtomicInteger(); protected String countSQL; protected String maxSQL; protected String tableName; // global table name protected long beginTime; // protected String columnExistSQL = "select count(*) as "+GlobalTableUtil.INNER_COLUMN // + " from information_schema.columns where column_name='" // + GlobalTableUtil.GLOBAL_TABLE_MYCAT_COLUMN + "' and table_name='"; // 此处用到了 mysql 多行转一行 group_concat 的用法，主要是为了简化对结果的处理 // 得到的结果类似于：id,name,_mycat_op_time protected String columnExistSQL = "select group_concat(COLUMN_NAME separator ',') as " + GlobalTableUtil.INNER_COLUMN +" from information_schema.columns where TABLE_NAME='"; //user' and TABLE_SCHEMA='db1'; protected List>> list = new ArrayList<>(); public MySQLConsistencyChecker(MySQLDataSource source, String tableName) { this.source = source; this.lock = new ReentrantLock(false); this.tableName = tableName; this.countSQL = " select count(*) as "+GlobalTableUtil.COUNT_COLUMN+" from " + this.tableName; this.maxSQL = " select max("+GlobalTableUtil.GLOBAL_TABLE_MYCAT_COLUMN+") as "+ GlobalTableUtil.MAX_COLUMN+" from " + this.tableName; this.columnExistSQL += this.tableName +"' "; } public void checkRecordCout() { // ["db3","db2","db1"] lock.lock(); try{ this.jobCount.set(0); beginTime = new Date().getTime(); String[] physicalSchemas = source.getDbPool().getSchemas(); for(String dbName : physicalSchemas){ MySQLConsistencyHelper detector = new MySQLConsistencyHelper(this, null); OneRawSQLQueryResultHandler resultHandler = new OneRawSQLQueryResultHandler(new String[] {GlobalTableUtil.COUNT_COLUMN}, detector); SQLJob sqlJob = new SQLJob(this.getCountSQL(), dbName, resultHandler, source); detector.setSqlJob(sqlJob); sqlJob.run(); this.jobCount.incrementAndGet(); } }finally{ lock.unlock(); } } public void checkMaxTimeStamp() { // ["db3","db2","db1"] lock.lock(); try{ this.jobCount.set(0); beginTime = new Date().getTime(); String[] physicalSchemas = source.getDbPool().getSchemas(); for(String dbName : physicalSchemas){ MySQLConsistencyHelper detector = new MySQLConsistencyHelper(this, null); OneRawSQLQueryResultHandler resultHandler = new OneRawSQLQueryResultHandler(new String[] {GlobalTableUtil.MAX_COLUMN}, detector); SQLJob sqlJob = new SQLJob(this.getMaxSQL(), dbName, resultHandler, source); detector.setSqlJob(sqlJob); sqlJob.run(); this.jobCount.incrementAndGet(); } }finally{ lock.unlock(); } } /** * check inner column exist or not */ public void checkInnerColumnExist() { // ["db3","db2","db1"] lock.lock(); try{ this.jobCount.set(0); beginTime = new Date().getTime(); String[] physicalSchemas = source.getDbPool().getSchemas(); for(String dbName : physicalSchemas){ MySQLConsistencyHelper detector = new MySQLConsistencyHelper(this, null, 1); OneRawSQLQueryResultHandler resultHandler = new OneRawSQLQueryResultHandler(new String[] {GlobalTableUtil.INNER_COLUMN}, detector); String db = " and table_schema='" + dbName + "'"; SQLJob sqlJob = new SQLJob(this.columnExistSQL + db , dbName, resultHandler, source); detector.setSqlJob(sqlJob);//table_schema='db1' LOGGER.debug(sqlJob.toString()); sqlJob.run(); this.jobCount.incrementAndGet(); } }finally{ lock.unlock(); } } public void setResult(SQLQueryResult> result) { // LOGGER.debug("setResult::::::::::" + JSON.toJSONString(result)); lock.lock(); try{ this.jobCount.decrementAndGet(); if(result != null && result.isSuccess()){ result.setTableName(tableName); list.add(result); }else{ if(result != null && result.getResult() != null){ String sql = null; if(result.getResult().containsKey(GlobalTableUtil.COUNT_COLUMN)) sql = this.getCountSQL(); if(result.getResult().containsKey(GlobalTableUtil.MAX_COLUMN)) sql = this.getMaxSQL(); if(result.getResult().containsKey(GlobalTableUtil.INNER_COLUMN)) sql = this.getColumnExistSQL(); LOGGER.warn(sql+ " execute failed in db: " + result.getDataNode() + " during global table consistency check task."); } } if(this.jobCount.get() <= 0 || isTimeOut()){ GlobalTableUtil.finished(list); } }finally{ lock.unlock(); } } public boolean isTimeOut(){ long duration = new Date().getTime() - this.beginTime; return TimeUnit.MINUTES.convert(duration, TimeUnit.MILLISECONDS) > 1; // 1分钟超时 } public String getCountSQL() { return countSQL; } public String getColumnExistSQL() { return columnExistSQL; } public void setColumnExistSQL(String columnExistSQL) { this.columnExistSQL = columnExistSQL; } public String getMaxSQL() { return maxSQL; } public String getTableName() { return tableName; } public MySQLDataSource getSource() { return source; } } ================================================ FILE: src/main/java/io/mycat/backend/heartbeat/MySQLConsistencyCheckerHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.heartbeat; import java.util.Date; import java.util.Map; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.backend.mysql.nio.MySQLDataSource; import io.mycat.server.interceptor.impl.GlobalTableUtil; import io.mycat.sqlengine.OneRawSQLQueryResultHandler; import io.mycat.sqlengine.SQLJob; import io.mycat.sqlengine.SQLQueryResult; /** * @author zwy */ public class MySQLConsistencyCheckerHandler extends MySQLConsistencyChecker{ private final ConsistenCollectHandler handler; private volatile int sqlSeq = 1; private final PhysicalDBNode dbNode; public MySQLConsistencyCheckerHandler(PhysicalDBNode dbNode, MySQLDataSource source, String tableName ,ConsistenCollectHandler handler) { super(source, tableName); this.handler = handler; this.dbNode = dbNode; } //2 public void checkRecordCout() { this.jobCount.set(0); beginTime = new Date().getTime(); String dbName = dbNode.getDatabase(); MySQLConsistencyHelper detector = new MySQLConsistencyHelper(this, null); OneRawSQLQueryResultHandler resultHandler = new OneRawSQLQueryResultHandler(new String[] {GlobalTableUtil.COUNT_COLUMN}, detector); SQLJob sqlJob = new SQLJob(this.getCountSQL(), dbName, resultHandler, source); detector.setSqlJob(sqlJob); this.jobCount.incrementAndGet(); sqlJob.run(); } //1 public void checkMaxTimeStamp() { this.jobCount.set(0); beginTime = new Date().getTime(); String dbName = dbNode.getDatabase(); MySQLConsistencyHelper detector = new MySQLConsistencyHelper(this, null, 0); OneRawSQLQueryResultHandler resultHandler = new OneRawSQLQueryResultHandler(new String[] {GlobalTableUtil.MAX_COLUMN}, detector); SQLJob sqlJob = new SQLJob(this.getMaxSQL(), dbName, resultHandler, source); detector.setSqlJob(sqlJob); this.jobCount.incrementAndGet(); sqlJob.run(); } /** * check inner column exist or not */ //0 public void checkInnerColumnExist() { // ["db3","db2","db1"] this.jobCount.set(0); beginTime = new Date().getTime(); String dbName = dbNode.getDatabase(); MySQLConsistencyHelper detector = new MySQLConsistencyHelper(this, null, 1); OneRawSQLQueryResultHandler resultHandler = new OneRawSQLQueryResultHandler(new String[] {GlobalTableUtil.INNER_COLUMN}, detector); String db = " and table_schema='" + dbName + "'"; SQLJob sqlJob = new SQLJob(this.columnExistSQL + db , dbName, resultHandler, source); detector.setSqlJob(sqlJob);//table_schema='db1' this.jobCount.incrementAndGet(); sqlJob.run(); } public volatile boolean isStop = false; // volatile SQLQueryResult> record = null; volatile SQLQueryResult> resultMap = null; public void setResult(SQLQueryResult> result) { // LOGGER.debug("setResult::::::::::" + JSON.toJSONString(result)); if(isStop){ return ; } if(result != null && result.isSuccess()){ jobCount.decrementAndGet(); String dataNode = result.getDataNode(); result.setTableName(this.getTableName()); if(resultMap == null) { resultMap = result; } else { // SQLQueryResult> r = resultMap; Map metaData = result.getResult(); for(String key : metaData.keySet()) { r.getResult().put(key, metaData.get(key)); } resultMap = r; } }else{ if(result != null && result.getResult() != null) { String sql = null; final int seq = sqlSeq ; if(seq == 0){ sql = this.getColumnExistSQL(); } else if(seq == 1) { sql = this.getMaxSQL(); } else if(seq == 2) { sql = this.getCountSQL(); } else { sql = result.getErrMsg(); } String errMsg = sql+ " execute failed in db: " + result.getDataNode() + " during global table consistency check task."; LOGGER.warn(errMsg); handler.onError(errMsg); } } //任务都完成之后 进行下一个sql校验 if(jobCount.get() == 0 ){ final int seq = ++sqlSeq ; if(seq == 1){ this.checkMaxTimeStamp(); } else if(seq == 2) { this.checkRecordCout(); } else { handler.onSuccess(resultMap); isStop = true; } } else if(isTimeOut()){ String execSql = ""; final int seq = sqlSeq ; if(seq == 0){ execSql = this.getColumnExistSQL(); } else if(seq == 1) { execSql = this.getMaxSQL(); } else if(seq == 2) { execSql = this.getCountSQL(); } isStop = true; handler.onError(String.format("sql %s time out", execSql)); } } } ================================================ FILE: src/main/java/io/mycat/backend/heartbeat/MySQLConsistencyHelper.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.heartbeat; import io.mycat.server.interceptor.impl.GlobalTableUtil; import io.mycat.sqlengine.SQLJob; import io.mycat.sqlengine.SQLQueryResult; import io.mycat.sqlengine.SQLQueryResultListener; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.fastjson.JSON; /** * @author digdeep@126.com */ public class MySQLConsistencyHelper implements SQLQueryResultListener>> { private static final Logger LOGGER = LoggerFactory.getLogger(MySQLConsistencyHelper.class); private MySQLConsistencyChecker heartbeat; private volatile SQLJob sqlJob; private int RETRY_TIMES = 5; private AtomicInteger retryTime = new AtomicInteger(); public MySQLConsistencyHelper(MySQLConsistencyChecker heartbeat, SQLJob sqlJob) { this.heartbeat = heartbeat; this.sqlJob = sqlJob; this.retryTime.set(RETRY_TIMES); } public MySQLConsistencyHelper(MySQLConsistencyChecker heartbeat, SQLJob sqlJob, int retryTime) { this.heartbeat = heartbeat; this.sqlJob = sqlJob; if(retryTime > 0 && retryTime < 10) this.retryTime.set(retryTime); else this.retryTime.set(RETRY_TIMES); } @Override public void onResult(SQLQueryResult> result) { // {"dataNode":"db2","result":{"max_timestamp":"1450423751170"},"success":true} // {"dataNode":"db3","result":{"count(*)":"1"},"success":true} // LOGGER.debug("result:" + JSON.toJSONString(result)); Map rowMap = null; String count = null; String innerCol = null; String maxTimestamp = null; if(result != null) rowMap = result.getResult(); if(rowMap != null){ maxTimestamp = rowMap.get(GlobalTableUtil.MAX_COLUMN); count = rowMap.get(GlobalTableUtil.COUNT_COLUMN); innerCol = rowMap.get(GlobalTableUtil.INNER_COLUMN); if((rowMap.containsKey(GlobalTableUtil.MAX_COLUMN) && StringUtils.isNotBlank(maxTimestamp)) || (rowMap.containsKey(GlobalTableUtil.COUNT_COLUMN) && StringUtils.isNotBlank(count)) || (rowMap.containsKey(GlobalTableUtil.INNER_COLUMN) && StringUtils.isNotBlank(innerCol))){ heartbeat.setResult(result); return; }else{ if(this.retryTime.get() > 0){ try { TimeUnit.MICROSECONDS.sleep(10); } catch (InterruptedException e) { } this.retryTime.decrementAndGet(); this.sqlJob.run(); return; } heartbeat.setResult(result); return; } }else{ if(this.retryTime.get() > 0){ try { TimeUnit.MICROSECONDS.sleep(3); } catch (InterruptedException e) { } this.retryTime.decrementAndGet(); this.sqlJob.run(); return; } heartbeat.setResult(result); return; } } public void close(String msg) { SQLJob curJob = sqlJob; if (curJob != null && !curJob.isFinished()) { curJob.teminate(msg); sqlJob = null; } } public MySQLConsistencyChecker getHeartbeat() { return heartbeat; } public SQLJob getSqlJob() { return sqlJob; } public void setSqlJob(SQLJob sqlJob) { this.sqlJob = sqlJob; } } ================================================ FILE: src/main/java/io/mycat/backend/heartbeat/MySQLDetector.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.heartbeat; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.mysql.nio.MySQLDataSource; import io.mycat.config.model.DataHostConfig; import io.mycat.sqlengine.OneRawSQLQueryResultHandler; import io.mycat.sqlengine.SQLJob; import io.mycat.sqlengine.SQLQueryResult; import io.mycat.sqlengine.SQLQueryResultListener; import io.mycat.util.TimeUtil; /** * @author mycat */ public class MySQLDetector implements SQLQueryResultListener>> { private MySQLHeartbeat heartbeat; private long heartbeatTimeout; private final AtomicBoolean isQuit; private volatile long lastSendQryTime; private volatile long lasstReveivedQryTime; private volatile SQLJob sqlJob; private static final String[] MYSQL_SLAVE_STAUTS_COLMS = new String[] { "Seconds_Behind_Master", "Slave_IO_Running", "Slave_SQL_Running", "Slave_IO_State", "Master_Host", "Master_User", "Master_Port", "Connect_Retry", "Last_IO_Error", "Last_SQL_Error", "Last_SQL_Errno"}; private static final String[] MYSQL_CLUSTER_STAUTS_COLMS = new String[] { "Variable_name", "Value"}; public MySQLDetector(MySQLHeartbeat heartbeat) { this.heartbeat = heartbeat; this.isQuit = new AtomicBoolean(false); } public MySQLHeartbeat getHeartbeat() { return heartbeat; } public long getHeartbeatTimeout() { return heartbeatTimeout; } public void setHeartbeatTimeout(long heartbeatTimeout) { this.heartbeatTimeout = heartbeatTimeout; } public boolean isHeartbeatTimeout() { return TimeUtil.currentTimeMillis() > Math.max(lastSendQryTime, lasstReveivedQryTime) + heartbeatTimeout; } public long getLastSendQryTime() { return lastSendQryTime; } public long getLasstReveivedQryTime() { return lasstReveivedQryTime; } public void heartbeat() { lastSendQryTime = System.currentTimeMillis(); MySQLDataSource ds = heartbeat.getSource(); String databaseName = ds.getDbPool().getSchemas()[0]; String[] fetchColms={}; if (heartbeat.getSource().getHostConfig().isShowSlaveSql() ) { fetchColms=MYSQL_SLAVE_STAUTS_COLMS; } if (heartbeat.getSource().getHostConfig().isShowClusterSql() ) { fetchColms=MYSQL_CLUSTER_STAUTS_COLMS; } OneRawSQLQueryResultHandler resultHandler = new OneRawSQLQueryResultHandler( fetchColms, this); sqlJob = new SQLJob(heartbeat.getHeartbeatSQL(), databaseName, resultHandler, ds); sqlJob.run(); } public void quit() { if (isQuit.compareAndSet(false, true)) { close("heart beat quit"); } } public boolean isQuit() { return isQuit.get(); } @Override public void onResult(SQLQueryResult> result) { if (result.isSuccess()) { int balance = heartbeat.getSource().getDbPool().getBalance(); PhysicalDatasource source = heartbeat.getSource(); int switchType = source.getHostConfig().getSwitchType(); Map resultResult = result.getResult(); if ( resultResult!=null&& !resultResult.isEmpty() &&switchType == DataHostConfig.SYN_STATUS_SWITCH_DS && source.getHostConfig().isShowSlaveSql()) { String Slave_IO_Running = resultResult != null ? resultResult.get("Slave_IO_Running") : null; String Slave_SQL_Running = resultResult != null ? resultResult.get("Slave_SQL_Running") : null; String Last_SQL_Error = resultResult != null ? resultResult.get("Last_SQL_Error") : null; Last_SQL_Error = Last_SQL_Error == null ? "" : Last_SQL_Error; if ("".equals(Last_SQL_Error) && Slave_IO_Running != null && Slave_IO_Running.equals(Slave_SQL_Running) && Slave_SQL_Running.equals("Yes") && source.isSalveOrRead()) { heartbeat.setDbSynStatus(DBHeartbeat.DB_SYN_NORMAL); String Seconds_Behind_Master = resultResult.get( "Seconds_Behind_Master"); if (null == Seconds_Behind_Master ){ MySQLHeartbeat.LOGGER.warn("Master is down but its relay log is clean."); heartbeat.setSlaveBehindMaster(0); }else if(!"".equals(Seconds_Behind_Master)) { int Behind_Master = Integer.parseInt(Seconds_Behind_Master); if ( Behind_Master > source.getHostConfig().getSlaveThreshold() ) { MySQLHeartbeat.LOGGER.warn("found MySQL master/slave Replication delay !!! " + heartbeat.getSource().getConfig() + ", binlog sync time delay: " + Behind_Master + "s" ); } heartbeat.setSlaveBehindMaster( Behind_Master ); } } else if( source.isSalveOrRead() ) { //String Last_IO_Error = resultResult != null ? resultResult.get("Last_IO_Error") : null; MySQLHeartbeat.LOGGER.warn("found MySQL master/slave Replication err !!! " + heartbeat.getSource().getConfig() + ", " + resultResult); heartbeat.setDbSynStatus(DBHeartbeat.DB_SYN_ERROR); } heartbeat.getAsynRecorder().set(resultResult, switchType); heartbeat.setResult(MySQLHeartbeat.OK_STATUS, this, null); } else if ( resultResult!=null&& !resultResult.isEmpty() && switchType==DataHostConfig.CLUSTER_STATUS_SWITCH_DS && source.getHostConfig().isShowClusterSql() ) { //String Variable_name = resultResult != null ? resultResult.get("Variable_name") : null; String wsrep_cluster_status = resultResult != null ? resultResult.get("wsrep_cluster_status") : null;// Primary String wsrep_connected = resultResult != null ? resultResult.get("wsrep_connected") : null;// ON String wsrep_ready = resultResult != null ? resultResult.get("wsrep_ready") : null;// ON if ("ON".equals(wsrep_connected) && "ON".equals(wsrep_ready) && "Primary".equals(wsrep_cluster_status)) { heartbeat.setDbSynStatus(DBHeartbeat.DB_SYN_NORMAL); heartbeat.setResult(MySQLHeartbeat.OK_STATUS, this, null); } else { MySQLHeartbeat.LOGGER.warn("found MySQL cluster status err !!! " + heartbeat.getSource().getConfig() + " wsrep_cluster_status: "+ wsrep_cluster_status + " wsrep_connected: "+ wsrep_connected + " wsrep_ready: "+ wsrep_ready ); heartbeat.setDbSynStatus(DBHeartbeat.DB_SYN_ERROR); heartbeat.setResult(MySQLHeartbeat.ERROR_STATUS, this, null); } heartbeat.getAsynRecorder().set(resultResult, switchType); } else { heartbeat.setResult(MySQLHeartbeat.OK_STATUS, this, null); } //监测数据库同步状态，在 switchType=-1或者1的情况下，也需要收集主从同步状态 heartbeat.getAsynRecorder().set(resultResult, switchType); } else { MySQLHeartbeat.LOGGER.warn("heart beat error: " + heartbeat.getSource().getName() + "/" + heartbeat.getSource().getHostConfig().getName() + " retry=" + heartbeat.getHeartbeatRetry() + " tmo=" + heartbeat.getHeartbeatTimeout()); heartbeat.setResult(MySQLHeartbeat.ERROR_STATUS, this, null); } lasstReveivedQryTime = System.currentTimeMillis(); heartbeat.getRecorder().set((lasstReveivedQryTime - lastSendQryTime)); } public void close(String msg) { SQLJob curJob = sqlJob; if (curJob != null && !curJob.isFinished()) { curJob.teminate(msg); sqlJob = null; } } } ================================================ FILE: src/main/java/io/mycat/backend/heartbeat/MySQLHeartbeat.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.heartbeat; import java.text.SimpleDateFormat; import java.util.Date; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.mysql.nio.MySQLDataSource; import io.mycat.config.model.DataHostConfig; /** * @author mycat */ public class MySQLHeartbeat extends DBHeartbeat { // private static final int MAX_RETRY_COUNT = 5; public static final Logger LOGGER = LoggerFactory.getLogger(MySQLHeartbeat.class); private final MySQLDataSource source; private final ReentrantLock lock; private final int maxRetryCount; private MySQLDetector detector; public MySQLHeartbeat(MySQLDataSource source) { this.source = source; this.lock = new ReentrantLock(false); this.maxRetryCount = source.getHostConfig().getMaxRetryCount(); this.status = INIT_STATUS; this.heartbeatSQL = source.getHostConfig().getHearbeatSQL(); } public MySQLDataSource getSource() { return source; } public MySQLDetector getDetector() { return detector; } public long getTimeout() { MySQLDetector detector = this.detector; if (detector == null) { return -1L; } return detector.getHeartbeatTimeout(); } public String getLastActiveTime() { MySQLDetector detector = this.detector; if (detector == null) { return null; } long t = detector.getLasstReveivedQryTime(); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); return sdf.format(new Date(t)); } public void start() { final ReentrantLock lock = this.lock; lock.lock(); try { isStop.compareAndSet(true, false); super.status = DBHeartbeat.OK_STATUS; } finally { lock.unlock(); } } public void stop() { final ReentrantLock lock = this.lock; lock.lock(); try { if (isStop.compareAndSet(false, true)) { if (isChecking.get()) { // nothing } else { MySQLDetector detector = this.detector; if (detector != null) { detector.quit(); isChecking.set(false); } } } } finally { lock.unlock(); } } /** * execute heart beat */ public void heartbeat() { final ReentrantLock lock = this.lock; if(!lock.tryLock()){ return; } try { if (isChecking.compareAndSet(false, true)) { MySQLDetector detector = this.detector; if (detector == null || detector.isQuit()) { try { detector = new MySQLDetector(this); //由于没有设置导致无限循环. modifyBy zwy todo 对应修改其他的心跳机制. detector.setHeartbeatTimeout(this.getHeartbeatTimeout()); detector.heartbeat(); } catch (Exception e) { LOGGER.warn(source.getConfig().toString(), e); setResult(ERROR_STATUS, detector, null); return; } this.detector = detector; } else { detector.heartbeat(); } } else { MySQLDetector detector = this.detector; if (detector != null) { if (detector.isQuit()) { isChecking.compareAndSet(true, false); } else if (detector.isHeartbeatTimeout()) { setResult(TIMEOUT_STATUS, detector, null); } } } } finally { lock.unlock(); } } public void setResult(int result, MySQLDetector detector, String msg) { this.isChecking.set(false); switch (result) { case OK_STATUS: setOk(detector); break; case ERROR_STATUS: setError(detector); break; case TIMEOUT_STATUS: setTimeout(detector); break; } if (this.status != OK_STATUS) { switchSourceIfNeed("heartbeat error"); } } private void setOk(MySQLDetector detector) { switch (status) { case DBHeartbeat.TIMEOUT_STATUS: writeStatusMsg(source.getDbPool().getHostName(), source.getName() ,DBHeartbeat.INIT_STATUS); this.status = DBHeartbeat.INIT_STATUS; this.errorCount.set(0); //前一个状态为超时 当前状态为正常状态 那就马上发送一个请求 来验证状态是否恢复为Ok if (isStop.get()) { detector.quit(); } else { heartbeat();// timeout, heart beat again } break; case DBHeartbeat.OK_STATUS: this.errorCount.set(0); break; default: writeStatusMsg(source.getDbPool().getHostName(), source.getName() ,DBHeartbeat.OK_STATUS); this.status = OK_STATUS; this.errorCount.set(0);; } if (isStop.get()) { detector.quit(); } } //发生错误了,是否进行下一次心跳检测的策略 . 是否进行下一次心跳检测. private void nextDector(MySQLDetector detector, int nextStatue) { if (isStop.get()) { detector.quit(); writeStatusMsg(source.getDbPool().getHostName(), source.getName() ,DBHeartbeat.OK_STATUS); this.status = nextStatue; } else { // should continues check error status if(errorCount.get() < maxRetryCount) { //设置3秒钟之后重试. if (detector != null && !detector.isQuit()) { LOGGER.error("set Error " + errorCount + " " + this.source.getConfig() ); // source.setHeartbeatRecoveryTime( TimeUtil.currentTimeMillis() + 3000); // heartbeat(); // error count not enough, heart beat again } } else { if (detector != null ) { detector.quit(); } writeStatusMsg(source.getDbPool().getHostName(), source.getName() ,nextStatue); this.status = nextStatue; this.errorCount.set(0); } } } private void setError(MySQLDetector detector) { errorCount.incrementAndGet() ; nextDector(detector, ERROR_STATUS); // should continues check error status // if (errorCount.incrementAndGet() < maxRetryCount) { // // if (detector != null && !detector.isQuit()) { // LOGGER.debug("set Error " + errorCount); // source.setHeartbeatRecoveryTime( TimeUtil.currentTimeMillis() + 3000); // // heartbeat(); // error count not enough, heart beat again // } // // }else // { // if (detector != null ) { // detector.quit(); // } // this.status = ERROR_STATUS; // this.errorCount.set(0); // } } private void setTimeout(MySQLDetector detector) { this.isChecking.set(false); errorCount.incrementAndGet() ; nextDector(detector, TIMEOUT_STATUS); //status = DBHeartbeat.TIMEOUT_STATUS; } /** * switch data source */ private void switchSourceIfNeed(String reason) { int switchType = source.getHostConfig().getSwitchType(); String notSwitch = source.getHostConfig().getNotSwitch(); if (notSwitch.equals(DataHostConfig.FOVER_NOT_SWITCH_DS) || switchType == DataHostConfig.NOT_SWITCH_DS) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("not switch datasource ,for switchType is " + DataHostConfig.NOT_SWITCH_DS); return; } return; } if (LOGGER.isDebugEnabled()) { LOGGER.debug("to switchSourceIfNeed function 进行读节点转换 " ); } PhysicalDBPool pool = this.source.getDbPool(); int curDatasourceHB = pool.getSource().getHeartbeat().getStatus(); // read node can't switch ,only write node can switch if (pool.getWriteType() == PhysicalDBPool.WRITE_ONLYONE_NODE && !source.isReadNode() && curDatasourceHB != DBHeartbeat.OK_STATUS && pool.getSources().length > 1) { synchronized (pool) { // try to see if need switch datasource curDatasourceHB = pool.getSource().getHeartbeat().getStatus(); if (curDatasourceHB != DBHeartbeat.INIT_STATUS && curDatasourceHB != DBHeartbeat.OK_STATUS) { int curIndex = pool.getActivedIndex(); int nextId = pool.next(curIndex); PhysicalDatasource[] allWriteNodes = pool.getSources(); while (true) { if (nextId == curIndex) { break; } PhysicalDatasource theSource = allWriteNodes[nextId]; DBHeartbeat theSourceHB = theSource.getHeartbeat(); int theSourceHBStatus = theSourceHB.getStatus(); if (theSourceHBStatus == DBHeartbeat.OK_STATUS) { if (switchType == DataHostConfig.SYN_STATUS_SWITCH_DS) { LOGGER.warn("switchSourceIfNeed: LagTime=" + theSourceHB.getSlaveBehindMaster()); if (Integer.valueOf(0).equals( theSourceHB.getSlaveBehindMaster())) { LOGGER.info("try to switch datasource ,slave is synchronized to master " + theSource.getConfig()); pool.switchSourceOrVoted(nextId, true, reason); break; } else { LOGGER.warn("ignored datasource ,slave is not synchronized to master , slave behind master :" + theSourceHB.getSlaveBehindMaster() + " " + theSource.getConfig()); } } else { // normal switch LOGGER.info("try to switch datasource ,not checked slave synchronize status " + theSource.getConfig()); pool.switchSourceOrVoted(nextId, true, reason); break; } } nextId = pool.next(nextId); } } } } } } ================================================ FILE: src/main/java/io/mycat/backend/heartbeat/zkprocess/ManageHeartBeatChange.java ================================================ package io.mycat.backend.heartbeat.zkprocess; import java.io.ByteArrayInputStream; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.NodeCache; import org.apache.curator.framework.recipes.cache.PathChildrenCache; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent.Type; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import org.apache.curator.framework.recipes.leader.Participant; import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.util.StringUtil; import io.mycat.util.TimeUtil; import io.mycat.util.ZKUtils; import io.netty.util.internal.ConcurrentSet; //作为leader节点去管理所有的 //注释： /* 源文件名：ManageHeartBeatChange.java * 文件版本：1.0.0 * 创建作者：zwy * 创建日期：2018年5月20日 */ public class ManageHeartBeatChange implements Runnable { public static final Logger LOGGER = LoggerFactory.getLogger(ManageHeartBeatChange.class); public static int ENTER_SELECT = 0; //每个人进行 投票 public static int IS_SELECT = 1; //开始进入统计票数 public static int IS_CHANGING = 2; //正在切换节点. public static int NOT_SELECT = -1; private final String dataHost; private volatile AtomicInteger statue = new AtomicInteger(NOT_SELECT); public ConcurrentSet voteSet = new ConcurrentSet<>(); //投票的结果集 private volatile PathChildrenCache manageVoteCache; //投票的结果集处理 private final CuratorFramework client; private final String path; final ScheduledExecutorService service = MycatServer.getInstance().getHeartbeatScheduler(); //定时器 private volatile NodeCache changingResultNode; //节点切换读写节点的状态的改变 的处理 private final MycatLeaderLatch mycatLeaderLatch; // private InterProcessMutex changingStatueLock; private final String manageVotePath; private final String changingResultPath; //节点切换读写节点的状态的改变的路径. private long maxTimeToWait = 60 * 1000; //最多的等待时间去进行投票结果 private final long minTimeToSwitched = 30 * 60 * 1000; //至少的等待时间去进行下一次切换 private volatile long changingFinishDate = 0; //节点切换读写节点的状态的改变的路径. private volatile ScheduledFuture future = null; private volatile ScheduledFuture changingResultFutrue = null; public ManageHeartBeatChange(MycatLeaderLatch myLeaderLatch, final String dataHost) throws Exception{ statue.set(NOT_SELECT); this.dataHost = dataHost; //dataSource的名称 this.path = ZKUtils.getZKBasePath() +"heartbeat/" + dataHost +"/"; this.manageVotePath = path+ "voteInformation"; this.changingResultPath = path + "changingStatue"; this.client = ZKUtils.getConnection(); changingStatueLock = new InterProcessMutex(client, ZKUtils.getZKBasePath() +"heartbeat/changingStatueLock"); this.mycatLeaderLatch = myLeaderLatch; } //收集投票结果 public boolean addPath(String nodePath) { LOGGER.debug("add vote information " + nodePath); //判断是否可以收集投票结果 如果不行直接删除 if(TimeUtil.currentTimeMillis() - changingFinishDate < minTimeToSwitched && statue.get() == NOT_SELECT ) { try { client.delete().deletingChildrenIfNeeded().forPath(nodePath); } catch (Exception e) { e.printStackTrace(); LOGGER.error("remove vote information debug during not voting time" ,e); } return false; } if(statue.compareAndSet(NOT_SELECT, ENTER_SELECT)){ // beginVoteTime = new Date(); // 开始投票时间 future = service.schedule(this, maxTimeToWait , TimeUnit.MILLISECONDS); } if(statue.get() == ENTER_SELECT) { boolean flag = voteSet.add(nodePath); return flag; } return false; } //清除投票结果 public boolean removePath(String nodePath) { LOGGER.debug("remove vote Information" + nodePath); //删除投票结果. if(statue.get() == IS_CHANGING) { boolean flag = voteSet.remove(nodePath); if(voteSet.isEmpty() && changingResultFutrue == null){ statue.set(NOT_SELECT); } return flag; } return false; } //如果是leader 节点 开始进行监听 public void leaderlisten() { try { if(manageVoteCache != null){ manageVoteCache.close(); } manageVoteCache = new PathChildrenCache(client, manageVotePath, true); final ManageHeartBeatChange manager = this; //监听投票结果, 然后决定需要选举哪一个为最终的结果。 manageVoteCache.getListenable().addListener(new PathChildrenCacheListener() { @Override public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) { // TODO Auto-generated method stub LOGGER.debug("event Type " + event.getType()); // LOGGER.debug( ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_MYID) +" is leader ? " + mycatLeaderLatch.isLeaderShip()); if(null != event.getData()) { Type type = event.getType(); switch(type) { case CHILD_ADDED : { } case CHILD_UPDATED: { manager.addPath(event.getData().getPath()); break; } case CHILD_REMOVED:{ manager.removePath(event.getData().getPath()); break; } default: break; } if(manager.hasCollectFinish()) { manager.run(); } } } }); manageVoteCache.start(); if(changingResultNode != null){ changingResultNode.close(); } //监听切换结果,如果全部完成写入完成的时间 changingResultNode = new NodeCache( client, changingResultPath); changingResultNode.start(); } catch (Exception e) { LOGGER.error(e.getMessage()); e.printStackTrace(); } } //如果不是leader 节点 停止进行监听 public void stop() { try { manageVoteCache.close(); manageVoteCache = null; changingResultNode.close(); changingResultNode = null; //isLeader.compareAndSet(true, false); } catch (Exception e) { LOGGER.error(e.getMessage()); e.printStackTrace(); } } /* * 决定哪个节点为最终的投票结果 * 所有的节点投票完成 或者 5分钟之内有投票的 * */ @Override public void run() { if(!statue.compareAndSet(ENTER_SELECT, IS_SELECT) || getNodeSize() == 0) { return; } if(future != null) { future.cancel(false); future = null; } //获取最后一次的投票时间 如果 List ChildDataList = manageVoteCache.getCurrentData(); Map countMap = new HashMap<>(); Integer maxIndex = -1 ; Integer maxCount= -1 ; try { Collection participants = mycatLeaderLatch.getParticipants(); for(ChildData childData : ChildDataList) { String data = new String(childData.getData()); LOGGER.debug(childData.getPath()+ " " + data); int index = data.indexOf("="); Integer key = Integer.valueOf(data.substring(index + 1)); String myId = data.substring(0, index); //只对在线的节点进行统计 如果某个节点挂了 不再进行统计了 boolean checkExist = false; for(Participant participant : participants) { if(participant.getId().equals(myId)) { checkExist = true; break; } } if(!checkExist){ continue; } Integer value = countMap.get(key); if(value == null) { value = new Integer(0); } value += 1; countMap.put(key, value); //所有总数最大的为投票结果 if(maxCount.compareTo(value) < 0) { maxCount = value; maxIndex = key; } } //节点切换 statue.set(IS_CHANGING); if(maxIndex != -1) { LOGGER.debug("投票结果：" + dataHost + " = " + maxIndex); //向集群写入修改的结果 ZKUtils.createPath(changingResultPath, ""); boolean result = MycatServer.getInstance().saveDataHostIndexToZk(dataHost, maxIndex); if(result) { //开启对结果切换的监控 startChangingResultListen(); } else { //删除投票结果。 try { for(ChildData childData : ChildDataList) { client.delete().deletingChildrenIfNeeded().forPath(childData.getPath()); } } catch (Exception e) { LOGGER.error(e.getMessage()); e.printStackTrace(); }; } } else { LOGGER.debug("投票错误：" + dataHost + " = " + maxIndex); } } catch (Exception e) { LOGGER.error(e.getMessage()); e.printStackTrace(); } } //对状态结果切换的主动监控 private void startChangingResultListen() { //主动监控切换节点的状态 changingResultFutrue = service.scheduleAtFixedRate(new Runnable() { @Override public void run() { ChildData currentData = changingResultNode.getCurrentData(); if(null != currentData) { try{ byte[] data = changingResultNode.getCurrentData().getData(); Properties properties = new Properties(); properties.load(new ByteArrayInputStream(data)); int count = 0; Collection participants = mycatLeaderLatch.getParticipants(); for(Participant participant : participants) { String key = participant.getId() + "_endTime"; String value = properties.getProperty(key); if(!StringUtil.isEmpty(value)) { count ++; }else { LOGGER.debug(String.format("%s 还未结束切换", participant.getId())); } } String changingFinishKey = dataHost + "_changing_finish_time"; String value = properties.getProperty(changingFinishKey); if(!StringUtil.isEmpty(value)) { changingFinishDate = Long.valueOf(value); } int onLineNode = participants.size(); //在线的节点 if(count == onLineNode ) { // LOGGER.debug("所有节点切换完成 ,当前时间" + TimeUtil.currentTimeMillis()); Map propertyMap = new HashMap<>(); propertyMap.put(changingFinishKey, TimeUtil.currentTimeMillis()+""); try{ changingStatueLock.acquire(30, TimeUnit.SECONDS); ZKUtils.writeProperty( changingResultPath, propertyMap); if(changingResultFutrue !=null) { changingResultFutrue.cancel(false); changingResultFutrue = null; } //删除投票结果。 List ChildDataList = manageVoteCache.getCurrentData(); for(ChildData childData : ChildDataList) { client.delete().deletingChildrenIfNeeded().forPath(childData.getPath()); } if(voteSet.isEmpty() ){ //删除投票结果。 statue.set(NOT_SELECT); } }finally { changingStatueLock.release(); } } } catch (Exception e) { LOGGER.error(e.getMessage()); e.printStackTrace(); } } else { LOGGER.debug("集群切换结果的状态文件夹 已经被删除！！！"); } } }, 100, 1000, TimeUnit.MILLISECONDS); } public int getNodeSize(){ return voteSet.size(); } //所有节点收集完毕. public boolean hasCollectFinish() { return voteSet.size() == mycatLeaderLatch.getParticipantsCount(); } } ================================================ FILE: src/main/java/io/mycat/backend/heartbeat/zkprocess/MycatLeaderLatch.java ================================================ package io.mycat.backend.heartbeat.zkprocess; /** * 重定义的leaderLatch 因为curator的选举存在着丢包的情况. * 源文件名：MyLeaderLatch.java * 文件版本：1.0.0 * 创建作者：zwy * 创建日期：2018年5月20日 */ import java.io.IOException; import java.util.Collection; import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.leader.LeaderLatch; import org.apache.curator.framework.recipes.leader.Participant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.config.loader.zkprocess.comm.ZkConfig; import io.mycat.config.loader.zkprocess.comm.ZkParamCfg; import io.mycat.util.ZKUtils; import io.netty.util.internal.ConcurrentSet; public class MycatLeaderLatch { public static final Logger LOGGER = LoggerFactory.getLogger(MycatLeaderLatch.class); // Runnable isLeaderRunnable = null; //当选leader之后的回调方法 // Runnable notLeaderRunnable = null; //失去leader之后的回调方法. private final String latchPath; volatile LeaderLatch latch; String myId; CuratorFramework client; int isLeaderCount = 0; int isSlaveCount = 0; volatile boolean isLeader; final ScheduledExecutorService service = Executors.newScheduledThreadPool(1); ConcurrentSet manageHeartBeatChangeSet = new ConcurrentSet<>(); public MycatLeaderLatch( String latchPath ) { this.myId = ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_MYID); this.latchPath = ZKUtils.getZKBasePath() + latchPath; this.client = ZKUtils.getConnection(); isLeader = false; //ZKUtils.createPath(this.latchPath, ""); latch = new LeaderLatch(client, this.latchPath ,this.myId); Map dataSourceHosts = MycatServer.getInstance().getConfig().getDataHosts(); try { for(String dataSource : dataSourceHosts.keySet()) { manageHeartBeatChangeSet.add(new ManageHeartBeatChange(this, dataSource)); } } catch (Exception e) { LOGGER.warn("init ManageHeartBeatChange err:", e); } } //成为leader的回调方法 private void isLeaderRunnable() { // LOGGER.debug(ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_MYID) + " success to leader"); for(ManageHeartBeatChange manageHeartBeatChange: manageHeartBeatChangeSet) { manageHeartBeatChange.leaderlisten(); } } //不再是leader的回调方法 private void notLeaderRunnable() { LOGGER.debug(ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_MYID) + " fail to leader, now is slave"); for(ManageHeartBeatChange manageHeartBeatChange: manageHeartBeatChangeSet) { manageHeartBeatChange.stop(); } } public void start() throws Exception { latch.start(); service.scheduleAtFixedRate(checkIsLeader(), 0, 5, TimeUnit.SECONDS); } public void stop() throws IOException { latch.close(); service.shutdown(); } private Runnable checkIsLeader() { return new Runnable() { @Override public void run() { boolean isExist = false; try { Collection participants = latch.getParticipants(); for (Participant participant : participants) { if (myId.equals(participant.getId())) { isExist = true; break; } } if(!isExist) { //输出已经不再集群中了哦 LOGGER.info(myId + " current does not exist on zk"); latch.close(); latch = new LeaderLatch(client, latchPath ,myId); latch.start(); LOGGER.info(myId + " success reset leaderLatch @ " + latchPath); } //查看当前leader是否是自己 //注意，不能用leaderLatch.hasLeadership()因为有zk数据丢失的不确定性 //利用serverId对比确认是否主为自己 Participant leader = latch.getLeader(); boolean hashLeaderShip = myId.equals(leader.getId()); judgeIsLeader(hashLeaderShip); } catch (Exception e) { judgeIsLeader(false); e.printStackTrace(); } } }; } //缓冲区 判断是否是leader public void judgeIsLeader(boolean hashLeaderShip) { //主从切换缓冲 if(hashLeaderShip) { isLeaderCount++; isSlaveCount = 0; } else { isLeaderCount = 0; isSlaveCount ++; } if (isLeaderCount > 3 && !isLeader) { LOGGER.info(myId + " Currently run as leader"); isLeader = true; //执行换为leader的方法 //isLeaderRunnable.run(); isLeaderRunnable(); } if (isSlaveCount > 3 && isLeader) { LOGGER.info(myId + " Currently run as slave"); isLeader = false; notLeaderRunnable(); } } //是否leader节点 public boolean isLeaderShip(){ return isSlaveCount == 0 && isLeader; } public Collection getParticipants() throws Exception { return latch.getParticipants(); } public int getParticipantsCount() { try { return latch.getParticipants().size(); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); LOGGER.error("get clusters number error"); return 0; } } } ================================================ FILE: src/main/java/io/mycat/backend/heartbeat/zkprocess/SwitchStatueToZK.java ================================================ package io.mycat.backend.heartbeat.zkprocess; import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.config.loader.zkprocess.comm.ZkConfig; import io.mycat.config.loader.zkprocess.comm.ZkParamCfg; import io.mycat.util.ZKUtils; public class SwitchStatueToZK{ public static final Logger LOGGER = LoggerFactory.getLogger(SwitchStatueToZK.class); private static InterProcessMutex changingStatueLock; static { changingStatueLock = new InterProcessMutex(ZKUtils.getConnection(), ZKUtils.getZKBasePath() +"heartbeat/changingStatueLock"); } public static boolean startSwitch(String dataHost) { String path = ZKUtils.getZKBasePath() +"heartbeat/" + dataHost +"/"; String changingResultPath = path + "changingStatue"; Map propertyMap = new HashMap<>(); String myId = ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_MYID); propertyMap.put(myId+"_changing_statue","switching now"); //状态 propertyMap.put(myId + "_startTime",new Date().toString()); //切换开始时间 propertyMap.put(myId + "_endTime", ""); //结束时间 try{ try { changingStatueLock.acquire(30000, TimeUnit.MILLISECONDS); ZKUtils.writeProperty(changingResultPath, propertyMap); } finally { changingStatueLock.release(); } return true; }catch (Exception e) { LOGGER.error(dataHost + " startSwitch err " , e); } return false; } public static boolean endSwitch(String dataHost) { String path = ZKUtils.getZKBasePath() +"heartbeat/" + dataHost +"/"; String changingResultPath = path + "changingStatue"; Map propertyMap = new HashMap<>(); String myId = ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_MYID); propertyMap.put(myId+"_changing_statue","switching success"); //状态 propertyMap.put(myId + "_endTime",new Date().toString()); //切换结束时间 try{ try { changingStatueLock.acquire(30000, TimeUnit.MILLISECONDS); ZKUtils.writeProperty(changingResultPath, propertyMap); } finally { changingStatueLock.release(); } return true; }catch (Exception e) { LOGGER.error(dataHost + " endSwitch err " , e); return false; } } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/JDBCConnection.java ================================================ package io.mycat.backend.jdbc; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.math.BigDecimal; import java.nio.ByteBuffer; import java.sql.CallableStatement; import java.sql.Connection; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.backend.BackendConnection; import io.mycat.backend.mysql.PacketUtil; import io.mycat.backend.mysql.nio.handler.ConnectionHeartBeatHandler; import io.mycat.backend.mysql.nio.handler.ResponseHandler; import io.mycat.config.ErrorCode; import io.mycat.config.Isolations; import io.mycat.net.NIOProcessor; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.ErrorPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.OkPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.route.Procedure; import io.mycat.route.ProcedureParameter; import io.mycat.route.RouteResultsetNode; import io.mycat.server.ServerConnection; import io.mycat.server.parser.ServerParse; import io.mycat.util.MysqlDefs; import io.mycat.util.ObjectUtil; import io.mycat.util.ResultSetUtil; import io.mycat.util.StringUtil; import io.mycat.util.TimeUtil; public class JDBCConnection implements BackendConnection { protected static final Logger LOGGER = LoggerFactory .getLogger(JDBCConnection.class); private JDBCDatasource pool; private volatile String schema; private volatile String dbType; private volatile String oldSchema; private byte packetId; private int txIsolation; private volatile boolean running = false; private volatile boolean borrowed; private long id = 0; private String host; private int port; private Connection con; private ResponseHandler respHandler; private volatile Object attachement; boolean headerOutputed = false; private volatile boolean modifiedSQLExecuted; private final long startTime; private long lastTime; private boolean isSpark = false; private NIOProcessor processor; private boolean setSchemaFail = false; private volatile int sqlSelectLimit = -1; private boolean fromSlaveDB; public NIOProcessor getProcessor() { return processor; } public void setProcessor(NIOProcessor processor) { this.processor = processor; } public JDBCConnection() { startTime = System.currentTimeMillis(); } public Connection getCon() { return con; } public void setCon(Connection con) { this.con = con; } @Override public void close(String reason) { try { try { if (!isAutocommit()) { rollback(); con.setAutoCommit(true); } }catch (Exception e){ LOGGER.error("close jdbc connection, found it is in transcation so try to rollback"); } con.close(); if(processor!=null){ processor.removeConnection(this); } } catch (SQLException e) { } } @Override public void closeWithoutRsp(String reason) { // TODO Auto-generated method stub close(reason); } public void setId(long id) { this.id = id; } public JDBCDatasource getPool() { return pool; } public void setPool(JDBCDatasource pool) { this.pool = pool; } public void setHost(String host) { this.host = host; } public void setPort(int port) { this.port = port; } @Override public boolean isClosed() { try { return con == null || con.isClosed(); } catch (SQLException e) { return true; } } @Override public void idleCheck() { if(TimeUtil.currentTimeMillis() > lastTime + pool.getConfig().getIdleTimeout()){ close(" idle check"); } } @Override public long getStartupTime() { return startTime; } @Override public String getHost() { return this.host; } @Override public int getPort() { return this.port; } @Override public int getLocalPort() { return 0; } @Override public long getNetInBytes() { return 0; } @Override public long getNetOutBytes() { return 0; } @Override public boolean isModifiedSQLExecuted() { return modifiedSQLExecuted; } @Override public boolean isFromSlaveDB() { return fromSlaveDB; } public String getDbType() { return this.dbType; } public void setDbType(String newDbType) { this.dbType = newDbType.toUpperCase(); this.isSpark = dbType.equals("SPARK"); } @Override public String getSchema() { return this.schema; } @Override public void setSchema(String newSchema) { this.oldSchema = this.schema; this.schema = newSchema; } @Override public long getLastTime() { return lastTime; } @Override public boolean isClosedOrQuit() { return this.isClosed(); } @Override public void setAttachment(Object attachment) { this.attachement = attachment; } @Override public void quit() { this.close("client quit"); } @Override public void setLastTime(long currentTimeMillis) { this.lastTime = currentTimeMillis; } @Override public void release() { modifiedSQLExecuted = false; setResponseHandler(null); pool.releaseChannel(this); } public void setRunning(boolean running) { this.running = running; } @Override public boolean setResponseHandler(ResponseHandler commandHandler) { respHandler = commandHandler; return false; } @Override public void commit() { try { if (con.getAutoCommit()) { LOGGER.warn("when jdbc con is autocommit call commit"); } else { con.commit(); } this.respHandler.okResponse(OkPacket.OK, this); } catch (SQLException e) { throw new RuntimeException(e); } } private int convertNativeIsolationToJDBC(int nativeIsolation) { if(nativeIsolation== Isolations.REPEATED_READ) { return Connection.TRANSACTION_REPEATABLE_READ; }else if(nativeIsolation== Isolations.SERIALIZABLE) { return Connection.TRANSACTION_SERIALIZABLE; } else { return nativeIsolation; } } private void syncTxReadonly(boolean txReadonly) { if(isTxReadonly() == txReadonly) { return; } try { con.setReadOnly(false); } catch (SQLException e) { LOGGER.warn("set setReadOnly error:",e); } } private void syncIsolation(int nativeIsolation) { int jdbcIsolation=convertNativeIsolationToJDBC(nativeIsolation); int srcJdbcIsolation= getTxIsolation(); if (jdbcIsolation == srcJdbcIsolation || "oracle".equalsIgnoreCase(getDbType()) && jdbcIsolation != Connection.TRANSACTION_READ_COMMITTED && jdbcIsolation != Connection.TRANSACTION_SERIALIZABLE) { return; } try { con.setTransactionIsolation(jdbcIsolation); } catch (SQLException e) { LOGGER.warn("set txisolation error:",e); } } private void executeSQL(RouteResultsetNode rrn, ServerConnection sc, boolean autocommit) throws IOException { String orgin = rrn.getStatement(); // String sql = rrn.getStatement().toLowerCase(); // LOGGER.info("JDBC SQL:"+orgin+"|"+sc.toString()); if (!modifiedSQLExecuted && rrn.isModifySQL()) { modifiedSQLExecuted = true; } try { syncIsolation(sc.getTxIsolation()) ; syncTxReadonly(sc.isTxReadonly()); if (!this.schema.equals(this.oldSchema)) { con.setCatalog(schema); if (!setSchemaFail) { try { con.setSchema(schema); //add@byron to test } catch (Throwable e) { LOGGER.error("JDBC setSchema Exception for " + schema, e); setSchemaFail = true; } } this.oldSchema = schema; } if (!this.isSpark) { con.setAutoCommit(autocommit); } int sqlType = rrn.getSqlType(); if(rrn.isCallStatement()&&"oracle".equalsIgnoreCase(getDbType())) { //存储过程暂时只支持oracle ouputCallStatement(rrn,sc,orgin); } else if (sqlType == ServerParse.SELECT || sqlType == ServerParse.SHOW) { if ((sqlType == ServerParse.SHOW) && (!dbType.equals("MYSQL"))) { // showCMD(sc, orgin); //ShowVariables.execute(sc, orgin); ShowVariables.execute(sc, orgin,this); } else if ("SELECT CONNECTION_ID()".equalsIgnoreCase(orgin)) { //ShowVariables.justReturnValue(sc,String.valueOf(sc.getId())); ShowVariables.justReturnValue(sc,String.valueOf(sc.getId()),this); } else { ouputResultSet(sc, orgin); } } else { executeddl(sc, orgin); } } catch (SQLException e) { String msg = e.getMessage(); ErrorPacket error = new ErrorPacket(); error.packetId = ++packetId; error.errno = e.getErrorCode(); error.message = msg.getBytes(); LOGGER.error("sql execute error, "+ msg , e); this.respHandler.errorResponse(error.writeToBytes(sc), this); } catch (Exception e) { String msg = e.getMessage(); ErrorPacket error = new ErrorPacket(); error.packetId = ++packetId; error.errno = ErrorCode.ER_UNKNOWN_ERROR; error.message = ((msg == null) ? e.toString().getBytes() : msg.getBytes()); String err = null; if(error.message!=null){ err = new String(error.message); } LOGGER.error("sql execute error, "+ err , e); this.respHandler.errorResponse(error.writeToBytes(sc), this); } finally { this.running = false; } } private FieldPacket getNewFieldPacket(String charset, String fieldName) { FieldPacket fieldPacket = new FieldPacket(); fieldPacket.orgName = StringUtil.encode(fieldName, charset); fieldPacket.name = StringUtil.encode(fieldName, charset); fieldPacket.length = 20; fieldPacket.flags = 0; fieldPacket.decimals = 0; int javaType = 12; fieldPacket.type = (byte) (MysqlDefs.javaTypeMysql(javaType) & 0xff); return fieldPacket; } private void executeddl(ServerConnection sc, String sql) throws SQLException { Statement stmt = null; try { stmt = con.createStatement(); int count = stmt.executeUpdate(sql,Statement.RETURN_GENERATED_KEYS); long lastInsertId = 0; if("mysql".equalsIgnoreCase(getDbType())) { ResultSet generatedKeys = stmt.getGeneratedKeys(); if (generatedKeys != null){ ResultSetMetaData metaData = generatedKeys.getMetaData(); if (metaData.getColumnCount() == 1){ lastInsertId = (generatedKeys.next() ? generatedKeys.getLong(1) : 0L); } } } OkPacket okPck = new OkPacket(); okPck.affectedRows = count; okPck.insertId = lastInsertId; okPck.packetId = ++packetId; okPck.message = " OK!".getBytes(); this.respHandler.okResponse(okPck.writeToBytes(sc), this); }catch (Exception e){ LOGGER.error("",e); throw e; }finally { if (stmt != null) { try { stmt.close(); } catch (SQLException e) { LOGGER.error("",e); } } } } private static int oracleCURSORTypeValue=-10; static { Object cursor = ObjectUtil.getStaticFieldValue("oracle.jdbc.OracleTypes", "CURSOR"); if(cursor!=null) { oracleCURSORTypeValue = (int) cursor; } } private void ouputCallStatement(RouteResultsetNode rrn,ServerConnection sc, String sql) throws SQLException { CallableStatement stmt = null; ResultSet rs = null; try { Procedure procedure = rrn.getProcedure(); Collection paramters= procedure.getParamterMap().values(); String callSql = procedure.toPreCallSql(null); stmt = con.prepareCall(callSql); if (sc.getSqlSelectLimit() > 0) { stmt.setMaxRows(sc.getSqlSelectLimit()); } for (ProcedureParameter paramter : paramters) { if((ProcedureParameter.IN.equalsIgnoreCase(paramter.getParameterType()) ||ProcedureParameter.INOUT.equalsIgnoreCase(paramter.getParameterType()))) { Object value= paramter.getValue()!=null ?paramter.getValue():paramter.getName(); stmt.setObject(paramter.getIndex(),value); } if(ProcedureParameter.OUT.equalsIgnoreCase(paramter.getParameterType()) ||ProcedureParameter.INOUT.equalsIgnoreCase(paramter.getParameterType()) ) { int jdbcType ="oracle".equalsIgnoreCase(getDbType())&& procedure.getListFields().contains(paramter.getName())?oracleCURSORTypeValue: paramter.getJdbcType(); stmt.registerOutParameter(paramter.getIndex(), jdbcType); } } boolean hadResults= stmt.execute(); ByteBuffer byteBuf = sc.allocate(); if(procedure.getSelectColumns().size()>0&&!procedure.isResultList()) { List fieldPks = new LinkedList(); for (ProcedureParameter paramter : paramters) { if (!procedure.getListFields().contains(paramter.getName())&&(ProcedureParameter.OUT.equalsIgnoreCase(paramter.getParameterType()) || ProcedureParameter.INOUT.equalsIgnoreCase(paramter.getParameterType())) ) { FieldPacket packet = PacketUtil.getField(paramter.getName(), MysqlDefs.javaTypeMysql(paramter.getJdbcType())); fieldPks.add(packet); } } int colunmCount = fieldPks.size(); ResultSetHeaderPacket headerPkg = new ResultSetHeaderPacket(); headerPkg.fieldCount = fieldPks.size(); headerPkg.packetId = ++packetId; byteBuf = headerPkg.write(byteBuf, sc, true); byteBuf.flip(); byte[] header = new byte[byteBuf.limit()]; byteBuf.get(header); byteBuf.clear(); List fields = new ArrayList(fieldPks.size()); Iterator itor = fieldPks.iterator(); while (itor.hasNext()) { FieldPacket curField = itor.next(); curField.packetId = ++packetId; byteBuf = curField.write(byteBuf, sc, false); byteBuf.flip(); byte[] field = new byte[byteBuf.limit()]; byteBuf.get(field); byteBuf.clear(); fields.add(field); itor.remove(); } EOFPacket eofPckg = new EOFPacket(); eofPckg.packetId = ++packetId; byteBuf = eofPckg.write(byteBuf, sc, false); byteBuf.flip(); byte[] eof = new byte[byteBuf.limit()]; byteBuf.get(eof); byteBuf.clear(); this.respHandler.fieldEofResponse(header, fields, eof, this); RowDataPacket curRow = new RowDataPacket(colunmCount); for (String name : procedure.getSelectColumns()) { ProcedureParameter procedureParameter= procedure.getParamterMap().get(name); Object object = stmt.getObject(procedureParameter.getIndex()); if (object != null){ curRow.add(StringUtil.encode(String.valueOf(object), sc.getCharset())); }else { curRow.add(null); } } curRow.packetId = ++packetId; byteBuf = curRow.write(byteBuf, sc, false); byteBuf.flip(); byte[] row = new byte[byteBuf.limit()]; byteBuf.get(row); byteBuf.clear(); this.respHandler.rowResponse(row, this); eofPckg = new EOFPacket(); eofPckg.packetId = ++packetId; if(procedure.isResultList()) { eofPckg.status = 42; } byteBuf = eofPckg.write(byteBuf, sc, false); byteBuf.flip(); eof = new byte[byteBuf.limit()]; byteBuf.get(eof); byteBuf.clear(); this.respHandler.rowEofResponse(eof, this); } if(procedure.isResultList()) { List fieldPks = new LinkedList(); int listSize=procedure.getListFields().size(); for (ProcedureParameter paramter : paramters) { if (procedure.getListFields().contains(paramter.getName())&&(ProcedureParameter.OUT.equalsIgnoreCase(paramter.getParameterType()) || ProcedureParameter.INOUT.equalsIgnoreCase(paramter.getParameterType())) ) { listSize--; Object object = stmt.getObject(paramter.getIndex()); rs= (ResultSet) object; if(rs==null) { continue; } ResultSetUtil.resultSetToFieldPacket(sc.getCharset(), fieldPks, rs, this.isSpark); int colunmCount = fieldPks.size(); ResultSetHeaderPacket headerPkg = new ResultSetHeaderPacket(); headerPkg.fieldCount = fieldPks.size(); headerPkg.packetId = ++packetId; byteBuf = headerPkg.write(byteBuf, sc, true); byteBuf.flip(); byte[] header = new byte[byteBuf.limit()]; byteBuf.get(header); byteBuf.clear(); List fields = new ArrayList(fieldPks.size()); Iterator itor = fieldPks.iterator(); while (itor.hasNext()) { FieldPacket curField = itor.next(); curField.packetId = ++packetId; byteBuf = curField.write(byteBuf, sc, false); byteBuf.flip(); byte[] field = new byte[byteBuf.limit()]; byteBuf.get(field); byteBuf.clear(); fields.add(field); itor.remove(); } EOFPacket eofPckg = new EOFPacket(); eofPckg.packetId = ++packetId; byteBuf = eofPckg.write(byteBuf, sc, false); byteBuf.flip(); byte[] eof = new byte[byteBuf.limit()]; byteBuf.get(eof); byteBuf.clear(); this.respHandler.fieldEofResponse(header, fields, eof, this); // output row while (rs.next()) { RowDataPacket curRow = new RowDataPacket(colunmCount); for (int i = 0; i < colunmCount; i++) { int j = i + 1; Object object1 = rs.getObject(j); if (object1 == null){ curRow.add(null); }else { curRow.add(StringUtil.encode(Objects.toString(object1), sc.getCharset())); } } curRow.packetId = ++packetId; byteBuf = curRow.write(byteBuf, sc, false); byteBuf.flip(); byte[] row = new byte[byteBuf.limit()]; byteBuf.get(row); byteBuf.clear(); this.respHandler.rowResponse(row, this); } eofPckg = new EOFPacket(); eofPckg.packetId = ++packetId; if(listSize!=0) { eofPckg.status = 42; } byteBuf = eofPckg.write(byteBuf, sc, false); byteBuf.flip(); eof = new byte[byteBuf.limit()]; byteBuf.get(eof); byteBuf.clear(); this.respHandler.rowEofResponse(eof, this); } } } if(!procedure.isResultSimpleValue()) { byte[] OK = new byte[] { 7, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0 }; OK[3]=++packetId; this.respHandler.okResponse(OK,this); } sc.recycle(byteBuf); } finally { if (rs != null) { try { rs.close(); } catch (SQLException e) { } } if (stmt != null) { try { stmt.close(); } catch (SQLException e) { } } } } private void ouputResultSet(ServerConnection sc, String sql) throws SQLException { ResultSet rs = null; Statement stmt = null; try { stmt = con.createStatement(); if (sc.getSqlSelectLimit() > 0) { stmt.setMaxRows(sc.getSqlSelectLimit()); } rs = stmt.executeQuery(sql); List fieldPks = new LinkedList(); ResultSetUtil.resultSetToFieldPacket(sc.getCharset(), fieldPks, rs, this.isSpark); int colunmCount = fieldPks.size(); ByteBuffer byteBuf = sc.allocate(); ResultSetHeaderPacket headerPkg = new ResultSetHeaderPacket(); headerPkg.fieldCount = fieldPks.size(); headerPkg.packetId = ++packetId; byteBuf = headerPkg.write(byteBuf, sc, true); byteBuf.flip(); byte[] header = new byte[byteBuf.limit()]; byteBuf.get(header); byteBuf.clear(); List fields = new ArrayList(fieldPks.size()); Iterator itor = fieldPks.iterator(); while (itor.hasNext()) { FieldPacket curField = itor.next(); curField.packetId = ++packetId; byteBuf = curField.write(byteBuf, sc, false); byteBuf.flip(); byte[] field = new byte[byteBuf.limit()]; byteBuf.get(field); byteBuf.clear(); fields.add(field); } EOFPacket eofPckg = new EOFPacket(); eofPckg.packetId = ++packetId; byteBuf = eofPckg.write(byteBuf, sc, false); byteBuf.flip(); byte[] eof = new byte[byteBuf.limit()]; byteBuf.get(eof); byteBuf.clear(); this.respHandler.fieldEofResponse(header, fields, eof, this); // output row while (rs.next()) { RowDataPacket curRow = new RowDataPacket(colunmCount); for (int i = 0; i < colunmCount; i++) { int j = i + 1; if(MysqlDefs.isBianry((byte) fieldPks.get(i).type)) { curRow.add(rs.getBytes(j)); } else if(fieldPks.get(i).type == MysqlDefs.FIELD_TYPE_DECIMAL || fieldPks.get(i).type == (MysqlDefs.FIELD_TYPE_NEW_DECIMAL - 256)) { // field type is unsigned byte // ensure that do not use scientific notation format BigDecimal val = rs.getBigDecimal(j); curRow.add(StringUtil.encode(val != null ? val.toPlainString() : null, sc.getCharset())); } else { curRow.add(StringUtil.encode(rs.getString(j), sc.getCharset())); } } curRow.packetId = ++packetId; byteBuf = curRow.write(byteBuf, sc, false); byteBuf.flip(); byte[] row = new byte[byteBuf.limit()]; byteBuf.get(row); byteBuf.clear(); this.respHandler.rowResponse(row, this); } fieldPks.clear(); // end row eofPckg = new EOFPacket(); eofPckg.packetId = ++packetId; byteBuf = eofPckg.write(byteBuf, sc, false); byteBuf.flip(); eof = new byte[byteBuf.limit()]; byteBuf.get(eof); sc.recycle(byteBuf); this.respHandler.rowEofResponse(eof, this); } finally { if (rs != null) { try { rs.close(); } catch (SQLException e) { } } if (stmt != null) { try { stmt.close(); } catch (SQLException e) { } } } } @Override public void query(final String sql) throws UnsupportedEncodingException { if(respHandler instanceof ConnectionHeartBeatHandler) { justForHeartbeat(sql); } else { throw new UnsupportedOperationException("global seq and share join and special io.mycat.backend.mysql.nio.handler.ResponseHandler are not unsupported in jdbc driver yet "); } } private void justForHeartbeat(String sql) { Statement stmt = null; try { stmt = con.createStatement(); stmt.execute(sql); if(!isAutocommit()){ //如果在写库上，如果是事务方式的连接，需要进行手动commit con.commit(); } this.respHandler.okResponse(OkPacket.OK, this); } catch (Exception e) { String msg = e.getMessage(); ErrorPacket error = new ErrorPacket(); error.packetId = ++packetId; error.errno = ErrorCode.ER_UNKNOWN_ERROR; error.message = msg.getBytes(); this.respHandler.errorResponse(error.writeToBytes(), this); } finally { if (stmt != null) { try { stmt.close(); } catch (SQLException e) { } } } } @Override public Object getAttachment() { return this.attachement; } @Override public String getCharset() { return null; } @Override public void execute(final RouteResultsetNode node, final ServerConnection source, final boolean autocommit) throws IOException { this.sqlSelectLimit = source.getSqlSelectLimit(); Runnable runnable = new Runnable() { @Override public void run() { try { executeSQL(node, source, autocommit); } catch (IOException e) { throw new RuntimeException(e); } } }; MycatServer.getInstance().getBusinessExecutor().execute(runnable); } @Override public void recordSql(String host, String schema, String statement) { } @Override public boolean syncAndExcute() { return true; } @Override public void rollback() { try { con.rollback(); this.respHandler.okResponse(OkPacket.OK, this); } catch (SQLException e) { throw new RuntimeException(e); } } public boolean isRunning() { return this.running; } @Override public boolean isBorrowed() { return this.borrowed; } @Override public void setBorrowed(boolean borrowed) { this.borrowed = borrowed; } @Override public int getTxIsolation() { if (con != null) { try { return con.getTransactionIsolation(); } catch (SQLException e) { return 0; } } else { return -1; } } @Override public boolean isAutocommit() { if (con == null) { return true; } else { try { return con.getAutoCommit(); } catch (SQLException e) { } } return true; } @Override public boolean isTxReadonly() { if (con == null) { return true; } else { try { return con.isReadOnly(); } catch (SQLException e) { } } return true; } @Override public int getSqlSelectLimit() { return sqlSelectLimit; } @Override public long getId() { return id; } @Override public String toString() { return "JDBCConnection [id=" + id +",autocommit="+this.isAutocommit()+",pool=" + pool + ", schema=" + schema + ", dbType=" + dbType + ", oldSchema=" + oldSchema + ", packetId=" + packetId + ", txIsolation=" + txIsolation + ", running=" + running + ", borrowed=" + borrowed + ", host=" + host + ", port=" + port + ", con=" + con + ", respHandler=" + respHandler + ", attachement=" + attachement + ", headerOutputed=" + headerOutputed + ", modifiedSQLExecuted=" + modifiedSQLExecuted + ", startTime=" + startTime + ", lastTime=" + lastTime + ", isSpark=" + isSpark + ", processor=" + processor + "]"; } @Override public void discardClose(String reason) { // TODO Auto-generated method stub } @Override public void query(String sql, int charsetIndex) { try { query(sql); } catch (UnsupportedEncodingException e) { e.printStackTrace(); LOGGER.debug("UnsupportedEncodingException :"+ e.getMessage()); } } @Override public boolean checkAlive() { try { if(!con.isClosed()){ if(pool.getConfig().isCheckAlive()){ try(Statement statement = con.createStatement()){ statement.execute(pool.getHeartbeat().getHeartbeatSQL()); } } return true; }else { return false; } } catch (SQLException e) { LOGGER.error("connection is closed",e); return false; } } @Override public void disableRead() { // TODO Auto-generated method stub } @Override public void enableRead() { // TODO Auto-generated method stub } public void setFromSlaveDB(boolean fromSlaveDB) { this.fromSlaveDB = fromSlaveDB; } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/JDBCDatasource.java ================================================ package io.mycat.backend.jdbc; import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; import java.util.List; import java.util.concurrent.TimeUnit; import com.alibaba.druid.pool.DruidDataSource; import com.google.common.collect.Lists; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.heartbeat.DBHeartbeat; import io.mycat.backend.mysql.nio.handler.ResponseHandler; import io.mycat.config.model.DBHostConfig; import io.mycat.config.model.DataHostConfig; import io.mycat.net.NIOConnector; import io.mycat.net.NIOProcessor; public class JDBCDatasource extends PhysicalDatasource { private DruidDataSource dataSource; static { // 加载可能的驱动 List drivers = Lists.newArrayList( "com.mysql.jdbc.Driver", "io.mycat.backend.jdbc.mongodb.MongoDriver", "io.mycat.backend.jdbc.sequoiadb.SequoiaDriver", "oracle.jdbc.OracleDriver", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "net.sourceforge.jtds.jdbc.Driver", "org.apache.hive.jdbc.HiveDriver", "com.ibm.db2.jcc.DB2Driver", "org.postgresql.Driver"); for (String driver : drivers) { try { Class.forName(driver); } catch (ClassNotFoundException ignored) { } } } public JDBCDatasource(DBHostConfig config, DataHostConfig hostConfig, boolean isReadNode) { super(config, hostConfig, isReadNode); DBHostConfig curConfig = getConfig(); this.dataSource = new DruidDataSource(); dataSource.setUrl(curConfig.getUrl()); dataSource.setUsername(curConfig.getUser()); dataSource.setPassword(curConfig.getPassword()); dataSource.setMaxWait(TimeUnit.SECONDS.toMillis(1)); dataSource.setMaxActive(curConfig.getMaxCon()); dataSource.setMinIdle(curConfig.getMinCon()); } public Connection getDruidConnection() throws SQLException { return this.dataSource.getConnection(); } @Override public DBHeartbeat createHeartBeat() { return new JDBCHeartbeat(this); } @Override public void createNewConnection(ResponseHandler handler,String schema) throws IOException { DBHostConfig cfg = getConfig(); JDBCConnection c = new JDBCConnection(); c.setHost(cfg.getIp()); c.setPort(cfg.getPort()); c.setPool(this); c.setSchema(schema); c.setDbType(cfg.getDbType()); c.setFromSlaveDB(isReadNode()); NIOProcessor processor = (NIOProcessor) MycatServer.getInstance().nextProcessor(); c.setProcessor(processor); c.setId(NIOConnector.ID_GENERATOR.getId()); //复用mysql的Backend的ID，需要在process中存储 processor.addBackend(c); try { Connection con = getConnection(); // c.setIdleTimeout(pool.getConfig().getIdleTimeout()); c.setCon(con); // notify handler handler.connectionAcquired(c); } catch (Exception e) { handler.connectionError(e, c); } } @Override public boolean testConnection(String schema) throws IOException { boolean isConnected = false; Connection connection = null; Statement statement = null; try { DBHostConfig cfg = getConfig(); connection = DriverManager.getConnection(cfg.getUrl(), cfg.getUser(), cfg.getPassword()); statement = connection.createStatement(); if (connection != null && statement != null) { isConnected = true; } } catch (SQLException e) { e.printStackTrace(); } finally { if (statement != null) { try { statement.close(); } catch (SQLException e) {} } if (connection != null) { try { connection.close(); } catch (SQLException e) {} } } return isConnected; } Connection getConnection() throws SQLException { DBHostConfig cfg = getConfig(); Connection connection = DriverManager.getConnection(cfg.getUrl(), cfg.getUser(), cfg.getPassword()); String initSql=getHostConfig().getConnectionInitSql(); if (initSql != null && !"".equals(initSql)) { Statement statement = null; try { statement = connection.createStatement(); statement.execute(initSql); } finally { if (statement != null) { statement.close(); } } } return connection; } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/JDBCHeartbeat.java ================================================ package io.mycat.backend.jdbc; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.heartbeat.MySQLHeartbeat; import io.mycat.config.model.DataHostConfig; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; import java.text.SimpleDateFormat; import java.util.Date; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.backend.heartbeat.DBHeartbeat; import io.mycat.statistic.HeartbeatRecorder; public class JDBCHeartbeat extends DBHeartbeat{ public static final Logger LOGGER = LoggerFactory.getLogger(JDBCHeartbeat.class); private final ReentrantLock lock; private final JDBCDatasource source; private final boolean heartbeatnull; private Long lastSendTime = System.currentTimeMillis(); private Long lastReciveTime = System.currentTimeMillis(); private final int maxRetryCount; private Logger logger = LoggerFactory.getLogger(this.getClass()); public JDBCHeartbeat(JDBCDatasource source) { this.source = source; lock = new ReentrantLock(false); this.status = INIT_STATUS; this.heartbeatSQL = source.getHostConfig().getHearbeatSQL().trim(); this.heartbeatnull= heartbeatSQL.length()==0; this.maxRetryCount = source.getHostConfig().getMaxRetryCount(); } @Override public void start() { if (this.heartbeatnull){ stop(); return; } lock.lock(); try { isStop.compareAndSet(true, false); this.status = DBHeartbeat.OK_STATUS; } finally { lock.unlock(); } } @Override public void stop() { lock.lock(); try { if (isStop.compareAndSet(false, true)) { isChecking.set(false); } } finally { lock.unlock(); } } @Override public String getLastActiveTime() { long t = lastReciveTime; SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); return sdf.format(new Date(t)); } @Override public long getTimeout() { return 0; } @Override public HeartbeatRecorder getRecorder() { recorder.set(lastReciveTime - lastSendTime); return recorder; } @Override public void heartbeat() { if (isStop.get()) { return; } lastSendTime = System.currentTimeMillis(); lock.lock(); try { isChecking.set(true); try (Connection c = source.getConnection()) { try (Statement s = c.createStatement()) { s.execute(heartbeatSQL); } c.close(); } setResult(OK_STATUS); if(logger.isDebugEnabled()){ logger.debug("JDBCHeartBeat con query sql: "+heartbeatSQL); } } catch (Exception ex) { logger.error("JDBCHeartBeat error",ex); // status = ERROR_STATUS; setResult(ERROR_STATUS); } finally { lock.unlock(); this.isChecking.set(false); lastReciveTime = System.currentTimeMillis(); } } public void setResult(int result) { switch (result) { case OK_STATUS: setOk(); break; case ERROR_STATUS: setError(); break; case TIMEOUT_STATUS: setTimeout(); break; } if (this.status != OK_STATUS) { switchSourceIfNeed("heartbeat error"); } } private void setOk() { switch (status) { case DBHeartbeat.TIMEOUT_STATUS: writeStatusMsg(source.getDbPool().getHostName(), source.getName() ,DBHeartbeat.INIT_STATUS); this.status = DBHeartbeat.INIT_STATUS; this.errorCount.set(0); //前一个状态为超时 当前状态为正常状态 那就马上发送一个请求 来验证状态是否恢复为Ok heartbeat();// timeout, heart beat again break; case DBHeartbeat.OK_STATUS: this.errorCount.set(0); break; default: writeStatusMsg(source.getDbPool().getHostName(), source.getName() ,DBHeartbeat.OK_STATUS); this.status = OK_STATUS; this.errorCount.set(0);; } } //发生错误了,是否进行下一次心跳检测的策略 . 是否进行下一次心跳检测. private void nextDector( int nextStatue) { if (isStop.get()) { writeStatusMsg(source.getDbPool().getHostName(), source.getName() ,DBHeartbeat.OK_STATUS); this.status = nextStatue; } else { // should continues check error status if(errorCount.get() < maxRetryCount) { } else { writeStatusMsg(source.getDbPool().getHostName(), source.getName() ,nextStatue); this.status = nextStatue; this.errorCount.set(0); } } } private void setError() { errorCount.incrementAndGet() ; nextDector( ERROR_STATUS); } private void setTimeout() { errorCount.incrementAndGet() ; nextDector( TIMEOUT_STATUS); //status = DBHeartbeat.TIMEOUT_STATUS; } /** * switch data source */ private void switchSourceIfNeed(String reason) { int switchType = source.getHostConfig().getSwitchType(); String notSwitch = source.getHostConfig().getNotSwitch(); if (notSwitch.equals(DataHostConfig.FOVER_NOT_SWITCH_DS) || switchType == DataHostConfig.NOT_SWITCH_DS) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("not switch datasource ,for switchType is " + DataHostConfig.NOT_SWITCH_DS); return; } return; } if (LOGGER.isDebugEnabled()) { LOGGER.debug("to switchSourceIfNeed function 进行读节点转换 " ); } PhysicalDBPool pool = this.source.getDbPool(); int curDatasourceHB = pool.getSource().getHeartbeat().getStatus(); // read node can't switch ,only write node can switch if (pool.getWriteType() == PhysicalDBPool.WRITE_ONLYONE_NODE && !source.isReadNode() && curDatasourceHB != DBHeartbeat.OK_STATUS && pool.getSources().length > 1) { synchronized (pool) { // try to see if need switch datasource curDatasourceHB = pool.getSource().getHeartbeat().getStatus(); if (curDatasourceHB != DBHeartbeat.INIT_STATUS && curDatasourceHB != DBHeartbeat.OK_STATUS) { int curIndex = pool.getActivedIndex(); int nextId = pool.next(curIndex); PhysicalDatasource[] allWriteNodes = pool.getSources(); while (true) { if (nextId == curIndex) { break; } PhysicalDatasource theSource = allWriteNodes[nextId]; DBHeartbeat theSourceHB = theSource.getHeartbeat(); int theSourceHBStatus = theSourceHB.getStatus(); if (theSourceHBStatus == DBHeartbeat.OK_STATUS) { if (switchType == DataHostConfig.SYN_STATUS_SWITCH_DS) { if (Integer.valueOf(0).equals( theSourceHB.getSlaveBehindMaster())) { LOGGER.info("try to switch datasource ,slave is synchronized to master " + theSource.getConfig()); pool.switchSourceOrVoted(nextId, true, reason); break; } else { LOGGER.warn("ignored datasource ,slave is not synchronized to master , slave behind master :" + theSourceHB.getSlaveBehindMaster() + " " + theSource.getConfig()); } } else { // normal switch LOGGER.info("try to switch datasource ,not checked slave synchronize status " + theSource.getConfig()); pool.switchSourceOrVoted(nextId, true, reason); break; } } nextId = pool.next(nextId); } } } } } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/ShowVariables.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.jdbc; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.backend.BackendConnection; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.NonBlockingSession; import io.mycat.server.ServerConnection; import io.mycat.util.StringUtil; /** * @author mycat */ public final class ShowVariables { private static final Logger LOGGER = LoggerFactory.getLogger(ShowVariables.class); private static final int FIELD_COUNT = 2; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); private static final Pattern pattern = Pattern.compile("(?:like|=)\\s*'([^']*(?:\\w+)+[^']*)+'",Pattern.CASE_INSENSITIVE); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("VARIABLE_NAME", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("VALUE", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } private static List parseVariable(String sql) { List variableList=new ArrayList<>(); Matcher matcher = pattern.matcher(sql); while (matcher.find()) { variableList.add(matcher.group(1)); } return variableList; } public static void execute(ServerConnection c, String sql) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; List variableList= parseVariable(sql); for (String key : variableList) { String value= variables.get(key) ; if(value!=null) { RowDataPacket row = getRow(key, value, c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } } // write lastEof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } public static void justReturnValue(ServerConnection c, String value) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; if(value!=null) { RowDataPacket row = new RowDataPacket(1); row.add(StringUtil.encode(value, c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // write lastEof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } private static RowDataPacket getRow(String name, String value, String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(name, charset)); row.add(StringUtil.encode(value, charset)); return row; } private static final Map variables = new HashMap(); static { variables.put("character_set_client", "utf8"); variables.put("character_set_connection", "utf8"); variables.put("character_set_results", "utf8"); variables.put("character_set_server", "utf8"); variables.put("init_connect", ""); variables.put("interactive_timeout", "172800"); variables.put("lower_case_table_names", "1"); variables.put("max_allowed_packet", "16777216"); variables.put("net_buffer_length", "16384"); variables.put("net_write_timeout", "60"); variables.put("query_cache_size", "0"); variables.put("query_cache_type", "OFF"); variables.put("sql_mode", "STRICT_TRANS_TABLES"); variables.put("system_time_zone", "CST"); variables.put("time_zone", "SYSTEM"); variables.put("tx_isolation", "REPEATABLE-READ"); variables.put("wait_timeout", "172800"); } public static void execute(ServerConnection sc, String orgin, BackendConnection jdbcConnection) { execute(sc, orgin); NonBlockingSession session = sc.getSession2(); session.releaseConnectionIfSafe(jdbcConnection, LOGGER.isDebugEnabled(), false); } public static void justReturnValue(ServerConnection sc, String orgin, BackendConnection jdbcConnection) { justReturnValue(sc, orgin); NonBlockingSession session = sc.getSession2(); session.releaseConnectionIfSafe(jdbcConnection, LOGGER.isDebugEnabled(), false); } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/mongodb/DriverPropertyInfoHelper.java ================================================ package io.mycat.backend.jdbc.mongodb; import java.sql.DriverPropertyInfo; import java.util.ArrayList; public class DriverPropertyInfoHelper{ public static final String AUTO_CONNECT_RETRY = "autoConnectRetry"; public static final String CONNECTIONS_PER_HOST = "connecionsPerHost"; public static final String CONNECT_TIMEOUT = "connectTimeout"; public static final String CURSOR_FINALIZER_ENABLED = "cursorFinalizerEnabled"; public static final String MAX_AUTO_CONNECT_RETRY_TIME = "maxAutoConnectRetryTime"; public static final String READ_PREFERENCE = "readPreference"; public static final String SOCKET_TIMEOUT = "socketTimeout"; public DriverPropertyInfo[] getPropertyInfo() { ArrayList propInfos = new ArrayList(); addPropInfo( propInfos, AUTO_CONNECT_RETRY, "false", "If true, the driver will keep trying to connect to the same server in case that the socket " + "cannot be established. There is maximum amount of time to keep retrying, which is 15s by " + "default.", null); addPropInfo(propInfos, CONNECTIONS_PER_HOST, "10", "The maximum number of connections allowed per " + "host for this Mongo instance. Those connections will be kept in a pool when idle.", null); addPropInfo(propInfos, CONNECT_TIMEOUT, "10000", "The connection timeout in milliseconds. ", null); addPropInfo(propInfos, CURSOR_FINALIZER_ENABLED, "true", "Sets whether there is a a finalize " + "method created that cleans up instances of DBCursor that the client does not close.", null); addPropInfo(propInfos, MAX_AUTO_CONNECT_RETRY_TIME, "0", "The maximum amount of time in MS to spend retrying to open connection to the same server." + "Default is 0, which means to use the default 15s if autoConnectRetry is on.", null); addPropInfo(propInfos, READ_PREFERENCE, "primary", "represents preferred replica set members to which a query or command can be sent", new String[] { "primary", "primary preferred", "secondary", "secondary preferred", "nearest" }); addPropInfo(propInfos, SOCKET_TIMEOUT, "0", "The socket timeout in milliseconds It is used for " + "I/O socket read and write operations " + "Socket.setSoTimeout(int) Default is 0 and means no timeout.", null); return propInfos.toArray(new DriverPropertyInfo[propInfos.size()]); } private void addPropInfo(final ArrayList propInfos, final String propName, final String defaultVal, final String description, final String[] choices) { DriverPropertyInfo newProp = new DriverPropertyInfo(propName, defaultVal); newProp.description = description; if (choices != null) { newProp.choices = choices; } propInfos.add(newProp); } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/mongodb/MongoClientPropertyHelper.java ================================================ package io.mycat.backend.jdbc.mongodb; import com.google.common.base.Joiner; import java.util.ArrayList; import java.util.List; import java.util.Properties; import java.util.Set; /** * @author liuxinsi * @mail akalxs@gmail.com */ public class MongoClientPropertyHelper { /** * 格式化pro中的属性为{@link com.mongodb.MongoClientURI}中要求的格式。 * * @param pro 配置参数 * @return 格式化后的字符串 */ public static String formatProperties(Properties pro) { if (pro == null || pro.isEmpty()) { return null; } Set keys = pro.keySet(); List props = new ArrayList<>(keys.size()); for (Object key : keys) { Object value = pro.get(key); props.add(key + "=" + value.toString()); } return Joiner.on(";").join(props); } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/mongodb/MongoConnection.java ================================================ package io.mycat.backend.jdbc.mongodb; import java.net.UnknownHostException; import java.sql.Array; import java.sql.Blob; import java.sql.CallableStatement; import java.sql.Clob; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.NClob; import java.sql.PreparedStatement; import java.sql.SQLClientInfoException; import java.sql.SQLException; import java.sql.SQLWarning; import java.sql.SQLXML; import java.sql.Savepoint; import java.sql.Statement; import java.sql.Struct; import java.util.Map; import java.util.Properties; import java.util.concurrent.Executor; import com.mongodb.DB; import com.mongodb.MongoClient; import com.mongodb.MongoClientURI; /** * 功能详细描述 * @author sohudo[http://blog.csdn.net/wind520] * @create 2014年12月19日 下午6:50:23 * @version 0.0.1 */ public class MongoConnection implements Connection { //private String url = null; private MongoClient mc = null; private boolean isClosed = false; private String _schema; private Properties _clientInfo; public MongoConnection(MongoClientURI mcu, String url) throws UnknownHostException { // this.url = url; this._schema = mcu.getDatabase(); mc = new MongoClient(mcu); } public DB getDB() { if (this._schema!=null) { return this.mc.getDB(this._schema); } else { return null; } } @Override public T unwrap(Class iface) throws SQLException { return null; } @Override public boolean isWrapperFor(Class iface) throws SQLException { return false; } @Override public String nativeSQL(String sql) throws SQLException { return sql; } @Override public void setAutoCommit(boolean autoCommit) throws SQLException { //if (!autoCommit) // throw new RuntimeException("autoCommit has to be on"); } @Override public boolean getAutoCommit() throws SQLException { return true;//return false; } @Override public void commit() throws SQLException { } @Override public void rollback() throws SQLException { //throw new RuntimeException("can't rollback"); } @Override public void close() throws SQLException { this.mc=null; isClosed=true; } @Override public boolean isClosed() throws SQLException { return isClosed;//return false; } @Override public DatabaseMetaData getMetaData() throws SQLException { // 获取一个 DatabaseMetaData 对象，该对象包含关于此 Connection 对象所连接的数据库的元数据。 return null; } @Override public void setReadOnly(boolean readOnly) throws SQLException { //if (readOnly) // throw new RuntimeException("no read only mode"); } @Override public boolean isReadOnly() throws SQLException { // 查询此 Connection 对象是否处于只读模式。 return false; } @Override public void setCatalog(String catalog) throws SQLException { this._schema=catalog; } @Override public String getCatalog() throws SQLException { // 获取此 Connection 对象的当前目录名称 return this._schema; } @Override public void setTransactionIsolation(int level) throws SQLException { //throw new RuntimeException("no TransactionIsolation"); } @Override public int getTransactionIsolation() throws SQLException { return 0; } @Override public SQLWarning getWarnings() throws SQLException { return null;//throw new RuntimeException("should do get last error"); } @Override public void clearWarnings() throws SQLException { } @Override public Map> getTypeMap() throws SQLException { return null; } @Override public void setTypeMap(Map> map) throws SQLException { } @Override public void setHoldability(int holdability) throws SQLException { // 将使用此 Connection 对象创建的 ResultSet 对象的默认可保存性 (holdability) 更改为给定可保存性。 } @Override public int getHoldability() throws SQLException { // 获取使用此 Connection 对象创建的 ResultSet 对象的当前可保存性。 return 0; } @Override public Savepoint setSavepoint() throws SQLException { return null;//throw new RuntimeException("no savepoints"); } @Override public Savepoint setSavepoint(String name) throws SQLException { return null; } @Override public void rollback(Savepoint savepoint) throws SQLException { // throw new RuntimeException("can't rollback"); } @Override public void releaseSavepoint(Savepoint savepoint) throws SQLException { } @Override public Statement createStatement() throws SQLException { // 创建一个 Statement 对象来将 SQL 语句发送到数据库。 return createStatement(0, 0, 0); } @Override public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { // 创建一个 Statement 对象，该对象将生成具有给定类型和并发性的 ResultSet 对象。 return createStatement(resultSetType, resultSetConcurrency, 0); } @Override public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { // 创建一个 Statement 对象，该对象将生成具有给定类型、并发性和可保存性的 ResultSet 对象。 return new MongoStatement(this, resultSetType, resultSetConcurrency, resultSetHoldability); } @Override public CallableStatement prepareCall(String sql) throws SQLException { return prepareCall(sql, 0, 0, 0); } @Override public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { return prepareCall(sql, resultSetType, resultSetConcurrency, 0); } @Override public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { //return null; throw new RuntimeException("CallableStatement not supported"); } @Override public PreparedStatement prepareStatement(String sql) throws SQLException { return prepareStatement(sql, 0, 0, 0); } @Override public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { return prepareStatement(sql, resultSetType, resultSetConcurrency, 0); } @Override public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { return new MongoPreparedStatement(this, resultSetType, resultSetConcurrency, resultSetHoldability,sql); } @Override public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { return null; } @Override public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { return null; } @Override public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { return null; } @Override public Clob createClob() throws SQLException { return null; } @Override public Blob createBlob() throws SQLException { return null; } @Override public NClob createNClob() throws SQLException { return null; } @Override public SQLXML createSQLXML() throws SQLException { return null; } @Override public boolean isValid(int timeout) throws SQLException { return this.mc.getDB(_schema) != null; } @Override public void setClientInfo(String name, String value) throws SQLClientInfoException { this._clientInfo.put(name, value); } @Override public void setClientInfo(Properties properties) throws SQLClientInfoException { this._clientInfo = properties; } @Override public String getClientInfo(String name) throws SQLException { // 返回通过名称指定的客户端信息属性的值。 return (String)this._clientInfo.get(name); } @Override public Properties getClientInfo() throws SQLException { // 返回一个列表，它包含驱动程序支持的每个客户端信息属性的名称和当前值。 return this._clientInfo; } @Override public Array createArrayOf(String typeName, Object[] elements) throws SQLException { return null; } @Override public Struct createStruct(String typeName, Object[] attributes) throws SQLException { return null; } @Override public void setSchema(String schema) throws SQLException { //this._schema=schema; } @Override public String getSchema() throws SQLException { return this._schema; } @Override public void abort(Executor executor) throws SQLException { } @Override public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException { } @Override public int getNetworkTimeout() throws SQLException { return 0; } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/mongodb/MongoData.java ================================================ package io.mycat.backend.jdbc.mongodb; import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; import java.sql.Types; import java.util.HashMap; import com.mongodb.DBCursor; import com.mongodb.DBObject; import com.mongodb.BasicDBList; public class MongoData { private DBCursor cursor; private long count; private String table; private DBObject groupby; private HashMap map = new HashMap(); private boolean type=false; public MongoData(){ this.count=0; this.cursor=null; } public long getCount() { return this.count; } public void setCount(long count) { this.count=count; } public String getTable() { return this.table; } public void setTable(String table) { this.table=table; } public DBObject getGrouyBy() { return this.groupby; } public BasicDBList getGrouyBys() { if (this.groupby instanceof BasicDBList) { return (BasicDBList)this.groupby; } else { return null; } } public void setGrouyBy(DBObject gb) { this.groupby = gb; this.type = true; if (gb instanceof BasicDBList) { BasicDBList basicDBList = (BasicDBList)gb; if(!basicDBList.isEmpty()){ Object gb2 = basicDBList.get(0); if (gb2 instanceof DBObject) { for (String field : ((DBObject) gb2).keySet()) { Object val = ((DBObject) gb2).get(field); setField(field, getObjectToType(val)); } } } } } public static int getObjectToType(Object ob){ if (ob instanceof Integer) { return Types.INTEGER; } else if (ob instanceof Boolean) { return Types.BOOLEAN; } else if (ob instanceof Byte) { return Types.BIT; } else if (ob instanceof Short) { return Types.INTEGER; } else if (ob instanceof Float) { return Types.FLOAT; } else if (ob instanceof Long) { return Types.BIGINT; } else if (ob instanceof Double) { return Types.DOUBLE; } else if (ob instanceof Date) { return Types.DATE; } else if (ob instanceof Time) { return Types.TIME; } else if (ob instanceof Timestamp) { return Types.TIMESTAMP; } else if (ob instanceof String) { return Types.VARCHAR; } else { return Types.VARCHAR; } } public void setField(String field,int ftype) { map.put(field, ftype); } public HashMap getFields() { return this.map; } public boolean getType() { return this.type; } public DBCursor getCursor() { return this.cursor; } public DBCursor setCursor(DBCursor cursor) { return this.cursor=cursor; } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/mongodb/MongoDriver.java ================================================ package io.mycat.backend.jdbc.mongodb; import java.sql.Connection; import java.sql.Driver; import java.sql.DriverManager; import java.sql.DriverPropertyInfo; import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; import java.util.Properties; import java.util.logging.Logger; import org.slf4j.LoggerFactory; import com.mongodb.MongoClientURI; /** * 功能详细描述 * @author sohudo[http://blog.csdn.net/wind520] * @create 2014年12月19日 下午6:50:23 * @version 0.0.1 */ public class MongoDriver implements Driver { private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(MongoDriver.class); static final String PREFIX = "mongodb://"; private DriverPropertyInfoHelper propertyInfoHelper = new DriverPropertyInfoHelper(); static{ try{ DriverManager.registerDriver(new MongoDriver()); }catch (SQLException e){ LOGGER.error("initError",e); } } @Override public Connection connect(String url, Properties info) throws SQLException { MongoClientURI mcu = null; if ((mcu = parseURL(url, info)) == null) { return null; } MongoConnection result = null; //System.out.print(info); try{ result = new MongoConnection(mcu, url); }catch (Exception e){ throw new SQLException("Unexpected exception: " + e.getMessage(), e); } return result; } private MongoClientURI parseURL(String url, Properties defaults) { if (url == null) { return null; } if (!StringUtils.startsWithIgnoreCase(url, PREFIX)) { return null; } //删掉开头的 jdbc: //url = url.replace(URL_JDBC, ""); //替换user: if(defaults.getProperty("user")!=null && defaults.getProperty("password")!=null ){ url = url.replace(PREFIX, PREFIX+defaults.getProperty("user")+":"+defaults.getProperty("password")+"@"); } LOGGER.info("Mongodb url:"+url); String options = MongoClientPropertyHelper.formatProperties(defaults); LOGGER.debug("the options:{}",options); try { return new MongoClientURI(options == null ? url : url + "?" + options); } catch (Exception e) { LOGGER.error("parseURLError",e); return null; } } @Override public boolean acceptsURL(String url) throws SQLException { if (StringUtils.startsWithIgnoreCase(url, PREFIX)) { return true; } return false; } @Override public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException { return propertyInfoHelper.getPropertyInfo(); } @Override public int getMajorVersion() { return 1; } @Override public int getMinorVersion() { return 0; } @Override public boolean jdbcCompliant() { return true; } @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException{ return null; } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/mongodb/MongoEmbeddedObjectProcessor.java ================================================ package io.mycat.backend.jdbc.mongodb; import com.mongodb.BasicDBList; import com.mongodb.BasicDBObject; import org.bson.types.ObjectId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.lang.reflect.Array; import java.lang.reflect.Field; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; /** * 处理从MongoDB中获取的内嵌对象(Embeedded Object|SubDocument)，将MongoDB对象转换为对应的Java对象。 *
* 支持： *
    *
  • {@link ObjectId}
    • *
  • 基本类型
    • *
  • 枚举
    • *
  • 内嵌对象
    • *
  • 内嵌数组
    • *
* eg.
* public class A{
*   private ObjectId _id;
*   private String name;
*   private Integer age;
*   private B b;
*   private Address[] addresses;
*   private String[] someCode;
*   ...
* } *

* 不支持： *

    *
  • 第一层的内嵌集合类型
    • *
* eg.
* public class A{
*   private ObjectId _id;
*   private String name;
*   private Integer age;
*   private B b;
*   private List<Address> addresses;
*   private Set<String> someCode;
*   ...
* } *
* 第一次拿不到范型，所以addresses、someCode不支持，直接返回null。B对象里的则没问题。
* * @author liuxinsi * @mail akalxs@gmail.com */ public class MongoEmbeddedObjectProcessor { private static final Logger LOG = LoggerFactory.getLogger(MongoEmbeddedObjectProcessor.class); /** * 将传入的值value转换成对应的类型type返回。 * * @param columnLabel 列名 * @param value 值 * @param type 对应的类型 * @return 转换后的对象 */ public static Object valueMapper(String columnLabel, Object value, Class type) { if (value == null) { return null; } // mongodb _id field if (type.isAssignableFrom(ObjectId.class) && (value instanceof ObjectId || value instanceof String)) { return new ObjectId(value.toString()); } // enum if (type.isEnum()) { return value.toString(); } // embedded collection，内嵌集合 if ((type.isAssignableFrom(List.class) || type.isAssignableFrom(Set.class)) && value instanceof BasicDBList) { // TODO 拿不到范型，list没法转 LOG.debug("column:[{}],type:[{}]为内嵌列表,无法获取范型类,无法映射.return null.", columnLabel, type); return null; } // embedded object，内嵌对象 if (value instanceof BasicDBObject) { BasicDBObject dbObj = (BasicDBObject) value; return beanMapper(dbObj, type); } // embedded array,内嵌数组 if (type.isArray() && value instanceof BasicDBList) { BasicDBList basicDBList = (BasicDBList) value; return arrayMapper(basicDBList, type); } LOG.debug("column:[{}],type:[{}] unsupported type yet.return null", columnLabel, type); return null; } /** * 加载clazzToMapper下所有field。 * * @param clazzToMapper class * @return filed map，k=field name，v=field */ private static Map loadFields(Class clazzToMapper) { Map fieldMap = new HashMap<>(); Field[] fields = clazzToMapper.getDeclaredFields(); for (Field field : fields) { field.setAccessible(true); fieldMap.put(field.getName(), field); } return fieldMap; } /** * 获取field字段的范型类。 * * @param field field * @return null 如果没有获取到或异常。 */ private static Class getParameterizedClass(Field field) { Type type = field.getGenericType(); String parameterizedType; if (type instanceof ParameterizedType) { ParameterizedType pt = (ParameterizedType) type; if (pt.getActualTypeArguments() == null || pt.getActualTypeArguments().length == 0) { return null; } parameterizedType = pt.getActualTypeArguments()[0].toString(); } else { return null; } Class clazz; try { clazz = Class.forName(parameterizedType); } catch (ClassNotFoundException e) { LOG.warn("获取field:{}的范型异常。", field.getName(), e); return null; } return clazz; } /** * 根据字段field类型创建对应的集合类。
* 仅支持List、Set。 * * @param field field * @param size 集合初始大小 * @return 对应集合的实现类 */ private static Collection createCollection(Field field, int size) { Class fieldType = field.getType(); Collection collection = null; if (fieldType.isAssignableFrom(List.class)) { collection = new ArrayList<>(size); } else if (fieldType.isAssignableFrom(Set.class)) { collection = new HashSet<>(size); } return collection; } /** * 将mongodb的数据对象dbObj转换成对应类型clazzToMapper的对象。
* key=fieldName。 * * @param dbObj mongodb数据对象 * @param clazzToMapper 目标对象类 * @return 转换后的对象 */ private static Object beanMapper(BasicDBObject dbObj, Class clazzToMapper) { // load all field Map fieldMap = loadFields(clazzToMapper); // 将dbObj中的数据映射到beanMap中，如数据包含BasicDBObject则递归映射为对应的bean // k=dbObj中的字段名，v=dbObj中对应的值或对象 Map beanMap = new HashMap<>(); for (String s : dbObj.keySet()) { Object o = dbObj.get(s); // 嵌套对象 if (o instanceof BasicDBObject) { Field field = fieldMap.get(s); o = beanMapper((BasicDBObject) o, field.getType()); // 钳套对象列表 } else if (o instanceof BasicDBList) { Field field = fieldMap.get(s); // 获取对应的范型 Class parameterizedClazz = getParameterizedClass(field); BasicDBList basicDBs = (BasicDBList) o; Collection collection = createCollection(field, basicDBs.size()); for (Object basicDbObj : basicDBs) { // 基本类型 if (parameterizedClazz.isPrimitive()) { collection.add(basicDbObj); } else if (parameterizedClazz.getName().startsWith("java.lang")) { collection.add(basicDbObj); } else { // 对象类型 collection.add(beanMapper((BasicDBObject) basicDbObj, parameterizedClazz)); } } o = collection; } beanMap.put(s, o); } // create Object instance; try { instance = clazzToMapper.newInstance(); } catch (InstantiationException | IllegalAccessException e) { LOG.warn("实例化:[{}]对象异常.", clazzToMapper, e); return null; } // 赋值 Set fieldNames = fieldMap.keySet(); for (String fieldName : fieldNames) { if (beanMap.containsKey(fieldName)) { Field field = fieldMap.get(fieldName); Object value = beanMap.get(fieldName); try { field.set(instance, value); } catch (IllegalAccessException e) { // 应该不会报 LOG.error("为字段:[{}]设置值异常", fieldName, e); } } } return instance; } /** * 将mongodb的数据对象列表basicDBList转换成对应类型arrayClass的数组。
* 基本类型直接转换，对象类型使用 {@link #beanMapper(BasicDBObject, Class)}。 * * @param basicDBList mongodb数据对象列表 * @param arrayClass 目标数组对象类 * @return 转换后的数组对象 * @see MongoEmbeddedObjectProcessor#beanMapper(BasicDBObject, Class) */ private static Object arrayMapper(BasicDBList basicDBList, Class arrayClass) { // 具体类 Class clazzToMapper; try { clazzToMapper = Class.forName(arrayClass.getName() .replace("[L", "") .replace(";", "")); } catch (ClassNotFoundException e) { LOG.warn("实例化:[{}]对象异常.", arrayClass, e); return null; } // 创建对应的数组 Object array = Array.newInstance(clazzToMapper, basicDBList.size()); // 数组赋值 int i = 0; for (Object basicDbObj : basicDBList) { Object value; // 基本类型 if (clazzToMapper.isPrimitive()) { value = basicDbObj; } else if (clazzToMapper.getName().startsWith("java.lang")) { value = basicDbObj; } else { // 对象类型 value = beanMapper((BasicDBObject) basicDbObj, clazzToMapper); } Array.set(array, i, value); i++; } return array; } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/mongodb/MongoPreparedStatement.java ================================================ package io.mycat.backend.jdbc.mongodb; import java.io.InputStream; import java.io.Reader; import java.math.BigDecimal; import java.net.URL; import java.sql.Array; import java.sql.Blob; import java.sql.Clob; import java.sql.Date; import java.sql.NClob; import java.sql.ParameterMetaData; import java.sql.PreparedStatement; import java.sql.Ref; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.RowId; import java.sql.SQLException; import java.sql.SQLXML; import java.sql.Time; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Calendar; import java.util.List; /** * 功能详细描述 * @author sohudo[http://blog.csdn.net/wind520] * @create 2014年12月19日 下午6:50:23 * @version 0.0.1 */ public class MongoPreparedStatement extends MongoStatement implements PreparedStatement { final String _sql; final MongoSQLParser _mongosql; List _params = new ArrayList(); public MongoPreparedStatement(MongoConnection conn, int type, int concurrency, int holdability, String sql) throws MongoSQLException { super(conn, type, concurrency, holdability); this._sql = sql; this._mongosql = new MongoSQLParser(conn.getDB(), sql); } @Override public ResultSet executeQuery() throws SQLException { return null; } @Override public int executeUpdate() throws SQLException { this._mongosql.setParams(this._params); return this._mongosql.executeUpdate(); } public void setValue(int idx, Object o) { while (this._params.size() <= idx) { this._params.add(null); } this._params.set(idx, o); } @Override public void setNull(int parameterIndex, int sqlType) throws SQLException { } @Override public void setBoolean(int parameterIndex, boolean x) throws SQLException { setValue(parameterIndex, Boolean.valueOf(x)); } @Override public void setByte(int parameterIndex, byte x) throws SQLException { setValue(parameterIndex, Byte.valueOf(x)); } @Override public void setShort(int parameterIndex, short x) throws SQLException { setValue(parameterIndex, Short.valueOf(x)); } @Override public void setInt(int parameterIndex, int x) throws SQLException { setValue(parameterIndex, Integer.valueOf(x)); } @Override public void setLong(int parameterIndex, long x) throws SQLException { setValue(parameterIndex, Long.valueOf(x)); } @Override public void setFloat(int parameterIndex, float x) throws SQLException { setValue(parameterIndex, Float.valueOf(x)); } @Override public void setDouble(int parameterIndex, double x) throws SQLException { setValue(parameterIndex, Double.valueOf(x)); } @Override public void setBigDecimal(int parameterIndex, BigDecimal x) throws SQLException { setValue(parameterIndex, x); } @Override public void setString(int parameterIndex, String x) throws SQLException { setValue(parameterIndex, x); } @Override public void setBytes(int parameterIndex, byte[] x) throws SQLException { setValue(parameterIndex, x); } @Override public void setDate(int parameterIndex, Date x) throws SQLException { setValue(parameterIndex, x); } @Override public void setTime(int parameterIndex, Time x) throws SQLException { setValue(parameterIndex, x); } @Override public void setTimestamp(int parameterIndex, Timestamp x) throws SQLException { setValue(parameterIndex, x); } @Override public void setAsciiStream(int parameterIndex, InputStream x, int length) throws SQLException { } @Override public void setUnicodeStream(int parameterIndex, InputStream x, int length) throws SQLException { } @Override public void setBinaryStream(int parameterIndex, InputStream x, int length) throws SQLException { } @Override public void clearParameters() throws SQLException { } @Override public void setObject(int parameterIndex, Object x, int targetSqlType) throws SQLException { } @Override public void setObject(int parameterIndex, Object x) throws SQLException { setValue(parameterIndex,x); } @Override public boolean execute() throws SQLException { return false; } @Override public void addBatch() throws SQLException { } @Override public void setCharacterStream(int parameterIndex, Reader reader, int length) throws SQLException { } @Override public void setRef(int parameterIndex, Ref x) throws SQLException { } @Override public void setBlob(int parameterIndex, Blob x) throws SQLException { } @Override public void setClob(int parameterIndex, Clob x) throws SQLException { } @Override public void setArray(int parameterIndex, Array x) throws SQLException { } @Override public ResultSetMetaData getMetaData() throws SQLException { return null; } @Override public void setDate(int parameterIndex, Date x, Calendar cal) throws SQLException { } @Override public void setTime(int parameterIndex, Time x, Calendar cal) throws SQLException { } @Override public void setTimestamp(int parameterIndex, Timestamp x, Calendar cal) throws SQLException { } @Override public void setNull(int parameterIndex, int sqlType, String typeName) throws SQLException { } @Override public void setURL(int parameterIndex, URL x) throws SQLException { } @Override public ParameterMetaData getParameterMetaData() throws SQLException { return null; } @Override public void setRowId(int parameterIndex, RowId x) throws SQLException { } @Override public void setNString(int parameterIndex, String value) throws SQLException { } @Override public void setNCharacterStream(int parameterIndex, Reader value, long length) throws SQLException { } @Override public void setNClob(int parameterIndex, NClob value) throws SQLException { } @Override public void setClob(int parameterIndex, Reader reader, long length) throws SQLException { } @Override public void setBlob(int parameterIndex, InputStream inputStream, long length) throws SQLException { } @Override public void setNClob(int parameterIndex, Reader reader, long length) throws SQLException { } @Override public void setSQLXML(int parameterIndex, SQLXML xmlObject) throws SQLException { } @Override public void setObject(int parameterIndex, Object x, int targetSqlType, int scaleOrLength) throws SQLException { } @Override public void setAsciiStream(int parameterIndex, InputStream x, long length) throws SQLException { } @Override public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException { } @Override public void setCharacterStream(int parameterIndex, Reader reader, long length) throws SQLException { } @Override public void setAsciiStream(int parameterIndex, InputStream x) throws SQLException { } @Override public void setBinaryStream(int parameterIndex, InputStream x) throws SQLException { } @Override public void setCharacterStream(int parameterIndex, Reader reader) throws SQLException { } @Override public void setNCharacterStream(int parameterIndex, Reader value) throws SQLException { } @Override public void setClob(int parameterIndex, Reader reader) throws SQLException { } @Override public void setBlob(int parameterIndex, InputStream inputStream) throws SQLException { } @Override public void setNClob(int parameterIndex, Reader reader) throws SQLException { } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/mongodb/MongoResultSet.java ================================================ package io.mycat.backend.jdbc.mongodb; import com.mongodb.BasicDBList; import com.mongodb.DBCursor; import com.mongodb.DBObject; import java.io.InputStream; import java.io.Reader; import java.math.BigDecimal; //import java.net.MalformedURLException; import java.net.URL; import java.sql.Array; import java.sql.Blob; import java.sql.Clob; import java.sql.Date; import java.sql.NClob; import java.sql.Ref; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.RowId; import java.sql.SQLException; import java.sql.SQLWarning; import java.sql.SQLXML; import java.sql.Statement; import java.sql.Time; import java.sql.Timestamp; import java.sql.Types; import java.util.Calendar; import java.util.HashMap; //import java.util.HashMap; import java.util.Map; import java.util.Set; /** * 功能详细描述 * @author sohudo[http://blog.csdn.net/wind520] * @create 2014年12月19日 下午6:50:23 * @version 0.0.1 */ public class MongoResultSet implements ResultSet { private final DBCursor _cursor; private DBObject _cur; private int _row = 0; private boolean _closed = false; private String[] select; private int[] fieldtype; private String _schema; private String _table; //支持聚合,包括count,group by private boolean isSum=false; //是group by private boolean isGroupBy=false; private long _sum=0; private BasicDBList dblist; public MongoResultSet(MongoData mongo,String schema) throws SQLException { this._cursor = mongo.getCursor(); this._schema = schema; this._table = mongo.getTable(); this.isSum = mongo.getCount()>0; this._sum = mongo.getCount(); this.isGroupBy= mongo.getType(); if (this.isGroupBy) { dblist = mongo.getGrouyBys(); this.isSum =true; } if (this._cursor!=null) { select = (String[]) _cursor.getKeysWanted().keySet().toArray(new String[0]); if ( this._cursor.hasNext()){ _cur= _cursor.next(); if (_cur!=null) { if (select.length==0) { SetFields(_cur.keySet()); } _row=1; } } if (select.length==0){ select =new String[]{"_id"}; SetFieldType(true); } else { SetFieldType(false); } } else{ SetFields(mongo.getFields().keySet());//new String[]{"COUNT(*)"}; SetFieldType(mongo.getFields()); } } public void SetFields(Set keySet) { this.select = new String[keySet.size()]; this.select = keySet.toArray(this.select); } public void SetFieldType(boolean isid) throws SQLException { if (isid) { fieldtype= new int[Types.VARCHAR]; } else { fieldtype = new int[this.select.length]; } if (_cur!=null) { for (int i=0;i map) throws SQLException { fieldtype= new int[this.select.length]; for (int i=0;i T unwrap(Class iface) throws SQLException { return null; } @Override public boolean isWrapperFor(Class iface) throws SQLException { return false; } @Override public boolean next() throws SQLException { if ( isSum){ if (isGroupBy){ _row++; if (_row<=dblist.size()) { return true; } else { return false; } } else { if (_row==1) { return false; } else { _row++; return true; } } } else { if (! this._cursor.hasNext()) { if (_row==1) { _row++; return true; } else { return false; } } else { if (_row!=1){ this._cur = this._cursor.next(); } _row++; return true; } } } @Override public void close() throws SQLException { this._closed = true; } public String getField(int columnIndex){ return select[columnIndex-1]; } @Override public boolean wasNull() throws SQLException { return false; } @Override public String getString(int columnIndex) throws SQLException { return getString(getField(columnIndex)); } @Override public boolean getBoolean(int columnIndex) throws SQLException { return getBoolean(getField(columnIndex)); } @Override public byte getByte(int columnIndex) throws SQLException { return getByte(getField(columnIndex)); } @Override public short getShort(int columnIndex) throws SQLException { return getShort(getField(columnIndex)); } @Override public int getInt(int columnIndex) throws SQLException { return getInt(getField(columnIndex)); } @Override public long getLong(int columnIndex) throws SQLException { return getLong(getField(columnIndex)); } @Override public float getFloat(int columnIndex) throws SQLException { return getFloat(getField(columnIndex)); } @Override public double getDouble(int columnIndex) throws SQLException { return getDouble(getField(columnIndex)); } @Override public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { return getBigDecimal(getField(columnIndex),scale); } @Override public byte[] getBytes(int columnIndex) throws SQLException { return getBytes(getField(columnIndex)); } @Override public Date getDate(int columnIndex) throws SQLException { return getDate(getField(columnIndex)); } @Override public Time getTime(int columnIndex) throws SQLException { return getTime(getField(columnIndex)); } @Override public Timestamp getTimestamp(int columnIndex) throws SQLException { return getTimestamp(getField(columnIndex)); } @Override public InputStream getAsciiStream(int columnIndex) throws SQLException { return getAsciiStream(getField(columnIndex)); } @Override public InputStream getUnicodeStream(int columnIndex) throws SQLException { return getUnicodeStream(getField(columnIndex)); } @Override public InputStream getBinaryStream(int columnIndex) throws SQLException { return getBinaryStream(getField(columnIndex)); } @Override public String getString(String columnLabel) throws SQLException { Object x = getObject(columnLabel); if (x == null) { return null; } return x.toString(); } @Override public boolean getBoolean(String columnLabel) throws SQLException { //return false; Object x = getObject(columnLabel); if (x == null) { return false; } return ((Boolean)x).booleanValue(); } public Number getNumber(String columnLabel) { Number x = (Number)this._cur.get(columnLabel); if (x == null) { return Integer.valueOf(0); } return x; } @Override public byte getByte(String columnLabel) throws SQLException { return getNumber(columnLabel).byteValue(); } @Override public short getShort(String columnLabel) throws SQLException { return getNumber(columnLabel).shortValue(); } @Override public int getInt(String columnLabel) throws SQLException { return getNumber(columnLabel).intValue(); } @Override public long getLong(String columnLabel) throws SQLException { return getNumber(columnLabel).longValue(); } @Override public float getFloat(String columnLabel) throws SQLException { return getNumber(columnLabel).floatValue(); } @Override public double getDouble(String columnLabel) throws SQLException { return getNumber(columnLabel).doubleValue(); } @Override public BigDecimal getBigDecimal(String columnLabel, int scale) throws SQLException { throw new UnsupportedOperationException();//return null; } @Override public byte[] getBytes(String columnLabel) throws SQLException { return (byte[])getObject(columnLabel); } @Override public Date getDate(String columnLabel) throws SQLException { return (Date)getObject(columnLabel); } @Override public Time getTime(String columnLabel) throws SQLException { return (Time)getObject(columnLabel); } @Override public Timestamp getTimestamp(String columnLabel) throws SQLException { Object obj = getObject(columnLabel); if(obj instanceof java.util.Date){ java.util.Date d= (java.util.Date) obj; return new Timestamp(d.getTime()); } return (Timestamp)obj;//throw new UnsupportedOperationException(); } @Override public InputStream getAsciiStream(String columnLabel) throws SQLException { throw new UnsupportedOperationException();//return null; } @Override public InputStream getUnicodeStream(String columnLabel) throws SQLException { throw new UnsupportedOperationException();//return null; } @Override public InputStream getBinaryStream(String columnLabel) throws SQLException { throw new UnsupportedOperationException();//return null; } @Override public SQLWarning getWarnings() throws SQLException { return null; } @Override public void clearWarnings() throws SQLException { } @Override public String getCursorName() throws SQLException { return this._cursor.toString(); } @Override public ResultSetMetaData getMetaData() throws SQLException { return new MongoResultSetMetaData(select,fieldtype,this._schema,this._table); /* if(_cur !=null){ return new MongoResultSetMetaData(_cur.keySet(),this._schema); } else{ return new MongoResultSetMetaData(select,this._schema); } */ } @Override public Object getObject(int columnIndex) throws SQLException { if (columnIndex == 0){ if (isSum) { return getObject(getField(1)); } else { return this._cur; } } else { return getObject(getField(columnIndex)); } } @Override public Object getObject(String columnLabel) throws SQLException { if (isSum) { if (isGroupBy){ Object ob=dblist.get(_row-1); if (ob instanceof DBObject) { return ((DBObject)ob).get(columnLabel); } else { return "0"; } } else{ return this._sum; } } else { return this._cur.get(columnLabel); } } @Override public int findColumn(String columnLabel) throws SQLException { return 0; } @Override public Reader getCharacterStream(int columnIndex) throws SQLException { return getCharacterStream(getField(columnIndex)); } @Override public Reader getCharacterStream(String columnLabel) throws SQLException { return null; } @Override public BigDecimal getBigDecimal(int columnIndex) throws SQLException { return getBigDecimal(getField(columnIndex)); } @Override public BigDecimal getBigDecimal(String columnLabel) throws SQLException { return null; } @Override public boolean isBeforeFirst() throws SQLException { return false; } @Override public boolean isAfterLast() throws SQLException { return false; } @Override public boolean isFirst() throws SQLException { return false; } @Override public boolean isLast() throws SQLException { return false; } @Override public void beforeFirst() throws SQLException { } @Override public void afterLast() throws SQLException { } @Override public boolean first() throws SQLException { return false; } @Override public boolean last() throws SQLException { return false; } @Override public int getRow() throws SQLException { return this._cursor.count(); } @Override public boolean absolute(int row) throws SQLException { return false; } @Override public boolean relative(int rows) throws SQLException { // 按相对行数（或正或负）移动光标。 return false; } @Override public boolean previous() throws SQLException { return false; } @Override public void setFetchDirection(int direction) throws SQLException { if (direction == getFetchDirection()) { return; } } @Override public int getFetchDirection() throws SQLException { return 0; } @Override public void setFetchSize(int rows) throws SQLException { // 设置此 ResultSet 对象需要更多行时应该从数据库获取的行数。 } @Override public int getFetchSize() throws SQLException { return 0; } @Override public int getType() throws SQLException { return 0; } @Override public int getConcurrency() throws SQLException { return 0; } @Override public boolean rowUpdated() throws SQLException { // 获取是否已更新当前行。 return false; } @Override public boolean rowInserted() throws SQLException { // 获取当前行是否已有插入。 return false; } @Override public boolean rowDeleted() throws SQLException { //获取是否已删除某行。 return false; } @Override public void updateNull(int columnIndex) throws SQLException { } @Override public void updateBoolean(int columnIndex, boolean x) throws SQLException { } @Override public void updateByte(int columnIndex, byte x) throws SQLException { } @Override public void updateShort(int columnIndex, short x) throws SQLException { } @Override public void updateInt(int columnIndex, int x) throws SQLException { } @Override public void updateLong(int columnIndex, long x) throws SQLException { } @Override public void updateFloat(int columnIndex, float x) throws SQLException { } @Override public void updateDouble(int columnIndex, double x) throws SQLException { } @Override public void updateBigDecimal(int columnIndex, BigDecimal x) throws SQLException { } @Override public void updateString(int columnIndex, String x) throws SQLException { } @Override public void updateBytes(int columnIndex, byte[] x) throws SQLException { } @Override public void updateDate(int columnIndex, Date x) throws SQLException { } @Override public void updateTime(int columnIndex, Time x) throws SQLException { } @Override public void updateTimestamp(int columnIndex, Timestamp x) throws SQLException { } @Override public void updateAsciiStream(int columnIndex, InputStream x, int length) throws SQLException { } @Override public void updateBinaryStream(int columnIndex, InputStream x, int length) throws SQLException { } @Override public void updateCharacterStream(int columnIndex, Reader x, int length) throws SQLException { } @Override public void updateObject(int columnIndex, Object x, int scaleOrLength) throws SQLException { } @Override public void updateObject(int columnIndex, Object x) throws SQLException { } @Override public void updateNull(String columnLabel) throws SQLException { } @Override public void updateBoolean(String columnLabel, boolean x) throws SQLException { } @Override public void updateByte(String columnLabel, byte x) throws SQLException { } @Override public void updateShort(String columnLabel, short x) throws SQLException { } @Override public void updateInt(String columnLabel, int x) throws SQLException { } @Override public void updateLong(String columnLabel, long x) throws SQLException { } @Override public void updateFloat(String columnLabel, float x) throws SQLException { } @Override public void updateDouble(String columnLabel, double x) throws SQLException { } @Override public void updateBigDecimal(String columnLabel, BigDecimal x) throws SQLException { } @Override public void updateString(String columnLabel, String x) throws SQLException { } @Override public void updateBytes(String columnLabel, byte[] x) throws SQLException { } @Override public void updateDate(String columnLabel, Date x) throws SQLException { } @Override public void updateTime(String columnLabel, Time x) throws SQLException { } @Override public void updateTimestamp(String columnLabel, Timestamp x) throws SQLException { } @Override public void updateAsciiStream(String columnLabel, InputStream x, int length) throws SQLException { } @Override public void updateBinaryStream(String columnLabel, InputStream x, int length) throws SQLException { } @Override public void updateCharacterStream(String columnLabel, Reader reader, int length) throws SQLException { } @Override public void updateObject(String columnLabel, Object x, int scaleOrLength) throws SQLException { } @Override public void updateObject(String columnLabel, Object x) throws SQLException { } @Override public void insertRow() throws SQLException { } @Override public void updateRow() throws SQLException { } @Override public void deleteRow() throws SQLException { } @Override public void refreshRow() throws SQLException { } @Override public void cancelRowUpdates() throws SQLException { } @Override public void moveToInsertRow() throws SQLException { } @Override public void moveToCurrentRow() throws SQLException { } @Override public Statement getStatement() throws SQLException { return null; } @Override public Object getObject(int columnIndex, Map> map) throws SQLException { return null; } @Override public Ref getRef(int columnIndex) throws SQLException { return null; } @Override public Blob getBlob(int columnIndex) throws SQLException { return null; } @Override public Clob getClob(int columnIndex) throws SQLException { return null; } @Override public Array getArray(int columnIndex) throws SQLException { return null;//getArray(_find(i)); } @Override public Object getObject(String columnLabel, Map> map) throws SQLException { return null; } @Override public Ref getRef(String columnLabel) throws SQLException { return null; } @Override public Blob getBlob(String columnLabel) throws SQLException { return null; } @Override public Clob getClob(String columnLabel) throws SQLException { return null; } @Override public Array getArray(String columnLabel) throws SQLException { return null; } @Override public Date getDate(int columnIndex, Calendar cal) throws SQLException { return null; } @Override public Date getDate(String columnLabel, Calendar cal) throws SQLException { return null; } @Override public Time getTime(int columnIndex, Calendar cal) throws SQLException { return null; } @Override public Time getTime(String columnLabel, Calendar cal) throws SQLException { return null; } @Override public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException { return null; } @Override public Timestamp getTimestamp(String columnLabel, Calendar cal) throws SQLException { return null; } @Override public URL getURL(int columnIndex) throws SQLException { return null; } @Override public URL getURL(String columnLabel) throws SQLException { return null; } @Override public void updateRef(int columnIndex, Ref x) throws SQLException { } @Override public void updateRef(String columnLabel, Ref x) throws SQLException { } @Override public void updateBlob(int columnIndex, Blob x) throws SQLException { } @Override public void updateBlob(String columnLabel, Blob x) throws SQLException { } @Override public void updateClob(int columnIndex, Clob x) throws SQLException { } @Override public void updateClob(String columnLabel, Clob x) throws SQLException { } @Override public void updateArray(int columnIndex, Array x) throws SQLException { } @Override public void updateArray(String columnLabel, Array x) throws SQLException { } @Override public RowId getRowId(int columnIndex) throws SQLException { return null; } @Override public RowId getRowId(String columnLabel) throws SQLException { return null; } @Override public void updateRowId(int columnIndex, RowId x) throws SQLException { } @Override public void updateRowId(String columnLabel, RowId x) throws SQLException { } @Override public int getHoldability() throws SQLException { return 0; } @Override public boolean isClosed() throws SQLException { return this._closed; } @Override public void updateNString(int columnIndex, String nString) throws SQLException { } @Override public void updateNString(String columnLabel, String nString) throws SQLException { } @Override public void updateNClob(int columnIndex, NClob nClob) throws SQLException { } @Override public void updateNClob(String columnLabel, NClob nClob) throws SQLException { } @Override public NClob getNClob(int columnIndex) throws SQLException { return null; } @Override public NClob getNClob(String columnLabel) throws SQLException { return null; } @Override public SQLXML getSQLXML(int columnIndex) throws SQLException { return null; } @Override public SQLXML getSQLXML(String columnLabel) throws SQLException { return null; } @Override public void updateSQLXML(int columnIndex, SQLXML xmlObject) throws SQLException { } @Override public void updateSQLXML(String columnLabel, SQLXML xmlObject) throws SQLException { } @Override public String getNString(int columnIndex) throws SQLException { return null; } @Override public String getNString(String columnLabel) throws SQLException { return null; } @Override public Reader getNCharacterStream(int columnIndex) throws SQLException { return null; } @Override public Reader getNCharacterStream(String columnLabel) throws SQLException { return null; } @Override public void updateNCharacterStream(int columnIndex, Reader x, long length) throws SQLException { } @Override public void updateNCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { } @Override public void updateAsciiStream(int columnIndex, InputStream x, long length) throws SQLException { } @Override public void updateBinaryStream(int columnIndex, InputStream x, long length) throws SQLException { } @Override public void updateCharacterStream(int columnIndex, Reader x, long length) throws SQLException { } @Override public void updateAsciiStream(String columnLabel, InputStream x, long length) throws SQLException { } @Override public void updateBinaryStream(String columnLabel, InputStream x, long length) throws SQLException { } @Override public void updateCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { } @Override public void updateBlob(int columnIndex, InputStream inputStream, long length) throws SQLException { } @Override public void updateBlob(String columnLabel, InputStream inputStream, long length) throws SQLException { } @Override public void updateClob(int columnIndex, Reader reader, long length) throws SQLException { } @Override public void updateClob(String columnLabel, Reader reader, long length) throws SQLException { } @Override public void updateNClob(int columnIndex, Reader reader, long length) throws SQLException { } @Override public void updateNClob(String columnLabel, Reader reader, long length) throws SQLException { } @Override public void updateNCharacterStream(int columnIndex, Reader x) throws SQLException { } @Override public void updateNCharacterStream(String columnLabel, Reader reader) throws SQLException { } @Override public void updateAsciiStream(int columnIndex, InputStream x) throws SQLException { } @Override public void updateBinaryStream(int columnIndex, InputStream x) throws SQLException { } @Override public void updateCharacterStream(int columnIndex, Reader x) throws SQLException { } @Override public void updateAsciiStream(String columnLabel, InputStream x) throws SQLException { } @Override public void updateBinaryStream(String columnLabel, InputStream x) throws SQLException { } @Override public void updateCharacterStream(String columnLabel, Reader reader) throws SQLException { } @Override public void updateBlob(int columnIndex, InputStream inputStream) throws SQLException { } @Override public void updateBlob(String columnLabel, InputStream inputStream) throws SQLException { } @Override public void updateClob(int columnIndex, Reader reader) throws SQLException { } @Override public void updateClob(String columnLabel, Reader reader) throws SQLException { } @Override public void updateNClob(int columnIndex, Reader reader) throws SQLException { } @Override public void updateNClob(String columnLabel, Reader reader) throws SQLException { } @Override public T getObject(int columnIndex, Class type) throws SQLException { Object value = getObject(columnIndex); return (T) MongoEmbeddedObjectProcessor.valueMapper(getField(columnIndex), value, type); } @Override public T getObject(String columnLabel, Class type) throws SQLException { Object value = getObject(columnLabel); return (T) MongoEmbeddedObjectProcessor.valueMapper(columnLabel, value, type); } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/mongodb/MongoResultSetMetaData.java ================================================ package io.mycat.backend.jdbc.mongodb; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Types; //import java.util.Arrays; import java.util.Set; /** * 功能详细描述 * @author sohudo[http://blog.csdn.net/wind520] * @create 2014年12月19日 下午6:50:23 * @version 0.0.1 */ public class MongoResultSetMetaData implements ResultSetMetaData { private String[] keySet ; private int[] keytype ; private String _schema; private String _table; /* public MongoResultSetMetaData(Set keySet,String schema) { super(); this.keySet = new String[keySet.size()]; this.keySet = keySet.toArray(this.keySet); this._schema = schema; } */ public MongoResultSetMetaData(String[] select,int [] ftype,String schema,String table) { super(); this.keySet = select; this.keytype=ftype; this._schema = schema; this._table =table; } @Override public T unwrap(Class iface) throws SQLException { return null; } @Override public boolean isWrapperFor(Class iface) throws SQLException { return false; } @Override public int getColumnCount() throws SQLException { if (keySet==null) { return 0; } else { return keySet.length; } } @Override public boolean isAutoIncrement(int column) throws SQLException { // 是否为自动编号的字段 return false; } @Override public boolean isCaseSensitive(int column) throws SQLException { //指示列的大小写是否有关系 return true; } @Override public boolean isSearchable(int column) throws SQLException { //指示是否可以在 where 子句中使用指定的列 return true; } @Override public boolean isCurrency(int column) throws SQLException { // 指示指定的列是否是一个哈希代码值 return false; } @Override public int isNullable(int column) throws SQLException { // 指示指定列中的值是否可以为 null。 return 0; } @Override public boolean isSigned(int column) throws SQLException { // 指示指定列中的值是否带正负号 return false; } @Override public int getColumnDisplaySize(int column) throws SQLException { return 50; } @Override public String getColumnLabel(int column) throws SQLException { return keySet[column-1]; } @Override public String getColumnName(int column) throws SQLException { return keySet[column-1]; } @Override public String getSchemaName(int column) throws SQLException { return this._schema; } @Override public int getPrecision(int column) throws SQLException { //获取指定列的指定列宽 return 0; } @Override public int getScale(int column) throws SQLException { // 检索指定参数的小数点右边的位数。 return 0; } @Override public String getTableName(int column) throws SQLException { return this._table; } @Override public String getCatalogName(int column) throws SQLException { return this._schema; } @Override public int getColumnType(int column) throws SQLException { // 字段类型 return keytype[column-1];//Types.VARCHAR; } @Override public String getColumnTypeName(int column) throws SQLException { // 数据库特定的类型名称 switch (keytype[column-1]){ case Types.INTEGER: return "INTEGER"; case Types.BOOLEAN: return "BOOLEAN"; case Types.BIT: return "BITT"; case Types.FLOAT: return "FLOAT"; case Types.BIGINT: return "BIGINT"; case Types.DOUBLE: return "DOUBLE"; case Types.DATE: return "DATE"; case Types.TIME: return "TIME"; case Types.TIMESTAMP: return "TIMESTAMP"; default: return "varchar"; } } @Override public boolean isReadOnly(int column) throws SQLException { //指示指定的列是否明确不可写入 return false; } @Override public boolean isWritable(int column) throws SQLException { return false; } @Override public boolean isDefinitelyWritable(int column) throws SQLException { return false; } @Override public String getColumnClassName(int column) throws SQLException { // 如果调用方法 ResultSet.getObject 从列中获取值，则返回构造其实例的 Java 类的完全限定名称 return "Object"; } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/mongodb/MongoSQLException.java ================================================ package io.mycat.backend.jdbc.mongodb; import java.sql.SQLException; @SuppressWarnings("serial") public class MongoSQLException extends SQLException { public MongoSQLException(String msg) { super(msg); } public static class ErrorSQL extends MongoSQLException { ErrorSQL(String sql) { super(sql); } } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/mongodb/MongoSQLParser.java ================================================ package io.mycat.backend.jdbc.mongodb; import java.sql.Types; import java.util.List; import com.mongodb.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLOrderingSpecification; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; import com.alibaba.druid.sql.ast.statement.*; import com.alibaba.druid.sql.ast.expr.*; import com.alibaba.druid.sql.ast.*; /** * 功能详细描述 * @author sohudo[http://blog.csdn.net/wind520] * @create 2014年12月19日 下午6:50:23 * @version 0.0.1 */ public class MongoSQLParser { private static final Logger LOGGER = LoggerFactory.getLogger(MongoSQLParser.class); private final DB _db; // private final String _sql; private final SQLStatement statement; private List _params; private int _pos; public MongoSQLParser(DB db, String sql) throws MongoSQLException { this._db = db; // this._sql = sql; this.statement = parser(sql); } public SQLStatement parser(String s) throws MongoSQLException { s = s.trim(); try { MySqlStatementParser parser = new MySqlStatementParser(s); return parser.parseStatement(); } catch (Exception e) { LOGGER.error("MongoSQLParser.parserError", e); } throw new MongoSQLException.ErrorSQL(s); } public void setParams(List params) { this._pos = 1; this._params = params; } public MongoData query() throws MongoSQLException{ if (!(statement instanceof SQLSelectStatement)) { //return null; throw new IllegalArgumentException("not a query sql statement"); } MongoData mongo=new MongoData(); DBCursor c=null; SQLSelectStatement selectStmt = (SQLSelectStatement)statement; SQLSelectQuery sqlSelectQuery =selectStmt.getSelect().getQuery(); int icount=0; if(sqlSelectQuery instanceof MySqlSelectQueryBlock) { MySqlSelectQueryBlock mysqlSelectQuery = (MySqlSelectQueryBlock)selectStmt.getSelect().getQuery(); BasicDBObject fields = new BasicDBObject(); //显示的字段 for(SQLSelectItem item : mysqlSelectQuery.getSelectList()) { //System.out.println(item.toString()); if (!(item.getExpr() instanceof SQLAllColumnExpr)) { if (item.getExpr() instanceof SQLAggregateExpr) { SQLAggregateExpr expr =(SQLAggregateExpr)item.getExpr(); if (expr.getMethodName().equals("COUNT")) { icount=1; mongo.setField(getExprFieldName(expr), Types.BIGINT); } fields.put(getExprFieldName(expr), Integer.valueOf(1)); } else { fields.put(getFieldName(item), Integer.valueOf(1)); } } } //表名 SQLTableSource table=mysqlSelectQuery.getFrom(); DBCollection coll =this._db.getCollection(table.toString()); mongo.setTable(table.toString()); SQLExpr expr=mysqlSelectQuery.getWhere(); DBObject query = parserWhere(expr); //System.out.println(query); SQLSelectGroupByClause groupby=mysqlSelectQuery.getGroupBy(); BasicDBObject gbkey = new BasicDBObject(); if (groupby!=null) { for (SQLExpr gbexpr:groupby.getItems()){ if (gbexpr instanceof SQLIdentifierExpr) { String name=((SQLIdentifierExpr) gbexpr).getName(); gbkey.put(name, Integer.valueOf(1)); } } icount=2; } int limitoff=0; int limitnum=0; if (mysqlSelectQuery.getLimit()!=null) { limitoff=getSQLExprToInt(mysqlSelectQuery.getLimit().getOffset()); limitnum=getSQLExprToInt(mysqlSelectQuery.getLimit().getRowCount()); } if (icount==1) { mongo.setCount(coll.count(query)); } else if (icount==2){ BasicDBObject initial = new BasicDBObject(); initial.put("num", 0); String reduce="function (obj, prev) { " +" prev.num++}"; mongo.setGrouyBy(coll.group(gbkey, query, initial, reduce)); } else { if ((limitoff>0) || (limitnum>0)) { c = coll.find(query, fields).skip(limitoff).limit(limitnum); } else { c = coll.find(query, fields); } SQLOrderBy orderby=mysqlSelectQuery.getOrderBy(); if (orderby != null ){ BasicDBObject order = new BasicDBObject(); for (int i = 0; i < orderby.getItems().size(); i++) { SQLSelectOrderByItem orderitem = orderby.getItems().get(i); order.put(orderitem.getExpr().toString(), Integer.valueOf(getSQLExprToAsc(orderitem.getType()))); } c.sort(order); // System.out.println(order); } } mongo.setCursor(c); } return mongo; } public int executeUpdate() throws MongoSQLException { if (statement instanceof SQLInsertStatement) { return InsertData((SQLInsertStatement)statement); } if (statement instanceof SQLUpdateStatement) { return UpData((SQLUpdateStatement)statement); } if (statement instanceof SQLDropTableStatement) { return dropTable((SQLDropTableStatement)statement); } if (statement instanceof SQLDeleteStatement) { return DeleteDate((SQLDeleteStatement)statement); } if (statement instanceof SQLCreateTableStatement) { return 1; } return 1; } private int InsertData(SQLInsertStatement state) { if (state.getValues().getValues().size() ==0 ){ throw new RuntimeException("number of columns error"); } if (state.getValues().getValues().size() != state.getColumns().size()){ throw new RuntimeException("number of values and columns have to match"); } SQLTableSource table=state.getTableSource(); BasicDBObject[] oList = new BasicDBObject[state.getValuesList().size()]; int i = 0; for(SQLInsertStatement.ValuesClause values : state.getValuesList()){ int j = 0; BasicDBObject o = new BasicDBObject(); oList[i++] = o ; for(SQLExpr col : state.getColumns()) { o.put(getFieldName2(col), getExpValue(values.getValues().get(j++))); } } DBCollection coll =this._db.getCollection(table.toString()); WriteResult result = coll.insert(oList); return i; // 这里result.getN 总是返回0 , 所以按插入数据量返回影响行数 } private int UpData(SQLUpdateStatement state) { SQLTableSource table=state.getTableSource(); DBCollection coll =this._db.getCollection(table.toString()); SQLExpr expr=state.getWhere(); DBObject query = parserWhere(expr); BasicDBObject set = new BasicDBObject(); for(SQLUpdateSetItem col : state.getItems()){ set.put(getFieldName2(col.getColumn()), getExpValue(col.getValue())); } DBObject mod = new BasicDBObject("$set", set); WriteResult result = coll.updateMulti(query, mod); //System.out.println("changs count:"+coll.getStats().size()); return result.getN(); } private int DeleteDate(SQLDeleteStatement state) { SQLTableSource table=state.getTableSource(); DBCollection coll =this._db.getCollection(table.toString()); SQLExpr expr=state.getWhere(); if (expr==null) { throw new RuntimeException("not where of sql"); } DBObject query = parserWhere(expr); WriteResult result = coll.remove(query); return result.getN(); } private int dropTable(SQLDropTableStatement state) { for (SQLTableSource table : state.getTableSources()){ DBCollection coll =this._db.getCollection(table.toString()); coll.drop(); } return 1; } private int getSQLExprToInt(SQLExpr expr){ if (expr instanceof SQLIntegerExpr){ return ((SQLIntegerExpr)expr).getNumber().intValue(); } return 0; } private int getSQLExprToAsc(SQLOrderingSpecification ASC){ if (ASC==null ) { return 1; } if (ASC==SQLOrderingSpecification.DESC){ return -1; } else { return 1; } } public String remove(String resource,char ch) { StringBuffer buffer=new StringBuffer(); int position=0; char currentChar; while(position")) { op = "$gt"; } if (expr.getOperator().getName().equals(">=")) { op = "$gte"; } if (expr.getOperator().getName().equals("!=")) { op = "$ne"; } if (expr.getOperator().getName().equals("<>")) { op = "$ne"; } //xo.put(op, getExpValue(expr.getRight())); // o.put(exprL.toString(),xo); parserDBObject(o,exprL.toString(),op, getExpValue(expr.getRight())); } } } private void parserWhere(SQLExpr aexpr,BasicDBObject o){ if(aexpr instanceof SQLBinaryOpExpr){ SQLBinaryOpExpr expr=(SQLBinaryOpExpr)aexpr; //处理AND和OR if (expr.getOperator().getName().equals("AND")) { parserWhere(expr.getLeft(),o); parserWhere(expr.getRight(),o); } else if (expr.getOperator().getName().equals("OR")) { orWhere(expr.getLeft(),expr.getRight(),o); } else { SQLExpr exprL=expr.getLeft(); if (!(exprL instanceof SQLBinaryOpExpr)){ if (expr.getOperator().getName().equals("=")) { o.put(exprL.toString(), getExpValue(expr.getRight())); }else if(("like").equals(expr.getOperator().getName().toLowerCase())){ //处理like以及正则转换 String likeString=""; try{ likeString=("%"+String.valueOf(getExpValue(expr.getRight()))+"%") .replace("%%","") .replace("%","^"); }catch (Exception e){ throw new RuntimeException("like SQL error"); } parserDBObject(o,exprL.toString(),"$regex", likeString); } else { String op=""; if (expr.getOperator().getName().equals("<")) { op = "$lt"; } if (expr.getOperator().getName().equals("<=")) { op = "$lte"; } if (expr.getOperator().getName().equals(">")) { op = "$gt"; } if (expr.getOperator().getName().equals(">=")) { op = "$gte"; } if (expr.getOperator().getName().equals("!=")) { op = "$ne"; } if (expr.getOperator().getName().equals("<>")) { op = "$ne"; } parserDBObject(o,exprL.toString(),op, getExpValue(expr.getRight())); } } } }else if(aexpr instanceof SQLInListExpr){ //处理IN和NOT IN SQLInListExpr expr= (SQLInListExpr)aexpr; List exprList= expr.getTargetList(); BasicDBList dbObject= new BasicDBList(); for(SQLExpr e:exprList){ dbObject.add(getExpValue(e)); } String op="$in"; if(expr.isNot()){ op="$nin"; } parserDBObject(o,expr.getExpr().toString(),op, dbObject); } } private void parserWhereOLD(SQLExpr aexpr,BasicDBObject o){ if(aexpr instanceof SQLBinaryOpExpr){ SQLBinaryOpExpr expr=(SQLBinaryOpExpr)aexpr; SQLExpr exprL=expr.getLeft(); if (!(exprL instanceof SQLBinaryOpExpr)) { //opSQLExpr((SQLBinaryOpExpr)aexpr,o); if (expr.getOperator().getName().equals("=")) { o.put(exprL.toString(), getExpValue(expr.getRight())); } else { String op=""; if (expr.getOperator().getName().equals("<")) { op = "$lt"; } if (expr.getOperator().getName().equals("<=")) { op = "$lte"; } if (expr.getOperator().getName().equals(">")) { op = "$gt"; } if (expr.getOperator().getName().equals(">=")) { op = "$gte"; } if (expr.getOperator().getName().equals("!=")) { op = "$ne"; } if (expr.getOperator().getName().equals("<>")) { op = "$ne"; } parserDBObject(o,exprL.toString(),op, getExpValue(expr.getRight())); } } else { if (expr.getOperator().getName().equals("AND")) { parserWhere(exprL,o); parserWhere(expr.getRight(),o); } else if (expr.getOperator().getName().equals("OR")) { orWhere(exprL,expr.getRight(),o); } else { throw new RuntimeException("Can't identify the operation of of where"); } } } } private void orWhere(SQLExpr exprL,SQLExpr exprR ,BasicDBObject ob){ BasicDBObject xo = new BasicDBObject(); BasicDBObject yo = new BasicDBObject(); parserWhere(exprL,xo); parserWhere(exprR,yo); ob.put("$or",new Object[]{xo,yo}); } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/mongodb/MongoStatement.java ================================================ package io.mycat.backend.jdbc.mongodb; import com.mongodb.DBCursor; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLWarning; import java.sql.Statement; /** * 功能详细描述 * @author sohudo[http://blog.csdn.net/wind520] * @create 2014年12月19日 下午6:50:23 * @version 0.0.1 */ public class MongoStatement implements Statement { private MongoConnection _conn; private final int _type; private final int _concurrency; private final int _holdability; private int _fetchSize = 0; //int _maxRows = 0; private MongoResultSet _last; public MongoStatement(MongoConnection conn, int type, int concurrency, int holdability) { this._conn = conn; this._type = type; this._concurrency = concurrency; this._holdability = holdability; if (this._type != 0) { throw new UnsupportedOperationException("type not supported yet"); } if (this._concurrency != 0) { throw new UnsupportedOperationException("concurrency not supported yet"); } if (this._holdability != 0) { throw new UnsupportedOperationException("holdability not supported yet"); } } @Override public T unwrap(Class iface) throws SQLException { throw new UnsupportedOperationException();//return null; } @Override public boolean isWrapperFor(Class iface) throws SQLException { return false; } @Override public ResultSet executeQuery(String sql) throws SQLException { MongoData mongo= new MongoSQLParser(this._conn.getDB(), sql).query(); if ((this._fetchSize > 0) && (mongo.getCursor()!=null)) { //设置每次网络请求的最大记录数 mongo.getCursor().batchSize(this._fetchSize); } /* if (this._maxRows > 0) { cursor.limit(this._maxRows); } */ this._last = new MongoResultSet(mongo,this._conn.getSchema()); return this._last; } @Override public int executeUpdate(String sql) throws SQLException { // 执行更新语句 return new MongoSQLParser(this._conn.getDB(), sql).executeUpdate(); } @Override public void close() throws SQLException { this._conn = null; } @Override public int getMaxFieldSize() throws SQLException { // 获取可以为此 Statement 对象所生成 ResultSet 对象中的字符和二进制列值返回的最大字节数。 return 0;//this._fetchSize; } @Override public void setMaxFieldSize(int max) throws SQLException { //this._fetchSize=max; } @Override public int getMaxRows() throws SQLException { // 获取由此 Statement 对象生成的 ResultSet 对象可以包含的最大行数。 return 0;//this._maxRows; } @Override public void setMaxRows(int max) throws SQLException { //this._maxRows = max; } @Override public void setEscapeProcessing(boolean enable) throws SQLException { // 将转义处理设置为开或关。 } @Override public int getQueryTimeout() throws SQLException { return 0; } @Override public void setQueryTimeout(int seconds) throws SQLException { // Statement 对象执行的秒数设置，超时设置。 } @Override public void cancel() throws SQLException { } @Override public SQLWarning getWarnings() throws SQLException { return null; } @Override public void clearWarnings() throws SQLException { } @Override public void setCursorName(String name) throws SQLException { // 将 SQL 指针名称设置为给定的 String，后续 Statement 对象的 execute 方法将使用此字符串。 } @Override public boolean execute(String sql) throws SQLException { return false; } @Override public ResultSet getResultSet() throws SQLException { return this._last; } @Override public int getUpdateCount() throws SQLException { // 记录变更的数量，ResultSet 对象或没有更多结果，则返回 -1 return 0; } @Override public boolean getMoreResults() throws SQLException { // 移动到此 Statement 对象的下一个结果，如果其为 ResultSet 对象，则返回 true，并隐式关闭利用方法 getResultSet 获取的所有当前 ResultSet 对象。 return false; } @Override public void setFetchDirection(int direction) throws SQLException { // 此 Statement 对象创建的 ResultSet 对象中将按该方向处理行。 } @Override public int getFetchDirection() throws SQLException { return 0; } @Override public void setFetchSize(int rows) throws SQLException { // 获取结果集合的行数，该数是根据此 Statement 对象生成的 ResultSet 对象的默认获取大小。 this._fetchSize=rows; } @Override public int getFetchSize() throws SQLException { // 获取结果集合的行数，该数是根据此 Statement 对象生成的 ResultSet 对象的默认获取大小。 return this._fetchSize; } @Override public int getResultSetConcurrency() throws SQLException { // 对象生成的 ResultSet 对象的结果集合并发性 return 0; } @Override public int getResultSetType() throws SQLException { // 对象生成的 ResultSet 对象的结果集合类型。 return 0; } @Override public void addBatch(String sql) throws SQLException { // 新增批处理 throw new UnsupportedOperationException("batch not supported"); } @Override public void clearBatch() throws SQLException { throw new UnsupportedOperationException(); } @Override public int[] executeBatch() throws SQLException { // 将一批命令提交给数据库来执行，如果全部命令执行成功，则返回更新计数组成的数组。 throw new UnsupportedOperationException(); } @Override public Connection getConnection() throws SQLException { return this._conn; } @Override public boolean getMoreResults(int current) throws SQLException { // 将此 Statement 对象移动到下一个结果，根据给定标志指定的指令处理所有当前 ResultSet 对象；如果下一个结果为 ResultSet 对象，则返回 true。 return false; } @Override public ResultSet getGeneratedKeys() throws SQLException { // 获取由于执行此 Statement 对象而创建的所有自动生成的键。 return null; } @Override public int executeUpdate(String sql, int autoGeneratedKeys) throws SQLException { return executeUpdate(sql); } @Override public int executeUpdate(String sql, int[] columnIndexes) throws SQLException { throw new UnsupportedOperationException(); } @Override public int executeUpdate(String sql, String[] columnNames) throws SQLException { throw new UnsupportedOperationException(); } @Override public boolean execute(String sql, int autoGeneratedKeys) throws SQLException { throw new UnsupportedOperationException(); } @Override public boolean execute(String sql, int[] columnIndexes) throws SQLException { throw new UnsupportedOperationException(); } @Override public boolean execute(String sql, String[] columnNames) throws SQLException { throw new UnsupportedOperationException(); } @Override public int getResultSetHoldability() throws SQLException { return 0; } @Override public boolean isClosed() throws SQLException { return this._conn == null; } @Override public void setPoolable(boolean poolable) throws SQLException { // 请求将 Statement 池化或非池化 } @Override public boolean isPoolable() throws SQLException { return false; } @Override public void closeOnCompletion() throws SQLException { } @Override public boolean isCloseOnCompletion() throws SQLException { return false; } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/mongodb/StringUtils.java ================================================ package io.mycat.backend.jdbc.mongodb; public class StringUtils { public static boolean startsWithIgnoreCase(String searchIn, int startAt, String searchFor) { return searchIn.regionMatches(true, startAt, searchFor, 0, searchFor .length()); } public static boolean startsWithIgnoreCase(String searchIn, String searchFor) { return startsWithIgnoreCase(searchIn, 0, searchFor); } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/sequoiadb/DriverPropertyInfoHelper.java ================================================ package io.mycat.backend.jdbc.sequoiadb; import java.sql.DriverPropertyInfo; import java.util.ArrayList; public class DriverPropertyInfoHelper{ public static final String AUTO_CONNECT_RETRY = "autoConnectRetry"; public static final String CONNECTIONS_PER_HOST = "connecionsPerHost"; public static final String CONNECT_TIMEOUT = "connectTimeout"; public static final String CURSOR_FINALIZER_ENABLED = "cursorFinalizerEnabled"; public static final String MAX_AUTO_CONNECT_RETRY_TIME = "maxAutoConnectRetryTime"; public static final String READ_PREFERENCE = "readPreference"; public static final String SOCKET_TIMEOUT = "socketTimeout"; public DriverPropertyInfo[] getPropertyInfo() { ArrayList propInfos = new ArrayList(); addPropInfo( propInfos, AUTO_CONNECT_RETRY, "false", "If true, the driver will keep trying to connect to the same server in case that the socket " + "cannot be established. There is maximum amount of time to keep retrying, which is 15s by " + "default.", null); addPropInfo(propInfos, CONNECTIONS_PER_HOST, "10", "The maximum number of connections allowed per " + "host for this Mongo instance. Those connections will be kept in a pool when idle.", null); addPropInfo(propInfos, CONNECT_TIMEOUT, "10000", "The connection timeout in milliseconds. ", null); addPropInfo(propInfos, CURSOR_FINALIZER_ENABLED, "true", "Sets whether there is a a finalize " + "method created that cleans up instances of DBCursor that the client does not close.", null); addPropInfo(propInfos, MAX_AUTO_CONNECT_RETRY_TIME, "0", "The maximum amount of time in MS to spend retrying to open connection to the same server." + "Default is 0, which means to use the default 15s if autoConnectRetry is on.", null); addPropInfo(propInfos, READ_PREFERENCE, "primary", "represents preferred replica set members to which a query or command can be sent", new String[] { "primary", "primary preferred", "secondary", "secondary preferred", "nearest" }); addPropInfo(propInfos, SOCKET_TIMEOUT, "0", "The socket timeout in milliseconds It is used for " + "I/O socket read and write operations " + "Socket.setSoTimeout(int) Default is 0 and means no timeout.", null); return propInfos.toArray(new DriverPropertyInfo[propInfos.size()]); } private void addPropInfo(final ArrayList propInfos, final String propName, final String defaultVal, final String description, final String[] choices) { DriverPropertyInfo newProp = new DriverPropertyInfo(propName, defaultVal); newProp.description = description; if (choices != null) { newProp.choices = choices; } propInfos.add(newProp); } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/sequoiadb/SequoiaConnection.java ================================================ package io.mycat.backend.jdbc.sequoiadb; import java.net.UnknownHostException; import java.sql.Array; import java.sql.Blob; import java.sql.CallableStatement; import java.sql.Clob; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.NClob; import java.sql.PreparedStatement; import java.sql.SQLClientInfoException; import java.sql.SQLException; import java.sql.SQLWarning; import java.sql.SQLXML; import java.sql.Savepoint; import java.sql.Statement; import java.sql.Struct; import java.util.Map; import java.util.Properties; import java.util.concurrent.Executor; import com.sequoiadb.base.Sequoiadb; import com.sequoiadb.base.CollectionSpace; import com.sequoiadb.exception.BaseException; /** * 功能详细描述 * @author sohudo[http://blog.csdn.net/wind520] * @create 2014年12月19日 下午6:50:23 * @version 0.0.1 */ public class SequoiaConnection implements Connection { //private String url = null; private Sequoiadb mc = null; private boolean isClosed = false; private String _schema; private Properties _clientInfo; public SequoiaConnection(String url, String db) throws UnknownHostException { // this.url = url; this._schema = db; try { mc = new Sequoiadb(url, "", ""); } catch (BaseException e) { throw new IllegalArgumentException("Failed to connect to database: " + url + ", error description" + e.getErrorType()); } } public CollectionSpace getDB() { if (this._schema!=null) { if (mc.isCollectionSpaceExist(this._schema)) { return this.mc.getCollectionSpace(this._schema); } else { return this.mc.createCollectionSpace(this._schema); } } else { return null; } } @Override public T unwrap(Class iface) throws SQLException { return null; } @Override public boolean isWrapperFor(Class iface) throws SQLException { return false; } @Override public String nativeSQL(String sql) throws SQLException { return sql; } @Override public void setAutoCommit(boolean autoCommit) throws SQLException { if (!autoCommit) { throw new RuntimeException("autoCommit has to be on"); } } @Override public boolean getAutoCommit() throws SQLException { return true;//return false; } @Override public void commit() throws SQLException { } @Override public void rollback() throws SQLException { //throw new RuntimeException("can't rollback"); } @Override public void close() throws SQLException { this.mc=null; isClosed=true; } @Override public boolean isClosed() throws SQLException { return isClosed;//return false; } @Override public DatabaseMetaData getMetaData() throws SQLException { // 获取一个 DatabaseMetaData 对象，该对象包含关于此 Connection 对象所连接的数据库的元数据。 return null; } @Override public void setReadOnly(boolean readOnly) throws SQLException { //if (readOnly) // throw new RuntimeException("no read only mode"); } @Override public boolean isReadOnly() throws SQLException { // 查询此 Connection 对象是否处于只读模式。 return false; } @Override public void setCatalog(String catalog) throws SQLException { this._schema=catalog; } @Override public String getCatalog() throws SQLException { // 获取此 Connection 对象的当前目录名称 return this._schema; } @Override public void setTransactionIsolation(int level) throws SQLException { //throw new RuntimeException("no TransactionIsolation"); } @Override public int getTransactionIsolation() throws SQLException { return 0; } @Override public SQLWarning getWarnings() throws SQLException { return null;//throw new RuntimeException("should do get last error"); } @Override public void clearWarnings() throws SQLException { } @Override public Map> getTypeMap() throws SQLException { return null; } @Override public void setTypeMap(Map> map) throws SQLException { } @Override public void setHoldability(int holdability) throws SQLException { // 将使用此 Connection 对象创建的 ResultSet 对象的默认可保存性 (holdability) 更改为给定可保存性。 } @Override public int getHoldability() throws SQLException { // 获取使用此 Connection 对象创建的 ResultSet 对象的当前可保存性。 return 0; } @Override public Savepoint setSavepoint() throws SQLException { return null;//throw new RuntimeException("no savepoints"); } @Override public Savepoint setSavepoint(String name) throws SQLException { return null; } @Override public void rollback(Savepoint savepoint) throws SQLException { // throw new RuntimeException("can't rollback"); } @Override public void releaseSavepoint(Savepoint savepoint) throws SQLException { } @Override public Statement createStatement() throws SQLException { // 创建一个 Statement 对象来将 SQL 语句发送到数据库。 return createStatement(0, 0, 0); } @Override public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { // 创建一个 Statement 对象，该对象将生成具有给定类型和并发性的 ResultSet 对象。 return createStatement(resultSetType, resultSetConcurrency, 0); } @Override public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { // 创建一个 Statement 对象，该对象将生成具有给定类型、并发性和可保存性的 ResultSet 对象。 return new SequoiaStatement(this, resultSetType, resultSetConcurrency, resultSetHoldability); } @Override public CallableStatement prepareCall(String sql) throws SQLException { return prepareCall(sql, 0, 0, 0); } @Override public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { return prepareCall(sql, resultSetType, resultSetConcurrency, 0); } @Override public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { //return null; throw new RuntimeException("CallableStatement not supported"); } @Override public PreparedStatement prepareStatement(String sql) throws SQLException { return prepareStatement(sql, 0, 0, 0); } @Override public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { return prepareStatement(sql, resultSetType, resultSetConcurrency, 0); } @Override public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { return new SequoiaPreparedStatement(this, resultSetType, resultSetConcurrency, resultSetHoldability,sql); } @Override public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { return null; } @Override public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { return null; } @Override public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { return null; } @Override public Clob createClob() throws SQLException { return null; } @Override public Blob createBlob() throws SQLException { return null; } @Override public NClob createNClob() throws SQLException { return null; } @Override public SQLXML createSQLXML() throws SQLException { return null; } @Override public boolean isValid(int timeout) throws SQLException { return getDB() != null; } @Override public void setClientInfo(String name, String value) throws SQLClientInfoException { this._clientInfo.put(name, value); } @Override public void setClientInfo(Properties properties) throws SQLClientInfoException { this._clientInfo = properties; } @Override public String getClientInfo(String name) throws SQLException { // 返回通过名称指定的客户端信息属性的值。 return (String)this._clientInfo.get(name); } @Override public Properties getClientInfo() throws SQLException { // 返回一个列表，它包含驱动程序支持的每个客户端信息属性的名称和当前值。 return this._clientInfo; } @Override public Array createArrayOf(String typeName, Object[] elements) throws SQLException { return null; } @Override public Struct createStruct(String typeName, Object[] attributes) throws SQLException { return null; } @Override public void setSchema(String schema) throws SQLException { //this._schema=schema; } @Override public String getSchema() throws SQLException { return this._schema; } @Override public void abort(Executor executor) throws SQLException { } @Override public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException { } @Override public int getNetworkTimeout() throws SQLException { return 0; } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/sequoiadb/SequoiaData.java ================================================ package io.mycat.backend.jdbc.sequoiadb; import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; import java.sql.Types; import java.util.HashMap; import org.bson.BSONObject; import org.bson.BasicBSONObject; import org.bson.types.BasicBSONList; import com.sequoiadb.base.DBCursor; public class SequoiaData { private DBCursor cursor; private long count; private String table; private BSONObject groupby; private HashMap map = new HashMap(); private boolean type=false; public SequoiaData(){ this.count=0; this.cursor=null; } public long getCount() { return this.count; } public void setCount(long count) { this.count=count; } public String getTable() { return this.table; } public void setTable(String table) { this.table=table; } public BSONObject getGrouyBy() { return this.groupby; } public BasicBSONList getGrouyBys() { if (this.groupby instanceof BasicBSONList) { return (BasicBSONList)this.groupby; } else { return null; } } public void setGrouyBy(BSONObject gb) { this.groupby=gb; this.type=true; if (gb instanceof BasicBSONList) { Object gb2=((BasicBSONList)gb).get(0); if (gb2 instanceof BSONObject) { for (String field :((BSONObject)gb2).keySet()) { Object val = ((BSONObject)gb2).get(field); setField(field,getObjectToType(val)); } } } } public static int getObjectToType(Object ob){ if (ob instanceof Integer) { return Types.INTEGER; } else if (ob instanceof Boolean) { return Types.BOOLEAN; } else if (ob instanceof Byte) { return Types.BIT; } else if (ob instanceof Short) { return Types.INTEGER; } else if (ob instanceof Float) { return Types.FLOAT; } else if (ob instanceof Long) { return Types.BIGINT; } else if (ob instanceof Double) { return Types.DOUBLE; } else if (ob instanceof Date) { return Types.DATE; } else if (ob instanceof Time) { return Types.TIME; } else if (ob instanceof Timestamp) { return Types.TIMESTAMP; } else if (ob instanceof String) { return Types.VARCHAR; } else { return Types.VARCHAR; } } public void setField(String field,int ftype) { map.put(field, ftype); } public HashMap getFields() { return this.map; } public boolean getType() { return this.type; } public DBCursor getCursor() { return this.cursor; } public DBCursor setCursor(DBCursor cursor) { return this.cursor=cursor; } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/sequoiadb/SequoiaDriver.java ================================================ package io.mycat.backend.jdbc.sequoiadb; import java.sql.Connection; import java.sql.Driver; import java.sql.DriverManager; import java.sql.DriverPropertyInfo; import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; import java.util.Properties; import java.util.logging.Logger; import org.slf4j.LoggerFactory; /** * 功能详细描述 * @author sohudo[http://blog.csdn.net/wind520] * @create 2014年12月19日 下午6:50:23 * @version 0.0.1 */ public class SequoiaDriver implements Driver { private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(SequoiaDriver.class); static final String PREFIX = "sequoiadb://"; private DriverPropertyInfoHelper propertyInfoHelper = new DriverPropertyInfoHelper(); static{ try{ DriverManager.registerDriver(new SequoiaDriver()); }catch (SQLException e){ LOGGER.error("initError",e); } } @Override public Connection connect(String url, Properties info) throws SQLException { if (url == null) { return null; } if (!StringUtils.startsWithIgnoreCase(url, PREFIX)) { return null;//throw new IllegalArgumentException("uri needs to start with " + PREFIX);//return null; } String uri=url; uri = uri.substring(PREFIX.length()); String serverPart; String nsPart; String optionsPart; { int idx = uri.lastIndexOf("/"); if (idx < 0) { if (uri.contains("?")) { throw new IllegalArgumentException("URI contains options without trailing slash"); } serverPart = uri; nsPart = null; optionsPart = ""; } else { serverPart = uri.substring(0, idx); nsPart = uri.substring(idx + 1); idx = nsPart.indexOf("?"); if (idx >= 0) { optionsPart = nsPart.substring(idx + 1); nsPart = nsPart.substring(0, idx); } else { optionsPart = ""; } } } SequoiaConnection result = null; //System.out.print(info); try{ result = new SequoiaConnection(serverPart, nsPart); }catch (Exception e){ throw new SQLException("Unexpected exception: " + e.getMessage(), e); } return result; } @Override public boolean acceptsURL(String url) throws SQLException { if (!StringUtils.startsWithIgnoreCase(url, PREFIX)) { return false; } return true; } @Override public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException { return propertyInfoHelper.getPropertyInfo(); } @Override public int getMajorVersion() { return 1; } @Override public int getMinorVersion() { return 0; } @Override public boolean jdbcCompliant() { return true; } @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException{ return null; } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/sequoiadb/SequoiaPreparedStatement.java ================================================ package io.mycat.backend.jdbc.sequoiadb; import java.io.InputStream; import java.io.Reader; import java.math.BigDecimal; import java.net.URL; import java.sql.Array; import java.sql.Blob; import java.sql.Clob; import java.sql.Date; import java.sql.NClob; import java.sql.ParameterMetaData; import java.sql.PreparedStatement; import java.sql.Ref; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.RowId; import java.sql.SQLException; import java.sql.SQLXML; import java.sql.Time; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Calendar; import java.util.List; /** * 功能详细描述 * @author sohudo[http://blog.csdn.net/wind520] * @create 2014年12月19日 下午6:50:23 * @version 0.0.1 */ public class SequoiaPreparedStatement extends SequoiaStatement implements PreparedStatement { final String _sql; final SequoiaSQLParser _mongosql; List _params = new ArrayList(); public SequoiaPreparedStatement(SequoiaConnection conn, int type, int concurrency, int holdability, String sql) throws SequoiaSQLException { super(conn, type, concurrency, holdability); this._sql = sql; this._mongosql = new SequoiaSQLParser(conn.getDB(), sql); } @Override public ResultSet executeQuery() throws SQLException { return null; } @Override public int executeUpdate() throws SQLException { this._mongosql.setParams(this._params); return this._mongosql.executeUpdate(); } public void setValue(int idx, Object o) { while (this._params.size() <= idx) { this._params.add(null); } this._params.set(idx, o); } @Override public void setNull(int parameterIndex, int sqlType) throws SQLException { } @Override public void setBoolean(int parameterIndex, boolean x) throws SQLException { setValue(parameterIndex, Boolean.valueOf(x)); } @Override public void setByte(int parameterIndex, byte x) throws SQLException { setValue(parameterIndex, Byte.valueOf(x)); } @Override public void setShort(int parameterIndex, short x) throws SQLException { setValue(parameterIndex, Short.valueOf(x)); } @Override public void setInt(int parameterIndex, int x) throws SQLException { setValue(parameterIndex, Integer.valueOf(x)); } @Override public void setLong(int parameterIndex, long x) throws SQLException { setValue(parameterIndex, Long.valueOf(x)); } @Override public void setFloat(int parameterIndex, float x) throws SQLException { setValue(parameterIndex, Float.valueOf(x)); } @Override public void setDouble(int parameterIndex, double x) throws SQLException { setValue(parameterIndex, Double.valueOf(x)); } @Override public void setBigDecimal(int parameterIndex, BigDecimal x) throws SQLException { setValue(parameterIndex, x); } @Override public void setString(int parameterIndex, String x) throws SQLException { setValue(parameterIndex, x); } @Override public void setBytes(int parameterIndex, byte[] x) throws SQLException { setValue(parameterIndex, x); } @Override public void setDate(int parameterIndex, Date x) throws SQLException { setValue(parameterIndex, x); } @Override public void setTime(int parameterIndex, Time x) throws SQLException { setValue(parameterIndex, x); } @Override public void setTimestamp(int parameterIndex, Timestamp x) throws SQLException { setValue(parameterIndex, x); } @Override public void setAsciiStream(int parameterIndex, InputStream x, int length) throws SQLException { } @Override public void setUnicodeStream(int parameterIndex, InputStream x, int length) throws SQLException { } @Override public void setBinaryStream(int parameterIndex, InputStream x, int length) throws SQLException { } @Override public void clearParameters() throws SQLException { } @Override public void setObject(int parameterIndex, Object x, int targetSqlType) throws SQLException { } @Override public void setObject(int parameterIndex, Object x) throws SQLException { setValue(parameterIndex,x); } @Override public boolean execute() throws SQLException { return false; } @Override public void addBatch() throws SQLException { } @Override public void setCharacterStream(int parameterIndex, Reader reader, int length) throws SQLException { } @Override public void setRef(int parameterIndex, Ref x) throws SQLException { } @Override public void setBlob(int parameterIndex, Blob x) throws SQLException { } @Override public void setClob(int parameterIndex, Clob x) throws SQLException { } @Override public void setArray(int parameterIndex, Array x) throws SQLException { } @Override public ResultSetMetaData getMetaData() throws SQLException { return null; } @Override public void setDate(int parameterIndex, Date x, Calendar cal) throws SQLException { } @Override public void setTime(int parameterIndex, Time x, Calendar cal) throws SQLException { } @Override public void setTimestamp(int parameterIndex, Timestamp x, Calendar cal) throws SQLException { } @Override public void setNull(int parameterIndex, int sqlType, String typeName) throws SQLException { } @Override public void setURL(int parameterIndex, URL x) throws SQLException { } @Override public ParameterMetaData getParameterMetaData() throws SQLException { return null; } @Override public void setRowId(int parameterIndex, RowId x) throws SQLException { } @Override public void setNString(int parameterIndex, String value) throws SQLException { } @Override public void setNCharacterStream(int parameterIndex, Reader value, long length) throws SQLException { } @Override public void setNClob(int parameterIndex, NClob value) throws SQLException { } @Override public void setClob(int parameterIndex, Reader reader, long length) throws SQLException { } @Override public void setBlob(int parameterIndex, InputStream inputStream, long length) throws SQLException { } @Override public void setNClob(int parameterIndex, Reader reader, long length) throws SQLException { } @Override public void setSQLXML(int parameterIndex, SQLXML xmlObject) throws SQLException { } @Override public void setObject(int parameterIndex, Object x, int targetSqlType, int scaleOrLength) throws SQLException { } @Override public void setAsciiStream(int parameterIndex, InputStream x, long length) throws SQLException { } @Override public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException { } @Override public void setCharacterStream(int parameterIndex, Reader reader, long length) throws SQLException { } @Override public void setAsciiStream(int parameterIndex, InputStream x) throws SQLException { } @Override public void setBinaryStream(int parameterIndex, InputStream x) throws SQLException { } @Override public void setCharacterStream(int parameterIndex, Reader reader) throws SQLException { } @Override public void setNCharacterStream(int parameterIndex, Reader value) throws SQLException { } @Override public void setClob(int parameterIndex, Reader reader) throws SQLException { } @Override public void setBlob(int parameterIndex, InputStream inputStream) throws SQLException { } @Override public void setNClob(int parameterIndex, Reader reader) throws SQLException { } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/sequoiadb/SequoiaResultSet.java ================================================ package io.mycat.backend.jdbc.sequoiadb; import com.sequoiadb.base.DBCursor; import org.bson.BSONObject; import org.bson.types.BasicBSONList; import java.io.InputStream; import java.io.Reader; import java.math.BigDecimal; //import java.net.MalformedURLException; import java.net.URL; import java.sql.Array; import java.sql.Blob; import java.sql.Clob; import java.sql.Date; import java.sql.NClob; import java.sql.Ref; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.RowId; import java.sql.SQLException; import java.sql.SQLWarning; import java.sql.SQLXML; import java.sql.Statement; import java.sql.Time; import java.sql.Timestamp; import java.sql.Types; import java.util.Calendar; import java.util.HashMap; //import java.util.HashMap; import java.util.Map; import java.util.Set; /** * 功能详细描述 * @author sohudo[http://blog.csdn.net/wind520] * @create 2014年12月19日 下午6:50:23 * @version 0.0.1 */ public class SequoiaResultSet implements ResultSet { private final DBCursor _cursor; private BSONObject _cur;//DBObject _cur; private int _row = 0; private boolean _closed = false; private String[] select; private int[] fieldtype; private String _schema; private String _table; //支持聚合,包括count,group by private boolean isSum=false; //是group by private boolean isGroupBy=false; private long _sum=0; private BasicBSONList dblist; public SequoiaResultSet(SequoiaData mongo,String schema) throws SQLException { this._cursor = mongo.getCursor(); this._schema = schema; this._table = mongo.getTable(); this.isSum = mongo.getCount()>0; this._sum = mongo.getCount(); this.isGroupBy= mongo.getType(); if (this.isGroupBy) { dblist = mongo.getGrouyBys(); this.isSum =true; } if (this._cursor!=null) { // select = (String[]) _cursor.getKeysWanted().keySet().toArray(new String[0]); if ( this._cursor.hasNext()){ _cur= _cursor.getNext(); if (_cur!=null) { if (select==null) { SetFields(_cur.keySet()); } _row=1; } } if (select==null){ select =new String[]{"_id"}; SetFieldType(true); } else { SetFieldType(false); } } else{ SetFields(mongo.getFields().keySet());//new String[]{"COUNT(*)"}; SetFieldType(mongo.getFields()); } } public void SetFields(Set keySet) { this.select = new String[keySet.size()]; this.select = keySet.toArray(this.select); } public void SetFieldType(boolean isid) throws SQLException { if (isid) { fieldtype= new int[Types.VARCHAR]; } else { fieldtype = new int[this.select.length]; } if (_cur!=null) { for (int i=0;i map) throws SQLException { fieldtype= new int[this.select.length]; for (int i=0;i T unwrap(Class iface) throws SQLException { return null; } @Override public boolean isWrapperFor(Class iface) throws SQLException { return false; } @Override public boolean next() throws SQLException { if ( isSum){ if (isGroupBy){ _row++; if (_row<=dblist.size()) { return true; } else { return false; } } else { if (_row==1) { return false; } else { _row++; return true; } } } else { if (! this._cursor.hasNext()) { if (_row==1) { _row++; return true; } else { return false; } } else { if (_row!=1){ this._cur = this._cursor.getNext(); } _row++; return true; } } } @Override public void close() throws SQLException { this._closed = true; } public String getField(int columnIndex){ return select[columnIndex-1]; } @Override public boolean wasNull() throws SQLException { return false; } @Override public String getString(int columnIndex) throws SQLException { return getString(getField(columnIndex)); } @Override public boolean getBoolean(int columnIndex) throws SQLException { return getBoolean(getField(columnIndex)); } @Override public byte getByte(int columnIndex) throws SQLException { return getByte(getField(columnIndex)); } @Override public short getShort(int columnIndex) throws SQLException { return getShort(getField(columnIndex)); } @Override public int getInt(int columnIndex) throws SQLException { return getInt(getField(columnIndex)); } @Override public long getLong(int columnIndex) throws SQLException { return getLong(getField(columnIndex)); } @Override public float getFloat(int columnIndex) throws SQLException { return getFloat(getField(columnIndex)); } @Override public double getDouble(int columnIndex) throws SQLException { return getDouble(getField(columnIndex)); } @Override public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { return getBigDecimal(getField(columnIndex),scale); } @Override public byte[] getBytes(int columnIndex) throws SQLException { return getBytes(getField(columnIndex)); } @Override public Date getDate(int columnIndex) throws SQLException { return getDate(getField(columnIndex)); } @Override public Time getTime(int columnIndex) throws SQLException { return getTime(getField(columnIndex)); } @Override public Timestamp getTimestamp(int columnIndex) throws SQLException { return getTimestamp(getField(columnIndex)); } @Override public InputStream getAsciiStream(int columnIndex) throws SQLException { return getAsciiStream(getField(columnIndex)); } @Override public InputStream getUnicodeStream(int columnIndex) throws SQLException { return getUnicodeStream(getField(columnIndex)); } @Override public InputStream getBinaryStream(int columnIndex) throws SQLException { return getBinaryStream(getField(columnIndex)); } @Override public String getString(String columnLabel) throws SQLException { Object x = getObject(columnLabel); if (x == null) { return null; } return x.toString(); } @Override public boolean getBoolean(String columnLabel) throws SQLException { //return false; Object x = getObject(columnLabel); if (x == null) { return false; } return ((Boolean)x).booleanValue(); } public Number getNumber(String columnLabel) { Number x = (Number)this._cur.get(columnLabel); if (x == null) { return Integer.valueOf(0); } return x; } @Override public byte getByte(String columnLabel) throws SQLException { return getNumber(columnLabel).byteValue(); } @Override public short getShort(String columnLabel) throws SQLException { return getNumber(columnLabel).shortValue(); } @Override public int getInt(String columnLabel) throws SQLException { return getNumber(columnLabel).intValue(); } @Override public long getLong(String columnLabel) throws SQLException { return getNumber(columnLabel).longValue(); } @Override public float getFloat(String columnLabel) throws SQLException { return getNumber(columnLabel).floatValue(); } @Override public double getDouble(String columnLabel) throws SQLException { return getNumber(columnLabel).doubleValue(); } @Override public BigDecimal getBigDecimal(String columnLabel, int scale) throws SQLException { throw new UnsupportedOperationException();//return null; } @Override public byte[] getBytes(String columnLabel) throws SQLException { return (byte[])getObject(columnLabel); } @Override public Date getDate(String columnLabel) throws SQLException { return (Date)getObject(columnLabel); } @Override public Time getTime(String columnLabel) throws SQLException { return (Time)getObject(columnLabel); } @Override public Timestamp getTimestamp(String columnLabel) throws SQLException { return (Timestamp)getObject(columnLabel);//throw new UnsupportedOperationException(); } @Override public InputStream getAsciiStream(String columnLabel) throws SQLException { throw new UnsupportedOperationException();//return null; } @Override public InputStream getUnicodeStream(String columnLabel) throws SQLException { throw new UnsupportedOperationException();//return null; } @Override public InputStream getBinaryStream(String columnLabel) throws SQLException { throw new UnsupportedOperationException();//return null; } @Override public SQLWarning getWarnings() throws SQLException { return null; } @Override public void clearWarnings() throws SQLException { } @Override public String getCursorName() throws SQLException { return this._cursor.toString(); } @Override public ResultSetMetaData getMetaData() throws SQLException { return new SequoiaResultSetMetaData(select,fieldtype,this._schema,this._table); /* if(_cur !=null){ return new MongoResultSetMetaData(_cur.keySet(),this._schema); } else{ return new MongoResultSetMetaData(select,this._schema); } */ } @Override public Object getObject(int columnIndex) throws SQLException { if (columnIndex == 0){ if (isSum) { return getObject(getField(1)); } else { return this._cur; } } else { return getObject(getField(columnIndex)); } } @Override public Object getObject(String columnLabel) throws SQLException { if (isSum) { if (isGroupBy){ Object ob=dblist.get(_row-1); if (ob instanceof BSONObject) { return ((BSONObject)ob).get(columnLabel); } else { return "0"; } } else{ return this._sum; } } else { return this._cur.get(columnLabel); } } @Override public int findColumn(String columnLabel) throws SQLException { return 0; } @Override public Reader getCharacterStream(int columnIndex) throws SQLException { return getCharacterStream(getField(columnIndex)); } @Override public Reader getCharacterStream(String columnLabel) throws SQLException { return null; } @Override public BigDecimal getBigDecimal(int columnIndex) throws SQLException { return getBigDecimal(getField(columnIndex)); } @Override public BigDecimal getBigDecimal(String columnLabel) throws SQLException { return null; } @Override public boolean isBeforeFirst() throws SQLException { return false; } @Override public boolean isAfterLast() throws SQLException { return false; } @Override public boolean isFirst() throws SQLException { return false; } @Override public boolean isLast() throws SQLException { return false; } @Override public void beforeFirst() throws SQLException { } @Override public void afterLast() throws SQLException { } @Override public boolean first() throws SQLException { return false; } @Override public boolean last() throws SQLException { return false; } @Override public int getRow() throws SQLException { return 0;//this._cursor.count(); } @Override public boolean absolute(int row) throws SQLException { return false; } @Override public boolean relative(int rows) throws SQLException { // 按相对行数（或正或负）移动光标。 return false; } @Override public boolean previous() throws SQLException { return false; } @Override public void setFetchDirection(int direction) throws SQLException { // if (direction == getFetchDirection()) // return; } @Override public int getFetchDirection() throws SQLException { return 0; } @Override public void setFetchSize(int rows) throws SQLException { // 设置此 ResultSet 对象需要更多行时应该从数据库获取的行数。 } @Override public int getFetchSize() throws SQLException { return 0; } @Override public int getType() throws SQLException { return 0; } @Override public int getConcurrency() throws SQLException { return 0; } @Override public boolean rowUpdated() throws SQLException { // 获取是否已更新当前行。 return false; } @Override public boolean rowInserted() throws SQLException { // 获取当前行是否已有插入。 return false; } @Override public boolean rowDeleted() throws SQLException { //获取是否已删除某行。 return false; } @Override public void updateNull(int columnIndex) throws SQLException { } @Override public void updateBoolean(int columnIndex, boolean x) throws SQLException { } @Override public void updateByte(int columnIndex, byte x) throws SQLException { } @Override public void updateShort(int columnIndex, short x) throws SQLException { } @Override public void updateInt(int columnIndex, int x) throws SQLException { } @Override public void updateLong(int columnIndex, long x) throws SQLException { } @Override public void updateFloat(int columnIndex, float x) throws SQLException { } @Override public void updateDouble(int columnIndex, double x) throws SQLException { } @Override public void updateBigDecimal(int columnIndex, BigDecimal x) throws SQLException { } @Override public void updateString(int columnIndex, String x) throws SQLException { } @Override public void updateBytes(int columnIndex, byte[] x) throws SQLException { } @Override public void updateDate(int columnIndex, Date x) throws SQLException { } @Override public void updateTime(int columnIndex, Time x) throws SQLException { } @Override public void updateTimestamp(int columnIndex, Timestamp x) throws SQLException { } @Override public void updateAsciiStream(int columnIndex, InputStream x, int length) throws SQLException { } @Override public void updateBinaryStream(int columnIndex, InputStream x, int length) throws SQLException { } @Override public void updateCharacterStream(int columnIndex, Reader x, int length) throws SQLException { } @Override public void updateObject(int columnIndex, Object x, int scaleOrLength) throws SQLException { } @Override public void updateObject(int columnIndex, Object x) throws SQLException { } @Override public void updateNull(String columnLabel) throws SQLException { } @Override public void updateBoolean(String columnLabel, boolean x) throws SQLException { } @Override public void updateByte(String columnLabel, byte x) throws SQLException { } @Override public void updateShort(String columnLabel, short x) throws SQLException { } @Override public void updateInt(String columnLabel, int x) throws SQLException { } @Override public void updateLong(String columnLabel, long x) throws SQLException { } @Override public void updateFloat(String columnLabel, float x) throws SQLException { } @Override public void updateDouble(String columnLabel, double x) throws SQLException { } @Override public void updateBigDecimal(String columnLabel, BigDecimal x) throws SQLException { } @Override public void updateString(String columnLabel, String x) throws SQLException { } @Override public void updateBytes(String columnLabel, byte[] x) throws SQLException { } @Override public void updateDate(String columnLabel, Date x) throws SQLException { } @Override public void updateTime(String columnLabel, Time x) throws SQLException { } @Override public void updateTimestamp(String columnLabel, Timestamp x) throws SQLException { } @Override public void updateAsciiStream(String columnLabel, InputStream x, int length) throws SQLException { } @Override public void updateBinaryStream(String columnLabel, InputStream x, int length) throws SQLException { } @Override public void updateCharacterStream(String columnLabel, Reader reader, int length) throws SQLException { } @Override public void updateObject(String columnLabel, Object x, int scaleOrLength) throws SQLException { } @Override public void updateObject(String columnLabel, Object x) throws SQLException { } @Override public void insertRow() throws SQLException { } @Override public void updateRow() throws SQLException { } @Override public void deleteRow() throws SQLException { } @Override public void refreshRow() throws SQLException { } @Override public void cancelRowUpdates() throws SQLException { } @Override public void moveToInsertRow() throws SQLException { } @Override public void moveToCurrentRow() throws SQLException { } @Override public Statement getStatement() throws SQLException { return null; } @Override public Object getObject(int columnIndex, Map> map) throws SQLException { return null; } @Override public Ref getRef(int columnIndex) throws SQLException { return null; } @Override public Blob getBlob(int columnIndex) throws SQLException { return null; } @Override public Clob getClob(int columnIndex) throws SQLException { return null; } @Override public Array getArray(int columnIndex) throws SQLException { return null;//getArray(_find(i)); } @Override public Object getObject(String columnLabel, Map> map) throws SQLException { return null; } @Override public Ref getRef(String columnLabel) throws SQLException { return null; } @Override public Blob getBlob(String columnLabel) throws SQLException { return null; } @Override public Clob getClob(String columnLabel) throws SQLException { return null; } @Override public Array getArray(String columnLabel) throws SQLException { return null; } @Override public Date getDate(int columnIndex, Calendar cal) throws SQLException { return null; } @Override public Date getDate(String columnLabel, Calendar cal) throws SQLException { return null; } @Override public Time getTime(int columnIndex, Calendar cal) throws SQLException { return null; } @Override public Time getTime(String columnLabel, Calendar cal) throws SQLException { return null; } @Override public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException { return null; } @Override public Timestamp getTimestamp(String columnLabel, Calendar cal) throws SQLException { return null; } @Override public URL getURL(int columnIndex) throws SQLException { return null; } @Override public URL getURL(String columnLabel) throws SQLException { return null; } @Override public void updateRef(int columnIndex, Ref x) throws SQLException { } @Override public void updateRef(String columnLabel, Ref x) throws SQLException { } @Override public void updateBlob(int columnIndex, Blob x) throws SQLException { } @Override public void updateBlob(String columnLabel, Blob x) throws SQLException { } @Override public void updateClob(int columnIndex, Clob x) throws SQLException { } @Override public void updateClob(String columnLabel, Clob x) throws SQLException { } @Override public void updateArray(int columnIndex, Array x) throws SQLException { } @Override public void updateArray(String columnLabel, Array x) throws SQLException { } @Override public RowId getRowId(int columnIndex) throws SQLException { return null; } @Override public RowId getRowId(String columnLabel) throws SQLException { return null; } @Override public void updateRowId(int columnIndex, RowId x) throws SQLException { } @Override public void updateRowId(String columnLabel, RowId x) throws SQLException { } @Override public int getHoldability() throws SQLException { return 0; } @Override public boolean isClosed() throws SQLException { return this._closed; } @Override public void updateNString(int columnIndex, String nString) throws SQLException { } @Override public void updateNString(String columnLabel, String nString) throws SQLException { } @Override public void updateNClob(int columnIndex, NClob nClob) throws SQLException { } @Override public void updateNClob(String columnLabel, NClob nClob) throws SQLException { } @Override public NClob getNClob(int columnIndex) throws SQLException { return null; } @Override public NClob getNClob(String columnLabel) throws SQLException { return null; } @Override public SQLXML getSQLXML(int columnIndex) throws SQLException { return null; } @Override public SQLXML getSQLXML(String columnLabel) throws SQLException { return null; } @Override public void updateSQLXML(int columnIndex, SQLXML xmlObject) throws SQLException { } @Override public void updateSQLXML(String columnLabel, SQLXML xmlObject) throws SQLException { } @Override public String getNString(int columnIndex) throws SQLException { return null; } @Override public String getNString(String columnLabel) throws SQLException { return null; } @Override public Reader getNCharacterStream(int columnIndex) throws SQLException { return null; } @Override public Reader getNCharacterStream(String columnLabel) throws SQLException { return null; } @Override public void updateNCharacterStream(int columnIndex, Reader x, long length) throws SQLException { } @Override public void updateNCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { } @Override public void updateAsciiStream(int columnIndex, InputStream x, long length) throws SQLException { } @Override public void updateBinaryStream(int columnIndex, InputStream x, long length) throws SQLException { } @Override public void updateCharacterStream(int columnIndex, Reader x, long length) throws SQLException { } @Override public void updateAsciiStream(String columnLabel, InputStream x, long length) throws SQLException { } @Override public void updateBinaryStream(String columnLabel, InputStream x, long length) throws SQLException { } @Override public void updateCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { } @Override public void updateBlob(int columnIndex, InputStream inputStream, long length) throws SQLException { } @Override public void updateBlob(String columnLabel, InputStream inputStream, long length) throws SQLException { } @Override public void updateClob(int columnIndex, Reader reader, long length) throws SQLException { } @Override public void updateClob(String columnLabel, Reader reader, long length) throws SQLException { } @Override public void updateNClob(int columnIndex, Reader reader, long length) throws SQLException { } @Override public void updateNClob(String columnLabel, Reader reader, long length) throws SQLException { } @Override public void updateNCharacterStream(int columnIndex, Reader x) throws SQLException { } @Override public void updateNCharacterStream(String columnLabel, Reader reader) throws SQLException { } @Override public void updateAsciiStream(int columnIndex, InputStream x) throws SQLException { } @Override public void updateBinaryStream(int columnIndex, InputStream x) throws SQLException { } @Override public void updateCharacterStream(int columnIndex, Reader x) throws SQLException { } @Override public void updateAsciiStream(String columnLabel, InputStream x) throws SQLException { } @Override public void updateBinaryStream(String columnLabel, InputStream x) throws SQLException { } @Override public void updateCharacterStream(String columnLabel, Reader reader) throws SQLException { } @Override public void updateBlob(int columnIndex, InputStream inputStream) throws SQLException { } @Override public void updateBlob(String columnLabel, InputStream inputStream) throws SQLException { } @Override public void updateClob(int columnIndex, Reader reader) throws SQLException { } @Override public void updateClob(String columnLabel, Reader reader) throws SQLException { } @Override public void updateNClob(int columnIndex, Reader reader) throws SQLException { } @Override public void updateNClob(String columnLabel, Reader reader) throws SQLException { } @Override public T getObject(int columnIndex, Class type) throws SQLException { return null; } @Override public T getObject(String columnLabel, Class type) throws SQLException { return null; } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/sequoiadb/SequoiaResultSetMetaData.java ================================================ package io.mycat.backend.jdbc.sequoiadb; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Types; /** * 功能详细描述 * @author sohudo[http://blog.csdn.net/wind520] * @create 2014年12月19日 下午6:50:23 * @version 0.0.1 */ public class SequoiaResultSetMetaData implements ResultSetMetaData { private String[] keySet ; private int[] keytype ; private String _schema; private String _table; /* public MongoResultSetMetaData(Set keySet,String schema) { super(); this.keySet = new String[keySet.size()]; this.keySet = keySet.toArray(this.keySet); this._schema = schema; } */ public SequoiaResultSetMetaData(String[] select,int [] ftype,String schema,String table) { super(); this.keySet = select; this.keytype=ftype; this._schema = schema; this._table =table; } @Override public T unwrap(Class iface) throws SQLException { return null; } @Override public boolean isWrapperFor(Class iface) throws SQLException { return false; } @Override public int getColumnCount() throws SQLException { if (keySet==null) { return 0; } else { return keySet.length; } } @Override public boolean isAutoIncrement(int column) throws SQLException { // 是否为自动编号的字段 return false; } @Override public boolean isCaseSensitive(int column) throws SQLException { //指示列的大小写是否有关系 return true; } @Override public boolean isSearchable(int column) throws SQLException { //指示是否可以在 where 子句中使用指定的列 return true; } @Override public boolean isCurrency(int column) throws SQLException { // 指示指定的列是否是一个哈希代码值 return false; } @Override public int isNullable(int column) throws SQLException { // 指示指定列中的值是否可以为 null。 return 0; } @Override public boolean isSigned(int column) throws SQLException { // 指示指定列中的值是否带正负号 return false; } @Override public int getColumnDisplaySize(int column) throws SQLException { return 50; } @Override public String getColumnLabel(int column) throws SQLException { return keySet[column-1]; } @Override public String getColumnName(int column) throws SQLException { return keySet[column-1]; } @Override public String getSchemaName(int column) throws SQLException { return this._schema; } @Override public int getPrecision(int column) throws SQLException { //获取指定列的指定列宽 return 0; } @Override public int getScale(int column) throws SQLException { // 检索指定参数的小数点右边的位数。 return 0; } @Override public String getTableName(int column) throws SQLException { return this._table; } @Override public String getCatalogName(int column) throws SQLException { return this._schema; } @Override public int getColumnType(int column) throws SQLException { // 字段类型 return keytype[column-1];//Types.VARCHAR; } @Override public String getColumnTypeName(int column) throws SQLException { // 数据库特定的类型名称 switch (keytype[column-1]){ case Types.INTEGER: return "INTEGER"; case Types.BOOLEAN: return "BOOLEAN"; case Types.BIT: return "BITT"; case Types.FLOAT: return "FLOAT"; case Types.BIGINT: return "BIGINT"; case Types.DOUBLE: return "DOUBLE"; case Types.DATE: return "DATE"; case Types.TIME: return "TIME"; case Types.TIMESTAMP: return "TIMESTAMP"; default: return "varchar"; } } @Override public boolean isReadOnly(int column) throws SQLException { //指示指定的列是否明确不可写入 return false; } @Override public boolean isWritable(int column) throws SQLException { return false; } @Override public boolean isDefinitelyWritable(int column) throws SQLException { return false; } @Override public String getColumnClassName(int column) throws SQLException { // 如果调用方法 ResultSet.getObject 从列中获取值，则返回构造其实例的 Java 类的完全限定名称 return "Object"; } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/sequoiadb/SequoiaSQLException.java ================================================ package io.mycat.backend.jdbc.sequoiadb; import java.sql.SQLException; @SuppressWarnings("serial") public class SequoiaSQLException extends SQLException { public SequoiaSQLException(String msg) { super(msg); } public static class ErrorSQL extends SequoiaSQLException { ErrorSQL(String sql) { super(sql); } } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/sequoiadb/SequoiaSQLParser.java ================================================ package io.mycat.backend.jdbc.sequoiadb; import java.sql.Types; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.sequoiadb.base.CollectionSpace; import com.sequoiadb.base.DBCollection; import com.sequoiadb.base.DBCursor; import org.bson.BSONObject; import org.bson.BasicBSONObject; import org.bson.types.BasicBSONList; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLOrderingSpecification; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; import com.alibaba.druid.sql.ast.statement.*; import com.alibaba.druid.sql.ast.expr.*; import com.alibaba.druid.sql.ast.*; /** * 功能详细描述 * @author sohudo[http://blog.csdn.net/wind520] * @create 2014年12月19日 下午6:50:23 * @version 0.0.1 */ public class SequoiaSQLParser { private static final Logger LOGGER = LoggerFactory.getLogger(SequoiaSQLParser.class); private final CollectionSpace _db; // private final String _sql; private final SQLStatement statement; private List _params; private int _pos; public SequoiaSQLParser(CollectionSpace db, String sql) throws SequoiaSQLException { this._db = db; // this._sql = sql; this.statement = parser(sql); } public SQLStatement parser(String s) throws SequoiaSQLException { s = s.trim(); try { MySqlStatementParser parser = new MySqlStatementParser(s); return parser.parseStatement(); } catch (Exception e) { LOGGER.error("MongoSQLParser.parserError", e); } throw new SequoiaSQLException.ErrorSQL(s); } public void setParams(List params) { this._pos = 1; this._params = params; } public SequoiaData query() throws SequoiaSQLException{ if (!(statement instanceof SQLSelectStatement)) { //return null; throw new IllegalArgumentException("not a query sql statement"); } SequoiaData mongo=new SequoiaData(); DBCursor c=null; SQLSelectStatement selectStmt = (SQLSelectStatement)statement; SQLSelectQuery sqlSelectQuery =selectStmt.getSelect().getQuery(); int icount=0; if(sqlSelectQuery instanceof MySqlSelectQueryBlock) { MySqlSelectQueryBlock mysqlSelectQuery = (MySqlSelectQueryBlock)selectStmt.getSelect().getQuery(); BasicBSONObject fields = new BasicBSONObject(); //显示的字段 for(SQLSelectItem item : mysqlSelectQuery.getSelectList()) { //System.out.println(item.toString()); if (!(item.getExpr() instanceof SQLAllColumnExpr)) { if (item.getExpr() instanceof SQLAggregateExpr) { SQLAggregateExpr expr =(SQLAggregateExpr)item.getExpr(); if (expr.getMethodName().equals("COUNT")) { icount=1; mongo.setField(getExprFieldName(expr), Types.BIGINT); } fields.put(getExprFieldName(expr), Integer.valueOf(1)); } else { fields.put(getFieldName(item), Integer.valueOf(1)); } } } //表名 SQLTableSource table=mysqlSelectQuery.getFrom(); DBCollection coll =this._db.getCollection(table.toString()); mongo.setTable(table.toString()); SQLExpr expr=mysqlSelectQuery.getWhere(); BSONObject query = parserWhere(expr); //System.out.println(query); SQLSelectGroupByClause groupby=mysqlSelectQuery.getGroupBy(); BasicBSONObject gbkey = new BasicBSONObject(); if (groupby!=null) { for (SQLExpr gbexpr:groupby.getItems()){ if (gbexpr instanceof SQLIdentifierExpr) { String name =((SQLIdentifierExpr) gbexpr).getName(); gbkey.put(name, Integer.valueOf(1)); } } icount=2; } int limitoff=0; int limitnum=0; if (mysqlSelectQuery.getLimit()!=null) { limitoff=getSQLExprToInt(mysqlSelectQuery.getLimit().getOffset()); limitnum=getSQLExprToInt(mysqlSelectQuery.getLimit().getRowCount()); } SQLOrderBy orderby=mysqlSelectQuery.getOrderBy(); BasicBSONObject order = new BasicBSONObject(); if (orderby != null ){ for (int i = 0; i < orderby.getItems().size(); i++) { SQLSelectOrderByItem orderitem = orderby.getItems().get(i); order.put(orderitem.getExpr().toString(), Integer.valueOf(getSQLExprToAsc(orderitem.getType()))); } // c.sort(order); // System.out.println(order); } if (icount==1) { mongo.setCount(coll.getCount(query)); } else if (icount==2){ BasicBSONObject initial = new BasicBSONObject(); initial.put("num", 0); String reduce="function (obj, prev) { " +" prev.num++}"; //mongo.setGrouyBy(coll.group(gbkey, query, initial, reduce)); } else { if ((limitoff>0) || (limitnum>0)) { c = coll.query(query, fields, order,null, limitoff, limitnum);//.skip(limitoff).limit(limitnum); } else { c = coll.query(query, fields, order,null, 0, -1); } } mongo.setCursor(c); } return mongo; } public int executeUpdate() throws SequoiaSQLException { if (statement instanceof SQLInsertStatement) { return InsertData((SQLInsertStatement)statement); } if (statement instanceof SQLUpdateStatement) { return UpData((SQLUpdateStatement)statement); } if (statement instanceof SQLDropTableStatement) { return dropTable((SQLDropTableStatement)statement); } if (statement instanceof SQLDeleteStatement) { return DeleteDate((SQLDeleteStatement)statement); } if (statement instanceof SQLCreateTableStatement) { return createTable((SQLCreateTableStatement)statement); } return 1; } private int InsertData(SQLInsertStatement state) { if (state.getValues().getValues().size() ==0 ){ throw new RuntimeException("number of columns error"); } if (state.getValues().getValues().size() != state.getColumns().size()){ throw new RuntimeException("number of values and columns have to match"); } SQLTableSource table=state.getTableSource(); BSONObject o = new BasicBSONObject(); int i=0; for(SQLExpr col : state.getColumns()) { o.put(getFieldName2(col), getExpValue(state.getValues().getValues().get(i))); i++; } DBCollection coll =this._db.getCollection(table.toString()); //coll.insert(new DBObject[] { o }); coll.insert(o); return 1; } private int UpData(SQLUpdateStatement state) { SQLTableSource table=state.getTableSource(); DBCollection coll =this._db.getCollection(table.toString()); SQLExpr expr=state.getWhere(); BSONObject query = parserWhere(expr); BasicBSONObject set = new BasicBSONObject(); for(SQLUpdateSetItem col : state.getItems()){ set.put(getFieldName2(col.getColumn()), getExpValue(col.getValue())); } BSONObject mod = new BasicBSONObject("$set", set); //coll.updateMulti(query, mod); coll.update(query, mod, null); //System.out.println("changs count:"+coll.getStats().size()); return 1; } private int DeleteDate(SQLDeleteStatement state) { SQLTableSource table=state.getTableSource(); DBCollection coll =this._db.getCollection(table.toString()); SQLExpr expr=state.getWhere(); if (expr==null) { throw new RuntimeException("not where of sql"); } BSONObject query = parserWhere(expr); //coll.remove(query); coll.delete(query); return 1; } private int dropTable(SQLDropTableStatement state) { for (SQLTableSource table : state.getTableSources()){ //DBCollection coll =this._db.getCollection(table.toString()); //coll.drop(); this._db.dropCollection(table.toString()); } return 1; } private int createTable(SQLCreateTableStatement state) { //for (SQLTableSource table : state.getTableSource()){ if (!this._db.isCollectionExist(state.getTableSource().toString())) { this._db.createCollection(state.getTableSource().toString()); } return 1; } private int getSQLExprToInt(SQLExpr expr){ if (expr instanceof SQLIntegerExpr){ return ((SQLIntegerExpr)expr).getNumber().intValue(); } return 0; } private int getSQLExprToAsc(SQLOrderingSpecification ASC){ if (ASC==null ) { return 1; } if (ASC==SQLOrderingSpecification.DESC){ return -1; } else { return 1; } } public String remove(String resource,char ch) { StringBuffer buffer=new StringBuffer(); int position=0; char currentChar; while(position")) { op = "$gt"; } if (expr.getOperator().getName().equals(">=")) { op = "$gte"; } if (expr.getOperator().getName().equals("!=")) { op = "$ne"; } if (expr.getOperator().getName().equals("<>")) { op = "$ne"; } //xo.put(op, getExpValue(expr.getRight())); // o.put(exprL.toString(),xo); parserDBObject(o,exprL.toString(),op, getExpValue(expr.getRight())); } } } private void parserWhere(SQLExpr aexpr,BasicBSONObject o){ if(aexpr instanceof SQLBinaryOpExpr){ SQLBinaryOpExpr expr=(SQLBinaryOpExpr)aexpr; SQLExpr exprL=expr.getLeft(); if (!(exprL instanceof SQLBinaryOpExpr)) { //opSQLExpr((SQLBinaryOpExpr)aexpr,o); if (expr.getOperator().getName().equals("=")) { o.put(exprL.toString(), getExpValue(expr.getRight())); } else { String op=""; if (expr.getOperator().getName().equals("<")) { op = "$lt"; } if (expr.getOperator().getName().equals("<=")) { op = "$lte"; } if (expr.getOperator().getName().equals(">")) { op = "$gt"; } if (expr.getOperator().getName().equals(">=")) { op = "$gte"; } if (expr.getOperator().getName().equals("!=")) { op = "$ne"; } if (expr.getOperator().getName().equals("<>")) { op = "$ne"; } parserDBObject(o,exprL.toString(),op, getExpValue(expr.getRight())); } } else { if (expr.getOperator().getName().equals("AND")) { parserWhere(exprL,o); parserWhere(expr.getRight(),o); } else if (expr.getOperator().getName().equals("OR")) { orWhere(exprL,expr.getRight(),o); } else { throw new RuntimeException("Can't identify the operation of of where"); } } } } private void orWhere(SQLExpr exprL,SQLExpr exprR ,BasicBSONObject ob){ BasicBSONObject xo = new BasicBSONObject(); BasicBSONObject yo = new BasicBSONObject(); parserWhere(exprL,xo); parserWhere(exprR,yo); ob.put("$or",new Object[]{xo,yo}); } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/sequoiadb/SequoiaStatement.java ================================================ package io.mycat.backend.jdbc.sequoiadb; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLWarning; import java.sql.Statement; /** * 功能详细描述 * @author sohudo[http://blog.csdn.net/wind520] * @create 2014年12月19日 下午6:50:23 * @version 0.0.1 */ public class SequoiaStatement implements Statement { private SequoiaConnection _conn; private final int _type; private final int _concurrency; private final int _holdability; private int _fetchSize = 0; //int _maxRows = 0; private SequoiaResultSet _last; public SequoiaStatement(SequoiaConnection conn, int type, int concurrency, int holdability) { this._conn = conn; this._type = type; this._concurrency = concurrency; this._holdability = holdability; if (this._type != 0) { throw new UnsupportedOperationException("type not supported yet"); } if (this._concurrency != 0) { throw new UnsupportedOperationException("concurrency not supported yet"); } if (this._holdability != 0) { throw new UnsupportedOperationException("holdability not supported yet"); } } @Override public T unwrap(Class iface) throws SQLException { throw new UnsupportedOperationException();//return null; } @Override public boolean isWrapperFor(Class iface) throws SQLException { return false; } @Override public ResultSet executeQuery(String sql) throws SQLException { SequoiaData mongo= new SequoiaSQLParser(this._conn.getDB(), sql).query(); // if (this._fetchSize > 0) { // //设置每次网络请求的最大记录数 // if (mongo.getCursor()!=null) { // //mongo.getCursor().batchSize(this._fetchSize); // } // } /* if (this._maxRows > 0) { cursor.limit(this._maxRows); } */ this._last = new SequoiaResultSet(mongo,this._conn.getSchema()); return this._last; } @Override public int executeUpdate(String sql) throws SQLException { // 执行更新语句 return new SequoiaSQLParser(this._conn.getDB(), sql).executeUpdate(); } @Override public void close() throws SQLException { this._conn = null; } @Override public int getMaxFieldSize() throws SQLException { // 获取可以为此 Statement 对象所生成 ResultSet 对象中的字符和二进制列值返回的最大字节数。 return 0;//this._fetchSize; } @Override public void setMaxFieldSize(int max) throws SQLException { //this._fetchSize=max; } @Override public int getMaxRows() throws SQLException { // 获取由此 Statement 对象生成的 ResultSet 对象可以包含的最大行数。 return 0;//this._maxRows; } @Override public void setMaxRows(int max) throws SQLException { //this._maxRows = max; } @Override public void setEscapeProcessing(boolean enable) throws SQLException { // 将转义处理设置为开或关。 } @Override public int getQueryTimeout() throws SQLException { return 0; } @Override public void setQueryTimeout(int seconds) throws SQLException { // Statement 对象执行的秒数设置，超时设置。 } @Override public void cancel() throws SQLException { } @Override public SQLWarning getWarnings() throws SQLException { return null; } @Override public void clearWarnings() throws SQLException { } @Override public void setCursorName(String name) throws SQLException { // 将 SQL 指针名称设置为给定的 String，后续 Statement 对象的 execute 方法将使用此字符串。 } @Override public boolean execute(String sql) throws SQLException { int i=0;//new SequoiaSQLParser(this._conn.getDB(), sql).executeUpdate(); return i>0; } @Override public ResultSet getResultSet() throws SQLException { return this._last; } @Override public int getUpdateCount() throws SQLException { // 记录变更的数量，ResultSet 对象或没有更多结果，则返回 -1 return 0; } @Override public boolean getMoreResults() throws SQLException { // 移动到此 Statement 对象的下一个结果，如果其为 ResultSet 对象，则返回 true，并隐式关闭利用方法 getResultSet 获取的所有当前 ResultSet 对象。 return false; } @Override public void setFetchDirection(int direction) throws SQLException { // 此 Statement 对象创建的 ResultSet 对象中将按该方向处理行。 } @Override public int getFetchDirection() throws SQLException { return 0; } @Override public void setFetchSize(int rows) throws SQLException { // 获取结果集合的行数，该数是根据此 Statement 对象生成的 ResultSet 对象的默认获取大小。 this._fetchSize=rows; } @Override public int getFetchSize() throws SQLException { // 获取结果集合的行数，该数是根据此 Statement 对象生成的 ResultSet 对象的默认获取大小。 return this._fetchSize; } @Override public int getResultSetConcurrency() throws SQLException { // 对象生成的 ResultSet 对象的结果集合并发性 return 0; } @Override public int getResultSetType() throws SQLException { // 对象生成的 ResultSet 对象的结果集合类型。 return 0; } @Override public void addBatch(String sql) throws SQLException { // 新增批处理 throw new UnsupportedOperationException("batch not supported"); } @Override public void clearBatch() throws SQLException { } @Override public int[] executeBatch() throws SQLException { // 将一批命令提交给数据库来执行，如果全部命令执行成功，则返回更新计数组成的数组。 return null; } @Override public Connection getConnection() throws SQLException { return this._conn; } @Override public boolean getMoreResults(int current) throws SQLException { // 将此 Statement 对象移动到下一个结果，根据给定标志指定的指令处理所有当前 ResultSet 对象；如果下一个结果为 ResultSet 对象，则返回 true。 return false; } @Override public ResultSet getGeneratedKeys() throws SQLException { // 获取由于执行此 Statement 对象而创建的所有自动生成的键。 return null; } @Override public int executeUpdate(String sql, int autoGeneratedKeys) throws SQLException { return 0; //throw new RuntimeException("executeUpdate not done"); } @Override public int executeUpdate(String sql, int[] columnIndexes) throws SQLException { return 0; //throw new RuntimeException("executeUpdate not done"); } @Override public int executeUpdate(String sql, String[] columnNames) throws SQLException { return 0; //throw new RuntimeException("executeUpdate not done"); } @Override public boolean execute(String sql, int autoGeneratedKeys) throws SQLException { return false; } @Override public boolean execute(String sql, int[] columnIndexes) throws SQLException { return false; } @Override public boolean execute(String sql, String[] columnNames) throws SQLException { return false; } @Override public int getResultSetHoldability() throws SQLException { return 0; } @Override public boolean isClosed() throws SQLException { return this._conn == null; } @Override public void setPoolable(boolean poolable) throws SQLException { // 请求将 Statement 池化或非池化 } @Override public boolean isPoolable() throws SQLException { return false; } @Override public void closeOnCompletion() throws SQLException { } @Override public boolean isCloseOnCompletion() throws SQLException { return false; } } ================================================ FILE: src/main/java/io/mycat/backend/jdbc/sequoiadb/StringUtils.java ================================================ package io.mycat.backend.jdbc.sequoiadb; public class StringUtils { public static boolean startsWithIgnoreCase(String searchIn, int startAt, String searchFor) { return searchIn.regionMatches(true, startAt, searchFor, 0, searchFor .length()); } public static boolean startsWithIgnoreCase(String searchIn, String searchFor) { return startsWithIgnoreCase(searchIn, 0, searchFor); } } ================================================ FILE: src/main/java/io/mycat/backend/loadbalance/LeastActiveLoadBalance.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.loadbalance; import io.mycat.backend.datasource.PhysicalDatasource; import java.util.ArrayList; import java.util.concurrent.ThreadLocalRandom; public class LeastActiveLoadBalance implements LoadBalance { @Override public PhysicalDatasource doSelect(String hostName, ArrayList okSources) { boolean sameWeight = true; int length = okSources.size(); int[] leastIndexes = new int[length]; int leastActive = -1; int leastCount = 0; int[] weights = new int[length]; int totalWeight = 0; int firstWeight = 0; for (int i = 0; i < length; i++) { PhysicalDatasource okSource = okSources.get(i); int active = okSource.getActiveCount(); int weight = okSource.getConfig().getWeight(); if (weight == 0) { continue; } weights[i] = weight; if (leastActive == -1 || active < leastActive) { sameWeight = true; leastIndexes[0] = i; leastActive = active; leastCount = 1; totalWeight = weight; firstWeight = weight; } else if (active == leastActive) { leastIndexes[leastCount++] = i; totalWeight += weight; if (sameWeight && i > 0 && weight != firstWeight) { sameWeight = false; } } } if (leastCount == 0) { return okSources.get(ThreadLocalRandom.current().nextInt(okSources.size())); } if (leastCount == 1) { return okSources.get(leastIndexes[0]); } if (!sameWeight && totalWeight > 0) { int offsetWeight = ThreadLocalRandom.current().nextInt(totalWeight); for (int i = 0; i < leastCount; i++) { int leastIndex = leastIndexes[i]; offsetWeight -= weights[leastIndex]; if (offsetWeight < 0) { return okSources.get(leastIndex); } } } return okSources.get(leastIndexes[ThreadLocalRandom.current().nextInt(leastCount)]); } } ================================================ FILE: src/main/java/io/mycat/backend/loadbalance/LoadBalance.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.loadbalance; import io.mycat.backend.datasource.PhysicalDatasource; import java.util.ArrayList; public interface LoadBalance { PhysicalDatasource doSelect(String hostName, ArrayList okSources); } ================================================ FILE: src/main/java/io/mycat/backend/loadbalance/RandomLoadBalance.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.loadbalance; import io.mycat.backend.datasource.PhysicalDatasource; import java.util.ArrayList; import java.util.concurrent.ThreadLocalRandom; public class RandomLoadBalance implements LoadBalance { @Override public PhysicalDatasource doSelect(String hostName, ArrayList okSources) { boolean sameWeight = true; int length = okSources.size(); int[] weights = new int[length]; int firstWeight = okSources.get(0).getConfig().getWeight(); weights[0] = firstWeight; int totalWeight = firstWeight; for (int i = 1; i < length; i++) { int weight = okSources.get(i).getConfig().getWeight(); weights[i] = weight; totalWeight += weight; if (sameWeight && weight != firstWeight) { sameWeight = false; } } if (!sameWeight && totalWeight > 0) { int offset = ThreadLocalRandom.current().nextInt(totalWeight); for (int i = 0; i < length; i++) { offset -= weights[i]; if (offset < 0) { return okSources.get(i); } } } return okSources.get(ThreadLocalRandom.current().nextInt(length)); } } ================================================ FILE: src/main/java/io/mycat/backend/loadbalance/WeightedRoundRobinLoadBalance.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.loadbalance; import io.mycat.backend.datasource.PhysicalDatasource; import java.util.ArrayList; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicInteger; public class WeightedRoundRobinLoadBalance implements LoadBalance { private Map> weightedRoundRobinMap = new ConcurrentHashMap<>(); protected static class WeightedRoundRobin { private int weight; private AtomicInteger current = new AtomicInteger(0); public int getWeight() { return weight; } public void setWeight(int weight) { this.weight = weight; current.set(0); } public int increaseCurrent() { return current.addAndGet(weight); } public void select(int total) { current.addAndGet(-1 * total); } } @Override public PhysicalDatasource doSelect(String hostName, ArrayList okSources) { Map map = weightedRoundRobinMap.get(hostName); if (map == null) { Map newMap = new ConcurrentHashMap<>(); map = weightedRoundRobinMap.putIfAbsent(hostName, newMap); if (map == null) { map = newMap; } } int totalWeight = 0; int maxCurrent = Integer.MIN_VALUE; PhysicalDatasource selectedOkSource = null; WeightedRoundRobin selectedWeightedRoundRobin = null; for (PhysicalDatasource okSource : okSources) { String name = okSource.getName(); WeightedRoundRobin weightedRoundRobin = map.get(name); int weight = okSource.getConfig().getWeight(); if (weight <= 0) { continue; } if (weightedRoundRobin == null) { weightedRoundRobin = new WeightedRoundRobin(); weightedRoundRobin.setWeight(weight); map.putIfAbsent(name, weightedRoundRobin); } int current = weightedRoundRobin.increaseCurrent(); if (current > maxCurrent) { maxCurrent = current; selectedOkSource = okSource; selectedWeightedRoundRobin = weightedRoundRobin; } totalWeight += weight; } if (selectedOkSource == null) { return okSources.get(ThreadLocalRandom.current().nextInt(okSources.size())); } selectedWeightedRoundRobin.select(totalWeight); return selectedOkSource; } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/BindValue.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql; /** * @author mycat */ public class BindValue { public boolean isNull; /* NULL indicator */ public boolean isLongData; /* long data indicator */ public boolean isSet; /* has this parameter been set */ public long length; /* Default length of data */ public int type; /* data type */ public byte scale; /** 数据值 **/ public byte byteBinding; public short shortBinding; public int intBinding; public float floatBinding; public long longBinding; public double doubleBinding; public Object value; /* Other value to store */ public void reset() { this.isNull = false; this.isLongData = false; this.isSet = false; this.length = 0; this.type = 0; this.scale = 0; this.byteBinding = 0; this.shortBinding = 0; this.intBinding = 0; this.floatBinding = 0; this.longBinding = 0L; this.doubleBinding = 0D; this.value = null; } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/BindValueUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql; import io.mycat.config.Fields; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; import java.nio.charset.Charset; import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; /** * @author mycat */ public class BindValueUtil { public static final void read(MySQLMessage mm, BindValue bv, String charsetName) throws UnsupportedEncodingException { switch (bv.type & 0xff) { case Fields.FIELD_TYPE_BIT: bv.value = mm.readBytesWithLength(); break; case Fields.FIELD_TYPE_TINY: bv.byteBinding = mm.read(); break; case Fields.FIELD_TYPE_SHORT: bv.shortBinding = (short) mm.readUB2(); break; case Fields.FIELD_TYPE_LONG: bv.intBinding = mm.readInt(); break; case Fields.FIELD_TYPE_LONGLONG: bv.longBinding = mm.readLong(); break; case Fields.FIELD_TYPE_FLOAT: bv.floatBinding = mm.readFloat(); break; case Fields.FIELD_TYPE_DOUBLE: bv.doubleBinding = mm.readDouble(); break; case Fields.FIELD_TYPE_TIME: bv.value = mm.readTime(); break; case Fields.FIELD_TYPE_DATE: case Fields.FIELD_TYPE_DATETIME: case Fields.FIELD_TYPE_TIMESTAMP: bv.value = mm.readDate(); break; case Fields.FIELD_TYPE_VAR_STRING: case Fields.FIELD_TYPE_STRING: case Fields.FIELD_TYPE_VARCHAR: case Fields.FIELD_TYPE_DECIMAL: case Fields.FIELD_TYPE_NEW_DECIMAL: case Fields.FIELD_TYPE_BLOB: byte[] vv = mm.readBytesWithLength(); if (vv == null) { bv.isNull = true; } else { if (charsetName == null) { charsetName = StandardCharsets.UTF_8.name(); } Charset charset = Charset.forName(charsetName); try { bv.value = charset.newDecoder().onMalformedInput(CodingErrorAction.REPORT).decode(ByteBuffer.wrap(vv)); } catch (CharacterCodingException e) { bv.value = vv; } } break; default: throw new IllegalArgumentException("bindValue error,unsupported type:" + bv.type); } bv.isSet = true; } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/BufferUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql; import java.nio.ByteBuffer; /** * @author mycat */ public class BufferUtil { public static final void writeUB2(ByteBuffer buffer, int i) { buffer.put((byte) (i & 0xff)); buffer.put((byte) (i >>> 8)); } public static final void writeUB3(ByteBuffer buffer, int i) { buffer.put((byte) (i & 0xff)); buffer.put((byte) (i >>> 8)); buffer.put((byte) (i >>> 16)); } public static final void writeInt(ByteBuffer buffer, int i) { buffer.put((byte) (i & 0xff)); buffer.put((byte) (i >>> 8)); buffer.put((byte) (i >>> 16)); buffer.put((byte) (i >>> 24)); } public static final void writeFloat(ByteBuffer buffer, float f) { writeInt(buffer, Float.floatToIntBits(f)); } public static final void writeUB4(ByteBuffer buffer, long l) { buffer.put((byte) (l & 0xff)); buffer.put((byte) (l >>> 8)); buffer.put((byte) (l >>> 16)); buffer.put((byte) (l >>> 24)); } public static final void writeLong(ByteBuffer buffer, long l) { buffer.put((byte) (l & 0xff)); buffer.put((byte) (l >>> 8)); buffer.put((byte) (l >>> 16)); buffer.put((byte) (l >>> 24)); buffer.put((byte) (l >>> 32)); buffer.put((byte) (l >>> 40)); buffer.put((byte) (l >>> 48)); buffer.put((byte) (l >>> 56)); } public static final void writeDouble(ByteBuffer buffer, double d) { writeLong(buffer, Double.doubleToLongBits(d)); } public static final void writeLength(ByteBuffer buffer, long l) { if(l < 0) { buffer.put((byte) 254); writeLong(buffer, l); } else if (l < 251) { buffer.put((byte) l); } else if (l < 0x10000L) { buffer.put((byte) 252); writeUB2(buffer, (int) l); } else if (l < 0x1000000L) { buffer.put((byte) 253); writeUB3(buffer, (int) l); } else { buffer.put((byte) 254); writeLong(buffer, l); } } public static final void writeWithNull(ByteBuffer buffer, byte[] src) { buffer.put(src); buffer.put((byte) 0); } public static final void writeWithLength(ByteBuffer buffer, byte[] src) { int length = src.length; if (length < 251) { buffer.put((byte) length); } else if (length < 0x10000L) { buffer.put((byte) 252); writeUB2(buffer, length); } else if (length < 0x1000000L) { buffer.put((byte) 253); writeUB3(buffer, length); } else { buffer.put((byte) 254); writeLong(buffer, length); } buffer.put(src); } public static final void writeWithLength(ByteBuffer buffer, byte[] src, byte nullValue) { if (src == null) { buffer.put(nullValue); } else { writeWithLength(buffer, src); } } public static final int getLength(long length) { if(length < 0){ return 9; } else if (length < 251) { return 1; } else if (length < 0x10000L) { return 3; } else if (length < 0x1000000L) { return 4; } else { return 9; } } public static final int getLength(byte[] src) { int length = src.length; if (length < 251) { return 1 + length; } else if (length < 0x10000L) { return 3 + length; } else if (length < 0x1000000L) { return 4 + length; } else { return 9 + length; } } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/ByteUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql; /** * @author mycat */ public class ByteUtil { public static int readUB2(byte[] data, int offset) { int i = data[offset] & 0xff; i |= (data[++offset] & 0xff) << 8; return i; } public static int readUB3(byte[] data, int offset) { int i = data[offset] & 0xff; i |= (data[++offset] & 0xff) << 8; i |= (data[++offset] & 0xff) << 16; return i; } public static long readUB4(byte[] data, int offset) { long l = data[offset] & 0xff; l |= (data[++offset] & 0xff) << 8; l |= (data[++offset] & 0xff) << 16; l |= (data[++offset] & 0xff) << 24; return l; } public static long readLong(byte[] data, int offset) { long l = (long) (data[offset] & 0xff); l |= (long) (data[++offset] & 0xff) << 8; l |= (long) (data[++offset] & 0xff) << 16; l |= (long) (data[++offset] & 0xff) << 24; l |= (long) (data[++offset] & 0xff) << 32; l |= (long) (data[++offset] & 0xff) << 40; l |= (long) (data[++offset] & 0xff) << 48; l |= (long) (data[++offset] & 0xff) << 56; return l; } public static long readLength(byte[] data, int offset) { int length = data[offset++] & 0xff; switch (length) { case 251: return MySQLMessage.NULL_LENGTH; case 252: return readUB2(data, offset); case 253: return readUB3(data, offset); case 254: return readLong(data, offset); default: return length; } } public static int lengthToZero(byte[] data, int offset) { int start = offset; for (int i = start; i < data.length; i++) { if (data[i] == 0) { return (i - start); } } int remaining = data.length - start; return remaining > 0 ? remaining : 0; } public static int decodeLength(byte[] src) { int length = src.length; if (length < 251) { return 1 + length; } else if (length < 0x10000L) { return 3 + length; } else if (length < 0x1000000L) { return 4 + length; } else { return 9 + length; } } public static int decodeLength(long length) { if (length < 251) { return 1; } else if (length < 0x10000L) { return 3; } else if (length < 0x1000000L) { return 4; } else { return 9; } } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/CharsetUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.FileInputStream; import java.util.*; /** * @author mycat */ public class CharsetUtil { public static final Logger logger = LoggerFactory .getLogger(CharsetUtil.class); private static final Map INDEX_TO_CHARSET = new HashMap<>(); private static final Map CHARSET_TO_INDEX = new HashMap<>(); static { // index_to_charset.properties INDEX_TO_CHARSET.put(1,"big5"); INDEX_TO_CHARSET.put(8,"latin1"); INDEX_TO_CHARSET.put(9,"latin2"); INDEX_TO_CHARSET.put(14,"cp1251"); INDEX_TO_CHARSET.put(28,"gbk"); INDEX_TO_CHARSET.put(24,"gb2312"); INDEX_TO_CHARSET.put(33,"utf8"); INDEX_TO_CHARSET.put(45,"utf8mb4"); String filePath = Thread.currentThread().getContextClassLoader() .getResource("").getPath().replaceAll("%20", " ") + "index_to_charset.properties"; Properties prop = new Properties(); try { prop.load(new FileInputStream(filePath)); for (Object index : prop.keySet()){ INDEX_TO_CHARSET.put(Integer.parseInt((String) index), prop.getProperty((String) index)); } } catch (Exception e) { logger.error("error:",e); } // charset --> index for(Integer key : INDEX_TO_CHARSET.keySet()){ String charset = INDEX_TO_CHARSET.get(key); if(charset != null && CHARSET_TO_INDEX.get(charset) == null){ CHARSET_TO_INDEX.put(charset, key); } } CHARSET_TO_INDEX.put("iso-8859-1", 14); CHARSET_TO_INDEX.put("iso_8859_1", 14); CHARSET_TO_INDEX.put("utf-8", 33); } public static final String getCharset(int index) { return INDEX_TO_CHARSET.get(index); } public static final int getIndex(String charset) { if (charset == null || charset.length() == 0) { return 0; } else { Integer i = CHARSET_TO_INDEX.get(charset.toLowerCase()); return (i == null) ? 0 : i; } } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/DataType.java ================================================ package io.mycat.backend.mysql; /** * 定义返回的数据类型 * @author huangyiming * */ public enum DataType { STRING("String"),DOUBLE("Double"),FLOAT("Float"),DATE("Date"),INT("Int"); private String type; private DataType(String type){ this.type = type; } public String getType() { return type; } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/LoadDataUtil.java ================================================ package io.mycat.backend.mysql; import java.io.*; import java.util.List; import io.mycat.MycatServer; import io.mycat.backend.BackendConnection; import io.mycat.net.BackendAIOConnection; import io.mycat.net.mysql.BinaryPacket; import io.mycat.net.mysql.CommandPacket; import io.mycat.net.mysql.MySQLPacket; import io.mycat.route.RouteResultsetNode; import io.mycat.sqlengine.mpp.LoadData; /** * Created by nange on 2015/3/31. */ public class LoadDataUtil { public static void requestFileDataResponse(byte[] data, BackendConnection conn) { byte packId= data[3]; BackendAIOConnection backendAIOConnection= (BackendAIOConnection) conn; RouteResultsetNode rrn= (RouteResultsetNode) conn.getAttachment(); LoadData loadData= rrn.getLoadData(); List loadDataData = loadData.getData(); try { if(loadDataData !=null&&loadDataData.size()>0) { ByteArrayOutputStream bos = new ByteArrayOutputStream(); for (int i = 0, loadDataDataSize = loadDataData.size(); i < loadDataDataSize; i++) { String line = loadDataData.get(i); String s =(i==loadDataDataSize-1)?line: line + loadData.getLineTerminatedBy(); byte[] bytes = s.getBytes(loadData.getCharset()); bos.write(bytes); } packId= writeToBackConnection(packId,new ByteArrayInputStream(bos.toByteArray()),backendAIOConnection); } else { //从文件读取 packId= writeToBackConnection(packId,new BufferedInputStream(new FileInputStream(loadData.getFileName())),backendAIOConnection); } }catch (IOException e) { throw new RuntimeException(e); } finally { //结束必须发空包 byte[] empty = new byte[] { 0, 0, 0,3 }; empty[3]=++packId; backendAIOConnection.write(empty); } } public static byte writeToBackConnection(byte packID,InputStream inputStream,BackendAIOConnection backendAIOConnection) throws IOException { try { int packSize = MycatServer.getInstance().getConfig().getSystem().getBufferPoolChunkSize() - 5; // int packSize = backendAIOConnection.getMaxPacketSize() / 32; // int packSize=65530; byte[] buffer = new byte[packSize]; int len = -1; while ((len = inputStream.read(buffer)) != -1) { byte[] temp = null; if (len == packSize) { temp = buffer; } else { temp = new byte[len]; System.arraycopy(buffer, 0, temp, 0, len); } BinaryPacket packet = new BinaryPacket(); packet.packetId = ++packID; packet.data = temp; packet.write(backendAIOConnection); } } finally { inputStream.close(); } return packID; } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/MySQLMessage.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql; import java.io.UnsupportedEncodingException; import java.math.BigDecimal; import java.sql.Time; import java.sql.Timestamp; import java.util.Arrays; import java.util.Calendar; /** * @author mycat */ public class MySQLMessage { public static final long NULL_LENGTH = -1; private static final byte[] EMPTY_BYTES = new byte[0]; private final byte[] data; private final int length; private int position; public MySQLMessage(byte[] data) { this.data = data; this.length = data.length; this.position = 0; } public int length() { return length; } public int position() { return position; } public byte[] bytes() { return data; } public void move(int i) { position += i; } public void position(int i) { this.position = i; } public boolean hasRemaining() { return length > position; } public byte read(int i) { return data[i]; } public byte read() { return data[position++]; } public int readUB2() { final byte[] b = this.data; int i = b[position++] & 0xff; i |= (b[position++] & 0xff) << 8; return i; } public int readUB3() { final byte[] b = this.data; int i = b[position++] & 0xff; i |= (b[position++] & 0xff) << 8; i |= (b[position++] & 0xff) << 16; return i; } public long readUB4() { final byte[] b = this.data; long l = (long) (b[position++] & 0xff); l |= (long) (b[position++] & 0xff) << 8; l |= (long) (b[position++] & 0xff) << 16; l |= (long) (b[position++] & 0xff) << 24; return l; } public int readInt() { final byte[] b = this.data; int i = b[position++] & 0xff; i |= (b[position++] & 0xff) << 8; i |= (b[position++] & 0xff) << 16; i |= (b[position++] & 0xff) << 24; return i; } public float readFloat() { return Float.intBitsToFloat(readInt()); } public long readLong() { final byte[] b = this.data; long l = (long) (b[position++] & 0xff); l |= (long) (b[position++] & 0xff) << 8; l |= (long) (b[position++] & 0xff) << 16; l |= (long) (b[position++] & 0xff) << 24; l |= (long) (b[position++] & 0xff) << 32; l |= (long) (b[position++] & 0xff) << 40; l |= (long) (b[position++] & 0xff) << 48; l |= (long) (b[position++] & 0xff) << 56; return l; } public double readDouble() { return Double.longBitsToDouble(readLong()); } public long readLength() { int length = data[position++] & 0xff; switch (length) { case 251: return NULL_LENGTH; case 252: return readUB2(); case 253: return readUB3(); case 254: return readLong(); default: return length; } } public byte[] readBytes() { if (position >= length) { return EMPTY_BYTES; } byte[] ab = new byte[length - position]; System.arraycopy(data, position, ab, 0, ab.length); position = length; return ab; } public byte[] readBytes(int length) { byte[] ab = new byte[length]; System.arraycopy(data, position, ab, 0, length); position += length; return ab; } public byte[] readBytesWithNull() { final byte[] b = this.data; if (position >= length) { return EMPTY_BYTES; } int offset = -1; for (int i = position; i < length; i++) { if (b[i] == 0) { offset = i; break; } } switch (offset) { case -1: byte[] ab1 = new byte[length - position]; System.arraycopy(b, position, ab1, 0, ab1.length); position = length; return ab1; case 0: position++; return EMPTY_BYTES; default: byte[] ab2 = new byte[offset - position]; System.arraycopy(b, position, ab2, 0, ab2.length); position = offset + 1; return ab2; } } public int getRowLength(int fileldCount) { int size = 0; int bak_position = position; position += 4; for(int i = 0 ; i < fileldCount; i++) { int length = (int) readLength(); if(length == NULL_LENGTH || length <= 0) { continue; } position += length; size += length; } position = bak_position; return size; } public byte[] readBytesWithLength() { int length = (int) readLength(); if(length==NULL_LENGTH) { return null; } if (length <= 0) { return EMPTY_BYTES; } byte[] ab = new byte[length]; System.arraycopy(data, position, ab, 0, ab.length); position += length; return ab; } public String readString() { if (position >= length) { return null; } String s = new String(data, position, length - position); position = length; return s; } public String readString(String charset) throws UnsupportedEncodingException { if (position >= length) { return null; } String s = new String(data, position, length - position, charset); position = length; return s; } public String readStringWithNull() { final byte[] b = this.data; if (position >= length) { return null; } int offset = -1; for (int i = position; i < length; i++) { if (b[i] == 0) { offset = i; break; } } if (offset == -1) { String s = new String(b, position, length - position); position = length; return s; } if (offset > position) { String s = new String(b, position, offset - position); position = offset + 1; return s; } else { position++; return null; } } public String readStringWithNull(String charset) throws UnsupportedEncodingException { final byte[] b = this.data; if (position >= length) { return null; } int offset = -1; for (int i = position; i < length; i++) { if (b[i] == 0) { offset = i; break; } } switch (offset) { case -1: String s1 = new String(b, position, length - position, charset); position = length; return s1; case 0: position++; return null; default: String s2 = new String(b, position, offset - position, charset); position = offset + 1; return s2; } } public String readStringWithLength() { int length = (int) readLength(); if (length <= 0) { return null; } String s = new String(data, position, length); position += length; return s; } public String readStringWithLength(String charset) throws UnsupportedEncodingException { int length = (int) readLength(); // if (length <= 0) { // return null; // } String s = new String(data, position, length, charset); position += length; return s; } public java.sql.Time readTime() { move(6); int hour = read(); int minute = read(); int second = read(); Calendar cal = getLocalCalendar(); cal.set(0, 0, 0, hour, minute, second); return new Time(cal.getTimeInMillis()); } public java.util.Date readDate() { byte length = read(); int year = readUB2(); byte month = read(); byte date = read(); int hour = read(); int minute = read(); int second = read(); if (length == 11) { long nanos = readUB4(); Calendar cal = getLocalCalendar(); cal.set(year, --month, date, hour, minute, second); Timestamp time = new Timestamp(cal.getTimeInMillis()); time.setNanos((int) nanos); return time; } else { Calendar cal = getLocalCalendar(); cal.set(year, --month, date, hour, minute, second); return new java.sql.Date(cal.getTimeInMillis()); } } public BigDecimal readBigDecimal() { String src = readStringWithLength(); return src == null ? null : new BigDecimal(src); } public String toString() { return new StringBuilder().append(Arrays.toString(data)).toString(); } private static final ThreadLocal localCalendar = new ThreadLocal(); private static final Calendar getLocalCalendar() { Calendar cal = localCalendar.get(); if (cal == null) { cal = Calendar.getInstance(); localCalendar.set(cal); } return cal; } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/PacketUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql; import java.io.UnsupportedEncodingException; import io.mycat.config.ErrorCode; import io.mycat.net.mysql.BinaryPacket; import io.mycat.net.mysql.ErrorPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; /** * @author mycat */ public class PacketUtil { private static final String CODE_PAGE_1252 = "Cp1252"; public static final ResultSetHeaderPacket getHeader(int fieldCount) { ResultSetHeaderPacket packet = new ResultSetHeaderPacket(); packet.packetId = 1; packet.fieldCount = fieldCount; return packet; } public static byte[] encode(String src, String charset) { if (src == null) { return null; } try { return src.getBytes(charset); } catch (UnsupportedEncodingException e) { return src.getBytes(); } } public static final FieldPacket getField(String name, String orgName, int type) { FieldPacket packet = new FieldPacket(); packet.charsetIndex = CharsetUtil.getIndex(CODE_PAGE_1252); packet.name = encode(name, CODE_PAGE_1252); packet.orgName = encode(orgName, CODE_PAGE_1252); packet.type = (byte) type; return packet; } public static final FieldPacket getField(String name, int type) { FieldPacket packet = new FieldPacket(); packet.charsetIndex = CharsetUtil.getIndex(CODE_PAGE_1252); packet.name = encode(name, CODE_PAGE_1252); packet.type = (byte) type; return packet; } public static final ErrorPacket getShutdown() { ErrorPacket error = new ErrorPacket(); error.packetId = 1; error.errno = ErrorCode.ER_SERVER_SHUTDOWN; error.message = "The server has been shutdown".getBytes(); return error; } public static final FieldPacket getField(BinaryPacket src, String fieldName) { FieldPacket field = new FieldPacket(); field.read(src); field.name = encode(fieldName, CODE_PAGE_1252); field.packetLength = field.calcPacketSize(); return field; } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/PreparedStatement.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.HashMap; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.ast.statement.SQLSelect; import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; import com.alibaba.druid.sql.parser.SQLParserUtils; import com.alibaba.druid.sql.parser.SQLStatementParser; import com.alibaba.druid.util.JdbcUtils; import io.mycat.net.mysql.FieldPacket; /** * @author mycat, CrazyPig */ public class PreparedStatement { private static final Logger LOGGER = LoggerFactory.getLogger(PreparedStatement.class); private long id; private String statement; private String[] columnNames; private int parametersNumber; private int[] parametersType; private FieldPacket[] params; private FieldPacket[] fields; /** * 存放COM_STMT_SEND_LONG_DATA命令发送过来的字节数据 * 
     * key : param_id
     * value : byte data
     *
*/ private Map longDataMap; public PreparedStatement(long id, String statement, int parametersNumber) { this.id = id; this.statement = statement; // this.columnNames = getColumns(statement); this.parametersNumber = parametersNumber; this.parametersType = new int[parametersNumber]; this.longDataMap = new HashMap(); constructColumns(); } // 获取预处理语句中column的个数 public void constructColumns() { String[] columnNames; try { SQLStatementParser sqlStatementParser = SQLParserUtils.createSQLStatementParser(statement, JdbcUtils.MYSQL); SQLStatement statement = sqlStatementParser.parseStatement(); if (statement instanceof SQLSelectStatement) { SQLSelect select = ((SQLSelectStatement) statement).getSelect(); com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock query = (com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock) select.getQuery(); int size = query.getSelectList().size(); if (size == 1){ if("*".equalsIgnoreCase( query.getSelectList().get(0).toString())){ throw new Exception("unsupport * in select items:" + statement); } } { columnNames = new String[size]; for (int i = 0; i < size; i++) { columnNames[i] = query.getSelectList().get(i).toString(); } this.columnNames = columnNames; } } }catch (Exception e){ LOGGER.error("can not get column count",e); } this.columnNames = new String[] {}; } public long getId() { return id; } public String getStatement() { return statement; } public int getColumnsNumber() { if (this.fields != null) { return this.fields.length; } else { return this.columnNames.length; } } public int getParametersNumber() { return parametersNumber; } public int[] getParametersType() { return parametersType; } public boolean hasLongData(long paramId) { return longDataMap.containsKey(paramId); } public ByteArrayOutputStream getLongData(long paramId) { return longDataMap.get(paramId); } /** * COM_STMT_RESET命令将调用该方法进行数据重置 */ public void resetLongData() { for(Long paramId : longDataMap.keySet()) { longDataMap.get(paramId).reset(); } } /** * 追加数据到指定的预处理参数 * @param paramId * @param data * @throws IOException */ public void appendLongData(long paramId, byte[] data) throws IOException { if(getLongData(paramId) == null) { ByteArrayOutputStream out = new ByteArrayOutputStream(); out.write(data); longDataMap.put(paramId, out); } else { longDataMap.get(paramId).write(data); } } public String[] getColumnNames() { return columnNames; } public FieldPacket[] getParams() { return params; } public void setParams(FieldPacket[] params) { this.params = params; this.parametersNumber = params == null ? 0 : params.length; } public FieldPacket[] getFields() { return fields; } public void setFields(FieldPacket[] fields) { this.fields = fields; } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/SecurityUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; /** * 加密解密工具类 * * @author mycat */ public class SecurityUtil { public static final byte[] scramble411(byte[] pass, byte[] seed) throws NoSuchAlgorithmException { MessageDigest md = MessageDigest.getInstance("SHA-1"); byte[] pass1 = md.digest(pass); md.reset(); byte[] pass2 = md.digest(pass1); md.reset(); md.update(seed); byte[] pass3 = md.digest(pass2); for (int i = 0; i < pass3.length; i++) { pass3[i] = (byte) (pass3[i] ^ pass1[i]); } return pass3; } public static final String scramble323(String pass, String seed) { if ((pass == null) || (pass.length() == 0)) { return pass; } byte b; double d; long[] pw = hash(seed); long[] msg = hash(pass); long max = 0x3fffffffL; long seed1 = (pw[0] ^ msg[0]) % max; long seed2 = (pw[1] ^ msg[1]) % max; char[] chars = new char[seed.length()]; for (int i = 0; i < seed.length(); i++) { seed1 = ((seed1 * 3) + seed2) % max; seed2 = (seed1 + seed2 + 33) % max; d = (double) seed1 / (double) max; b = (byte) java.lang.Math.floor((d * 31) + 64); chars[i] = (char) b; } seed1 = ((seed1 * 3) + seed2) % max; // seed2 = (seed1 + seed2 + 33) % max; d = (double) seed1 / (double) max; b = (byte) java.lang.Math.floor(d * 31); for (int i = 0; i < seed.length(); i++) { chars[i] ^= (char) b; } return new String(chars); } private static long[] hash(String src) { long nr = 1345345333L; long add = 7; long nr2 = 0x12345671L; long tmp; for (int i = 0; i < src.length(); ++i) { switch (src.charAt(i)) { case ' ': case '\t': continue; default: tmp = (0xff & src.charAt(i)); nr ^= ((((nr & 63) + add) * tmp) + (nr << 8)); nr2 += ((nr2 << 8) ^ nr); add += tmp; } } long[] result = new long[2]; result[0] = nr & 0x7fffffffL; result[1] = nr2 & 0x7fffffffL; return result; } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/StreamUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; /** * @author mycat */ public class StreamUtil { private static final long NULL_LENGTH = -1; private static final byte[] EMPTY_BYTES = new byte[0]; public static final void read(InputStream in, byte[] b, int offset, int length) throws IOException { for (int got = 0; length > 0;) { got = in.read(b, offset, length); if (got < 0) { throw new EOFException(); } offset += got; length -= got; } } public static final byte read(InputStream in) throws IOException { int got = in.read(); if (got < 0) { throw new EOFException(); } return (byte) (got & 0xff); } public static final int readUB2(InputStream in) throws IOException { byte[] b = new byte[2]; read(in, b, 0, b.length); int i = b[0] & 0xff; i |= (b[1] & 0xff) << 8; return i; } public static final int readUB3(InputStream in) throws IOException { byte[] b = new byte[3]; read(in, b, 0, b.length); int i = b[0] & 0xff; i |= (b[1] & 0xff) << 8; i |= (b[2] & 0xff) << 16; return i; } public static final int readInt(InputStream in) throws IOException { byte[] b = new byte[4]; read(in, b, 0, b.length); int i = b[0] & 0xff; i |= (b[1] & 0xff) << 8; i |= (b[2] & 0xff) << 16; i |= (b[3] & 0xff) << 24; return i; } public static final float readFloat(InputStream in) throws IOException { return Float.intBitsToFloat(readInt(in)); } public static final long readUB4(InputStream in) throws IOException { byte[] b = new byte[4]; read(in, b, 0, b.length); long l = (long) (b[0] & 0xff); l |= (long) (b[1] & 0xff) << 8; l |= (long) (b[2] & 0xff) << 16; l |= (long) (b[3] & 0xff) << 24; return l; } public static final long readLong(InputStream in) throws IOException { byte[] b = new byte[8]; read(in, b, 0, b.length); long l = (long) (b[0] & 0xff); l |= (long) (b[1] & 0xff) << 8; l |= (long) (b[2] & 0xff) << 16; l |= (long) (b[3] & 0xff) << 24; l |= (long) (b[4] & 0xff) << 32; l |= (long) (b[5] & 0xff) << 40; l |= (long) (b[6] & 0xff) << 48; l |= (long) (b[7] & 0xff) << 56; return l; } public static final double readDouble(InputStream in) throws IOException { return Double.longBitsToDouble(readLong(in)); } public static final byte[] readWithLength(InputStream in) throws IOException { int length = (int) readLength(in); if (length <= 0) { return EMPTY_BYTES; } byte[] b = new byte[length]; read(in, b, 0, b.length); return b; } public static final void write(OutputStream out, byte b) throws IOException { out.write(b & 0xff); } public static final void writeUB2(OutputStream out, int i) throws IOException { byte[] b = new byte[2]; b[0] = (byte) (i & 0xff); b[1] = (byte) (i >>> 8); out.write(b); } public static final void writeUB3(OutputStream out, int i) throws IOException { byte[] b = new byte[3]; b[0] = (byte) (i & 0xff); b[1] = (byte) (i >>> 8); b[2] = (byte) (i >>> 16); out.write(b); } public static final void writeInt(OutputStream out, int i) throws IOException { byte[] b = new byte[4]; b[0] = (byte) (i & 0xff); b[1] = (byte) (i >>> 8); b[2] = (byte) (i >>> 16); b[3] = (byte) (i >>> 24); out.write(b); } public static final void writeFloat(OutputStream out, float f) throws IOException { writeInt(out, Float.floatToIntBits(f)); } public static final void writeUB4(OutputStream out, long l) throws IOException { byte[] b = new byte[4]; b[0] = (byte) (l & 0xff); b[1] = (byte) (l >>> 8); b[2] = (byte) (l >>> 16); b[3] = (byte) (l >>> 24); out.write(b); } public static final void writeLong(OutputStream out, long l) throws IOException { byte[] b = new byte[8]; b[0] = (byte) (l & 0xff); b[1] = (byte) (l >>> 8); b[2] = (byte) (l >>> 16); b[3] = (byte) (l >>> 24); b[4] = (byte) (l >>> 32); b[5] = (byte) (l >>> 40); b[6] = (byte) (l >>> 48); b[7] = (byte) (l >>> 56); out.write(b); } public static final void writeDouble(OutputStream out, double d) throws IOException { writeLong(out, Double.doubleToLongBits(d)); } public static final long readLength(InputStream in) throws IOException { int length = in.read(); if (length < 0) { throw new EOFException(); } switch (length) { case 251: return NULL_LENGTH; case 252: return readUB2(in); case 253: return readUB3(in); case 254: return readLong(in); default: return length; } } public static final void writeLength(OutputStream out, long length) throws IOException { if (length < 251) { out.write((byte) length); } else if (length < 0x10000L) { out.write((byte) 252); writeUB2(out, (int) length); } else if (length < 0x1000000L) { out.write((byte) 253); writeUB3(out, (int) length); } else { out.write((byte) 254); writeLong(out, length); } } public static final void writeWithNull(OutputStream out, byte[] src) throws IOException { out.write(src); out.write((byte) 0); } public static final void writeWithLength(OutputStream out, byte[] src) throws IOException { int length = src.length; if (length < 251) { out.write((byte) length); } else if (length < 0x10000L) { out.write((byte) 252); writeUB2(out, length); } else if (length < 0x1000000L) { out.write((byte) 253); writeUB3(out, length); } else { out.write((byte) 254); writeLong(out, length); } out.write(src); } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/listener/DefaultSqlExecuteStageListener.java ================================================ package io.mycat.backend.mysql.listener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.server.ServerConnection; /** * sql各自执行阶段（读取，解析，路由，执行，完成） * @author funny 2020年7月26日 下午9:36:59 * @since 1.0.0 */ public class DefaultSqlExecuteStageListener implements SqlExecuteStageListener { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultSqlExecuteStageListener.class); private ServerConnection source; public DefaultSqlExecuteStageListener(ServerConnection source) { this.source = source; } @Override public void fireEvent(SqlExecuteStage stage) { switch (stage) { case READ: onReadCompleted(); break; case PARSE: onParseCompleted(); break; case ROUTE: onRouteCompleted(); break; case EXECUTE: onExecuteCompleted(); break; case MERGE: onMergeCompleted(); break; case END: onEndCompleted(); break; } } private void onReadCompleted() { } private void onParseCompleted() { } private void onRouteCompleted() { } private void onExecuteCompleted() { } private void onMergeCompleted() { } private void onEndCompleted() { LOGGER.debug("on event sql end"); source.onEventSqlCompleted(); } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/listener/SqlExecuteStage.java ================================================ package io.mycat.backend.mysql.listener; public enum SqlExecuteStage { READ, PARSE, ROUTE, EXECUTE, MERGE, END } ================================================ FILE: src/main/java/io/mycat/backend/mysql/listener/SqlExecuteStageListener.java ================================================ package io.mycat.backend.mysql.listener; /** * 定义sql各个执行阶段（读取，解析，路由，执行，完成）监听事件，比如写日志，写执行统计 * @author funnyAnt 2020年7月26日 下午9:54:51 * @since 1.0.0 */ public interface SqlExecuteStageListener { void fireEvent(SqlExecuteStage stage); } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/MySQLConnection.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql.nio; import java.io.UnsupportedEncodingException; import java.nio.channels.NetworkChannel; import java.security.NoSuchAlgorithmException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.backend.mysql.CharsetUtil; import io.mycat.backend.mysql.SecurityUtil; import io.mycat.backend.mysql.nio.handler.ResponseHandler; import io.mycat.backend.mysql.xa.TxState; import io.mycat.config.Capabilities; import io.mycat.config.Isolations; import io.mycat.net.BackendAIOConnection; import io.mycat.net.mysql.AuthPacket; import io.mycat.net.mysql.CommandPacket; import io.mycat.net.mysql.HandshakePacket; import io.mycat.net.mysql.MySQLPacket; import io.mycat.net.mysql.QuitPacket; import io.mycat.route.RouteResultsetNode; import io.mycat.server.ServerConnection; import io.mycat.server.parser.ServerParse; import io.mycat.util.TimeUtil; import io.mycat.util.exception.UnknownTxIsolationException; /** * @author mycat */ public class MySQLConnection extends BackendAIOConnection { private static final Logger LOGGER = LoggerFactory .getLogger(MySQLConnection.class); private static final long CLIENT_FLAGS = initClientFlags(); private volatile long lastTime; private volatile String schema = null; private volatile String oldSchema; private volatile boolean borrowed = false; private volatile boolean modifiedSQLExecuted = false; private volatile int batchCmdCount = 0; private static long initClientFlags() { int flag = 0; flag |= Capabilities.CLIENT_LONG_PASSWORD; flag |= Capabilities.CLIENT_FOUND_ROWS; flag |= Capabilities.CLIENT_LONG_FLAG; flag |= Capabilities.CLIENT_CONNECT_WITH_DB; // flag |= Capabilities.CLIENT_NO_SCHEMA; boolean usingCompress=MycatServer.getInstance().getConfig().getSystem().getUseCompression()==1 ; if(usingCompress) { flag |= Capabilities.CLIENT_COMPRESS; } flag |= Capabilities.CLIENT_ODBC; flag |= Capabilities.CLIENT_LOCAL_FILES; flag |= Capabilities.CLIENT_IGNORE_SPACE; flag |= Capabilities.CLIENT_PROTOCOL_41; flag |= Capabilities.CLIENT_INTERACTIVE; // flag |= Capabilities.CLIENT_SSL; flag |= Capabilities.CLIENT_IGNORE_SIGPIPE; flag |= Capabilities.CLIENT_TRANSACTIONS; // flag |= Capabilities.CLIENT_RESERVED; flag |= Capabilities.CLIENT_SECURE_CONNECTION; // client extension flag |= Capabilities.CLIENT_MULTI_STATEMENTS; flag |= Capabilities.CLIENT_MULTI_RESULTS; return flag; } private static final CommandPacket _READ_UNCOMMITTED = new CommandPacket(); private static final CommandPacket _READ_COMMITTED = new CommandPacket(); private static final CommandPacket _REPEATED_READ = new CommandPacket(); private static final CommandPacket _SERIALIZABLE = new CommandPacket(); private static final CommandPacket _AUTOCOMMIT_ON = new CommandPacket(); private static final CommandPacket _AUTOCOMMIT_OFF = new CommandPacket(); private static final CommandPacket _COMMIT = new CommandPacket(); private static final CommandPacket _ROLLBACK = new CommandPacket(); static { _READ_UNCOMMITTED.packetId = 0; _READ_UNCOMMITTED.command = MySQLPacket.COM_QUERY; _READ_UNCOMMITTED.arg = "SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED" .getBytes(); _READ_COMMITTED.packetId = 0; _READ_COMMITTED.command = MySQLPacket.COM_QUERY; _READ_COMMITTED.arg = "SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED" .getBytes(); _REPEATED_READ.packetId = 0; _REPEATED_READ.command = MySQLPacket.COM_QUERY; _REPEATED_READ.arg = "SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ" .getBytes(); _SERIALIZABLE.packetId = 0; _SERIALIZABLE.command = MySQLPacket.COM_QUERY; _SERIALIZABLE.arg = "SET SESSION TRANSACTION ISOLATION LEVEL SERIALIZABLE" .getBytes(); _AUTOCOMMIT_ON.packetId = 0; _AUTOCOMMIT_ON.command = MySQLPacket.COM_QUERY; _AUTOCOMMIT_ON.arg = "SET autocommit=1".getBytes(); _AUTOCOMMIT_OFF.packetId = 0; _AUTOCOMMIT_OFF.command = MySQLPacket.COM_QUERY; _AUTOCOMMIT_OFF.arg = "SET autocommit=0".getBytes(); _COMMIT.packetId = 0; _COMMIT.command = MySQLPacket.COM_QUERY; _COMMIT.arg = "commit".getBytes(); _ROLLBACK.packetId = 0; _ROLLBACK.command = MySQLPacket.COM_QUERY; _ROLLBACK.arg = "rollback".getBytes(); } private MySQLDataSource pool; private boolean fromSlaveDB; private long threadId; private HandshakePacket handshake; private volatile int txIsolation; private volatile boolean autocommit; private volatile boolean txReadonly; /** 保存SET SQL_SELECT_LIMIT的值, default 解析为-1. */ private volatile int sqlSelectLimit = -1; private long clientFlags; private boolean isAuthenticated; private String user; private String password; private Object attachment; private volatile ResponseHandler respHandler; private final AtomicBoolean isQuit; private volatile StatusSync statusSync; private volatile boolean metaDataSyned = true; private volatile int xaStatus = 0; public MySQLConnection(NetworkChannel channel, boolean fromSlaveDB) { super(channel); this.clientFlags = CLIENT_FLAGS; this.lastTime = TimeUtil.currentTimeMillis(); this.isQuit = new AtomicBoolean(false); this.autocommit = true; this.fromSlaveDB = fromSlaveDB; // 设为默认值，免得每个初始化好的连接都要去同步一下 this.txIsolation = MycatServer.getInstance().getConfig().getSystem().getTxIsolation(); this.txReadonly = false; } public int getXaStatus() { return xaStatus; } public void setXaStatus(int xaStatus) { this.xaStatus = xaStatus; } public void onConnectFailed(Throwable t) { if (handler instanceof MySQLConnectionHandler) { MySQLConnectionHandler theHandler = (MySQLConnectionHandler) handler; theHandler.connectionError(t); } else { ((MySQLConnectionAuthenticator) handler).connectionError(this, t); } } public String getSchema() { return this.schema; } public void setSchema(String newSchema) { String curSchema = schema; if (curSchema == null) { this.schema = newSchema; this.oldSchema = newSchema; } else { this.oldSchema = curSchema; this.schema = newSchema; } } public MySQLDataSource getPool() { return pool; } public void setPool(MySQLDataSource pool) { this.pool = pool; } public String getUser() { return user; } public void setUser(String user) { this.user = user; } public void setPassword(String password) { this.password = password; } public HandshakePacket getHandshake() { return handshake; } public void setHandshake(HandshakePacket handshake) { this.handshake = handshake; } public long getThreadId() { return threadId; } public void setThreadId(long threadId) { this.threadId = threadId; } public boolean isAuthenticated() { return isAuthenticated; } public void setAuthenticated(boolean isAuthenticated) { this.isAuthenticated = isAuthenticated; } public String getPassword() { return password; } public void authenticate() { AuthPacket packet = new AuthPacket(); packet.packetId = 1; packet.clientFlags = clientFlags; packet.maxPacketSize = maxPacketSize; packet.charsetIndex = this.charsetIndex; packet.user = user; try { packet.password = passwd(password, handshake); } catch (NoSuchAlgorithmException e) { throw new RuntimeException(e.getMessage()); } packet.database = schema; packet.write(this); } public boolean isAutocommit() { return autocommit; } public boolean isTxReadonly() { return txReadonly; } public int getSqlSelectLimit() { return sqlSelectLimit; } public Object getAttachment() { return attachment; } public void setAttachment(Object attachment) { this.attachment = attachment; } public boolean isClosedOrQuit() { return isClosed() || isQuit.get(); } /** * 
	 * 用于解决mysql协议中com_field_list类似的命令的支持 
	 * （https://dev.mysql.com/doc/internals/en/com-field-list.html）
	 * 如ogg工具中使用到此命令。
	 *
*/ private void sendComFieldListCmd(String query) { CommandPacket packet = new CommandPacket(); packet.packetId = 0; packet.command = MySQLPacket.COM_FIELD_LIST; try { //只需要命令中最后的具体信息 int index = query.indexOf(ServerParse.COM_FIELD_LIST_FLAG); query = query.substring(index + 17); packet.arg = query.getBytes(charset); //把query中最后一个改为协议中 0x00 packet.arg[query.length() - 1] = (byte) 0x00; } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } lastTime = TimeUtil.currentTimeMillis(); packet.write(this); } protected void sendQueryCmd(String query) { CommandPacket packet = new CommandPacket(); packet.packetId = 0; packet.command = MySQLPacket.COM_QUERY; try { packet.arg = query.getBytes(charset); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } lastTime = TimeUtil.currentTimeMillis(); packet.write(this); } private static void getCharsetCommand(StringBuilder sb, int clientCharIndex) { sb.append("SET names ").append(CharsetUtil.getCharset(clientCharIndex)) .append(";"); } private static void getTxIsolationCommand(StringBuilder sb, int txIsolation) { switch (txIsolation) { case Isolations.READ_UNCOMMITTED: sb.append("SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;"); return; case Isolations.READ_COMMITTED: sb.append("SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED;"); return; case Isolations.REPEATED_READ: sb.append("SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ;"); return; case Isolations.SERIALIZABLE: sb.append("SET SESSION TRANSACTION ISOLATION LEVEL SERIALIZABLE;"); return; default: throw new UnknownTxIsolationException("txIsolation:" + txIsolation); } } private void getAutocommitCommand(StringBuilder sb, boolean autoCommit) { if (autoCommit) { sb.append("SET autocommit=1;"); } else { sb.append("SET autocommit=0;"); } } private void getTxReadonly(StringBuilder sb, boolean txReadonly) { if (txReadonly) { sb.append("SET SESSION TRANSACTION READ ONLY;"); } else { sb.append("SET SESSION TRANSACTION READ WRITE;"); } } private void getSqlSelectLimit(StringBuilder sb, int sqlSelectLimit) { if (sqlSelectLimit == -1) { sb.append("SET SQL_SELECT_LIMIT=DEFAULT;"); } else { sb.append("SET SQL_SELECT_LIMIT=").append(sqlSelectLimit).append(";"); } } private static class StatusSync { private final String schema; private final Integer charsetIndex; private final Integer txtIsolation; private final Boolean autocommit; private final AtomicInteger synCmdCount; private final boolean xaStarted; private final Boolean txReadonly; private final Integer sqlSelectLimit; public StatusSync(boolean xaStarted, String schema, Integer charsetIndex, Integer txtIsolation, Boolean autocommit, int synCount, boolean txReadonly, Integer sqlSelectLimit) { super(); this.xaStarted = xaStarted; this.schema = schema; this.charsetIndex = charsetIndex; this.txtIsolation = txtIsolation; this.autocommit = autocommit; this.synCmdCount = new AtomicInteger(synCount); this.txReadonly = txReadonly; this.sqlSelectLimit = sqlSelectLimit; } public boolean synAndExecuted(MySQLConnection conn) { int remains = synCmdCount.decrementAndGet(); if (remains == 0) {// syn command finished this.updateConnectionInfo(conn); conn.metaDataSyned = true; return false; } else if (remains < 0) { return true; } return false; } private void updateConnectionInfo(MySQLConnection conn) { if (schema != null) { conn.schema = schema; conn.oldSchema = conn.schema; } if (charsetIndex != null) { conn.setCharset(CharsetUtil.getCharset(charsetIndex)); } if (txtIsolation != null) { conn.txIsolation = txtIsolation; } if (autocommit != null) { conn.autocommit = autocommit; } if (txReadonly != null) { conn.txReadonly = txReadonly; } if (sqlSelectLimit != null) { conn.sqlSelectLimit = sqlSelectLimit; } } } /** * @return if synchronization finished and execute-sql has already been sent * before */ public boolean syncAndExcute() { StatusSync sync = this.statusSync; if (sync == null) { return true; } else { boolean executed = sync.synAndExecuted(this); if (executed) { statusSync = null; } return executed; } } public void execute(RouteResultsetNode rrn, ServerConnection sc, boolean autocommit) throws UnsupportedEncodingException { if (!modifiedSQLExecuted && rrn.isModifySQL()) { modifiedSQLExecuted = true; } String xaTXID = null; if(sc.getSession2().getXaTXID()!=null){ xaTXID = sc.getSession2().getXaTXID()+",'"+getSchema()+"'"; } synAndDoExecute(xaTXID, rrn, sc.getCharsetIndex(), sc.getTxIsolation(), autocommit, sc.isTxReadonly(), sc.getSqlSelectLimit()); } private void synAndDoExecute(String xaTxID, RouteResultsetNode rrn, int clientCharSetIndex, int clientTxIsoLation, boolean clientAutoCommit, boolean clientTxReadonly, int clientSqlSelectLimit) { String xaCmd = null; boolean conAutoComit = this.autocommit; boolean conTxReadonly = this.txReadonly; int conSqlSelectLimit = this.sqlSelectLimit; String conSchema = this.schema; boolean strictTxIsolation = MycatServer.getInstance().getConfig().getSystem().isStrictTxIsolation(); boolean expectAutocommit = false; // 如果在非自动提交情况下,如果需要严格保证事务级别,则需做下列判断 if (strictTxIsolation) { expectAutocommit = isFromSlaveDB() || clientAutoCommit; } else { // never executed modify sql,so auto commit expectAutocommit = (!modifiedSQLExecuted || isFromSlaveDB() || clientAutoCommit); } if (expectAutocommit == false && xaTxID != null && xaStatus == TxState.TX_INITIALIZE_STATE) { //clientTxIsoLation = Isolations.SERIALIZABLE; xaCmd = "XA START " + xaTxID + ';'; this.xaStatus = TxState.TX_STARTED_STATE; } int schemaSyn = conSchema.equals(oldSchema) ? 0 : 1; int charsetSyn = 0; if (this.charsetIndex != clientCharSetIndex) { //need to syn the charset of connection. //set current connection charset to client charset. //otherwise while sending commend to server the charset will not coincidence. setCharset(CharsetUtil.getCharset(clientCharSetIndex)); charsetSyn = 1; } int txIsoLationSyn = (txIsolation == clientTxIsoLation) ? 0 : 1; int autoCommitSyn = (conAutoComit == expectAutocommit) ? 0 : 1; int txReadonlySyn = (conTxReadonly == clientTxReadonly) ? 0 : 1; int sqlSelectLimitSyn = (conSqlSelectLimit == clientSqlSelectLimit) ? 0 : 1; int synCount = schemaSyn + charsetSyn + txIsoLationSyn + autoCommitSyn + (xaCmd!=null?1:0) + txReadonlySyn + sqlSelectLimitSyn; // if (synCount == 0 && this.xaStatus != TxState.TX_STARTED_STATE) { if (synCount == 0 ) { // not need syn connection // if (LOGGER.isDebugEnabled()) { // LOGGER.debug("not need syn connection :\n" + this+"\n to send query cmd:\n"+rrn.getStatement() // +"\n in pool\n" // +this.getPool().getConfig()); // } if (rrn.getSqlType() == ServerParse.COMMAND) { this.sendComFieldListCmd(rrn.getStatement() + ";"); return; } sendQueryCmd(rrn.getStatement()); return; } CommandPacket schemaCmd = null; StringBuilder sb = new StringBuilder(); if (schemaSyn == 1) { schemaCmd = getChangeSchemaCommand(conSchema); // getChangeSchemaCommand(sb, conSchema); } if (charsetSyn == 1) { getCharsetCommand(sb, clientCharSetIndex); } if (txIsoLationSyn == 1) { getTxIsolationCommand(sb, clientTxIsoLation); } if (autoCommitSyn == 1) { getAutocommitCommand(sb, expectAutocommit); } if (txReadonlySyn == 1) { getTxReadonly(sb, false); } if (sqlSelectLimitSyn == 1) { getSqlSelectLimit(sb, clientSqlSelectLimit); } if (xaCmd != null) { sb.append(xaCmd); } metaDataSyned = false; statusSync = new StatusSync(xaCmd != null, conSchema, clientCharSetIndex, clientTxIsoLation, expectAutocommit, synCount, clientTxReadonly, clientSqlSelectLimit); if (LOGGER.isDebugEnabled()) { LOGGER.debug("con need syn ,total syn cmd " + synCount + " commands " + sb.toString() + "schema change:" + (schemaCmd != null) + " con:" + this); } // syn schema if (schemaCmd != null) { schemaCmd.write(this); } if(rrn.getSqlType() == ServerParse.COMMAND ) { if(sb.length() > 0 ) { statusSync.synCmdCount.incrementAndGet(); this.sendQueryCmd(sb.toString()); } this.sendComFieldListCmd(rrn.getStatement()+";"); return ; } // and our query sql to multi command at last sb.append(rrn.getStatement()+";"); // syn and execute others this.sendQueryCmd(sb.toString()); // waiting syn result... } private static CommandPacket getChangeSchemaCommand(String schema) { CommandPacket cmd = new CommandPacket(); cmd.packetId = 0; cmd.command = MySQLPacket.COM_INIT_DB; cmd.arg = schema.getBytes(); return cmd; } /** * by wuzh ,execute a query and ignore transaction settings for performance * * @param query * @throws UnsupportedEncodingException */ public void query(String query) throws UnsupportedEncodingException { RouteResultsetNode rrn = new RouteResultsetNode("default", ServerParse.SELECT, query); synAndDoExecute(null, rrn, this.charsetIndex, this.txIsolation, true, this.txReadonly, this.sqlSelectLimit); } /** * by zwy ,execute a query with charsetIndex * * @param query * @throws UnsupportedEncodingException */ @Override public void query(String query, int charsetIndex) { RouteResultsetNode rrn = new RouteResultsetNode("default", ServerParse.SELECT, query); synAndDoExecute(null, rrn, charsetIndex, this.txIsolation, true, this.txReadonly, this.sqlSelectLimit); } public long getLastTime() { return lastTime; } public void setLastTime(long lastTime) { this.lastTime = lastTime; } public void quit() { if (isQuit.compareAndSet(false, true) && !isClosed()) { if (isAuthenticated) { write(writeToBuffer(QuitPacket.QUIT, allocate())); write(allocate()); } else { close("normal"); } } } @Override public void close(String reason) { if (!isClosed.get()) { isQuit.set(true); ResponseHandler tmpRespHandlers= respHandler; setResponseHandler(null); super.close(reason); pool.connectionClosed(this); if (tmpRespHandlers != null) { tmpRespHandlers.connectionClose(this, reason); } if( this.handler instanceof MySQLConnectionAuthenticator) { ((MySQLConnectionAuthenticator) this.handler).connectionError(this, new Throwable(reason)); } } else { //主要起一个清理资源的作用 super.close(reason); } } @Override public void closeWithoutRsp(String reason) { // TODO Auto-generated method stub this.respHandler = null; this.close(reason); } public void commit() { _COMMIT.write(this); } public boolean batchCmdFinished() { batchCmdCount--; return (batchCmdCount == 0); } public void execCmd(String cmd) { this.sendQueryCmd(cmd); } public void execBatchCmd(String[] batchCmds) { // "XA END "+xaID+";"+"XA PREPARE "+xaID this.batchCmdCount = batchCmds.length; StringBuilder sb = new StringBuilder(); for (String sql : batchCmds) { sb.append(sql).append(';'); } this.sendQueryCmd(sb.toString()); } public void rollback() { _ROLLBACK.write(this); } public void release() { if (metaDataSyned == false) {// indicate connection not normalfinished // ,and // we can't know it's syn status ,so // close // it LOGGER.warn("can't sure connection syn result,so close it " + this); this.respHandler = null; this.close("syn status unkown "); return; } metaDataSyned = true; attachment = null; statusSync = null; modifiedSQLExecuted = false; xaStatus = TxState.TX_INITIALIZE_STATE; setResponseHandler(null); pool.releaseChannel(this); } public boolean setResponseHandler(ResponseHandler queryHandler) { if (handler instanceof MySQLConnectionHandler) { ((MySQLConnectionHandler) handler).setResponseHandler(queryHandler); respHandler = queryHandler; return true; } else if (queryHandler != null) { LOGGER.warn("set not MySQLConnectionHandler " + queryHandler.getClass().getCanonicalName()); } return false; } /** * 写队列为空，可以继续写数据 */ public void writeQueueAvailable() { if (respHandler != null) { respHandler.writeQueueAvailable(); } } /** * 记录sql执行信息 */ public void recordSql(String host, String schema, String stmt) { // final long now = TimeUtil.currentTimeMillis(); // if (now > this.lastTime) { // // long time = now - this.lastTime; // // SQLRecorder sqlRecorder = this.pool.getSqlRecorder(); // // if (sqlRecorder.check(time)) { // // SQLRecord recorder = new SQLRecord(); // // recorder.host = host; // // recorder.schema = schema; // // recorder.statement = stmt; // // recorder.startTime = lastTime; // // recorder.executeTime = time; // // recorder.dataNode = pool.getName(); // // recorder.dataNodeIndex = pool.getIndex(); // // sqlRecorder.add(recorder); // // } // } // this.lastTime = now; } private static byte[] passwd(String pass, HandshakePacket hs) throws NoSuchAlgorithmException { if (pass == null || pass.length() == 0) { return null; } byte[] passwd = pass.getBytes(); int sl1 = hs.seed.length; int sl2 = hs.restOfScrambleBuff.length; byte[] seed = new byte[sl1 + sl2]; System.arraycopy(hs.seed, 0, seed, 0, sl1); System.arraycopy(hs.restOfScrambleBuff, 0, seed, sl1, sl2); return SecurityUtil.scramble411(passwd, seed); } @Override public boolean isFromSlaveDB() { return fromSlaveDB; } @Override public boolean isBorrowed() { return borrowed; } @Override public void setBorrowed(boolean borrowed) { this.lastTime = TimeUtil.currentTimeMillis(); this.borrowed = borrowed; } @Override public String toString() { return "MySQLConnection@"+ hashCode() +" [id=" + id + ", lastTime=" + lastTime + ", user=" + user + ", schema=" + schema + ", old shema=" + oldSchema + ", borrowed=" + borrowed + ", fromSlaveDB=" + fromSlaveDB + ", threadId=" + threadId + ", charset=" + charset + ", txIsolation=" + txIsolation + ", autocommit=" + autocommit + ", txReadonly=" + txReadonly + ", attachment=" + attachment + ", respHandler=" + respHandler + ", host=" + host + ", port=" + port + ", statusSync=" + statusSync + ", writeQueue=" + this.getWriteQueue().size() + ", modifiedSQLExecuted=" + modifiedSQLExecuted + "]"; } @Override public boolean isModifiedSQLExecuted() { return modifiedSQLExecuted; } @Override public int getTxIsolation() { return txIsolation; } @Override public void disableRead() { this.getSocketWR().disableRead(); } @Override public void enableRead() { this.getSocketWR().enableRead(); } // 是否需要同步schmea public boolean isNeedSyncSchema() { return schema.equals(oldSchema) ? false : true; } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/MySQLConnectionAuthenticator.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql.nio; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.backend.mysql.CharsetUtil; import io.mycat.backend.mysql.SecurityUtil; import io.mycat.backend.mysql.nio.handler.ResponseHandler; import io.mycat.config.Capabilities; import io.mycat.net.ConnectionException; import io.mycat.net.NIOHandler; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.ErrorPacket; import io.mycat.net.mysql.HandshakePacket; import io.mycat.net.mysql.OkPacket; import io.mycat.net.mysql.Reply323Packet; /** * MySQL 验证处理器 * * @author mycat */ public class MySQLConnectionAuthenticator implements NIOHandler { private static final Logger LOGGER = LoggerFactory .getLogger(MySQLConnectionAuthenticator.class); private final MySQLConnection source; private final ResponseHandler listener; public MySQLConnectionAuthenticator(MySQLConnection source, ResponseHandler listener) { this.source = source; this.listener = listener; } public void connectionError(MySQLConnection source, Throwable e) { listener.connectionError(e, source); } @Override public void handle(byte[] data) { try { switch (data[4]) { case OkPacket.FIELD_COUNT: HandshakePacket packet = source.getHandshake(); if (packet == null) { processHandShakePacket(data); // 发送认证数据包 source.authenticate(); break; } // 处理认证结果 source.setHandler(new MySQLConnectionHandler(source)); source.setAuthenticated(true); boolean clientCompress = Capabilities.CLIENT_COMPRESS==(Capabilities.CLIENT_COMPRESS & packet.serverCapabilities); boolean usingCompress= MycatServer.getInstance().getConfig().getSystem().getUseCompression()==1 ; if(clientCompress&&usingCompress) { source.setSupportCompress(true); } if (listener != null) { listener.connectionAcquired(source); } break; case ErrorPacket.FIELD_COUNT: ErrorPacket err = new ErrorPacket(); err.read(data); String errMsg = new String(err.message); LOGGER.warn("can't connect to mysql server ,errmsg:"+errMsg+" "+source); //source.close(errMsg); throw new ConnectionException(err.errno, errMsg); case EOFPacket.FIELD_COUNT: auth323(data[3]); break; default: packet = source.getHandshake(); if (packet == null) { processHandShakePacket(data); // 发送认证数据包 source.authenticate(); break; } else { throw new RuntimeException("Unknown Packet!"); } } } catch (RuntimeException e) { if (listener != null) { listener.connectionError(e, source); return; } throw e; } } private void processHandShakePacket(byte[] data) { // 设置握手数据包 HandshakePacket packet= new HandshakePacket(); packet.read(data); source.setHandshake(packet); source.setThreadId(packet.threadId); // 设置字符集编码 int charsetIndex = (packet.serverCharsetIndex & 0xff); String charset = CharsetUtil.getCharset(charsetIndex); if (charset != null) { source.setCharset(charset); } else { throw new RuntimeException("Unknown charsetIndex:" + charsetIndex); } } private void auth323(byte packetId) { // 发送323响应认证数据包 Reply323Packet r323 = new Reply323Packet(); r323.packetId = ++packetId; String pass = source.getPassword(); if (pass != null && pass.length() > 0) { byte[] seed = source.getHandshake().seed; r323.seed = SecurityUtil.scramble323(pass, new String(seed)) .getBytes(); } r323.write(source); } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/MySQLConnectionFactory.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql.nio; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.channels.AsynchronousSocketChannel; import java.nio.channels.CompletionHandler; import java.nio.channels.NetworkChannel; import io.mycat.MycatServer; import io.mycat.backend.mysql.nio.handler.ResponseHandler; import io.mycat.config.model.DBHostConfig; import io.mycat.net.NIOConnector; import io.mycat.net.factory.BackendConnectionFactory; /** * @author mycat */ public class MySQLConnectionFactory extends BackendConnectionFactory { @SuppressWarnings({ "unchecked", "rawtypes" }) public MySQLConnection make(MySQLDataSource pool, ResponseHandler handler, String schema) throws IOException { DBHostConfig dsc = pool.getConfig(); NetworkChannel channel = openSocketChannel(MycatServer.getInstance() .isAIO()); MySQLConnection c = new MySQLConnection(channel, pool.isReadNode()); MycatServer.getInstance().getConfig().setSocketParams(c, false); c.setHost(dsc.getIp()); c.setPort(dsc.getPort()); c.setUser(dsc.getUser()); c.setPassword(dsc.getPassword()); c.setSchema(schema); c.setHandler(new MySQLConnectionAuthenticator(c, handler)); c.setPool(pool); c.setIdleTimeout(pool.getConfig().getIdleTimeout()); if (channel instanceof AsynchronousSocketChannel) { ((AsynchronousSocketChannel) channel).connect( new InetSocketAddress(dsc.getIp(), dsc.getPort()), c, (CompletionHandler) MycatServer.getInstance() .getConnector()); } else { ((NIOConnector) MycatServer.getInstance().getConnector()) .postConnect(c); } return c; } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/MySQLConnectionHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql.nio; import java.util.ArrayList; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.backend.mysql.ByteUtil; import io.mycat.backend.mysql.nio.handler.LoadDataResponseHandler; import io.mycat.backend.mysql.nio.handler.PrepareRequestHandler; import io.mycat.backend.mysql.nio.handler.ResponseHandler; import io.mycat.net.handler.BackendAsyncHandler; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.ErrorPacket; import io.mycat.net.mysql.OkPacket; import io.mycat.net.mysql.PreparedOkPacket; import io.mycat.net.mysql.RequestFilePacket; import io.mycat.route.RouteResultsetNode; /** * life cycle: from connection establish to close
* * @author mycat */ public class MySQLConnectionHandler extends BackendAsyncHandler { private static final Logger logger = LoggerFactory .getLogger(MySQLConnectionHandler.class); private static final int RESULT_STATUS_INIT = 0; private static final int RESULT_STATUS_HEADER = 1; private static final int RESULT_STATUS_FIELD_EOF = 2; private static final int RESULT_STATUS_PREPARE_RESPONSE_OK = 3; private final MySQLConnection source; private volatile int resultStatus; private volatile byte[] header; private volatile List fields; private static final Logger LOGGER = LoggerFactory.getLogger(MySQLConnectionHandler.class); /** * life cycle: one SQL execution */ private volatile ResponseHandler responseHandler; public MySQLConnectionHandler(MySQLConnection source) { this.source = source; this.resultStatus = RESULT_STATUS_INIT; } public void connectionError(Throwable e) { if (responseHandler != null) { responseHandler.connectionError(e, source); } LOGGER.error("connectionError but not handle"); } public MySQLConnection getSource() { return source; } @Override public void handle(byte[] data) { offerData(data, source.getProcessor().getExecutor()); } @Override protected void offerDataError() { resultStatus = RESULT_STATUS_INIT; throw new RuntimeException("offer data error!"); } @Override protected void handleData(byte[] data) { switch (resultStatus) { case RESULT_STATUS_INIT: switch (data[4]) { case OkPacket.FIELD_COUNT: int packetSize = (int) ByteUtil.readLength(data, 0); // COM_STMT_PREPARE_OK状态标识 [00]，和OK包一致，区别是COM_STMT_PREPARE_OK固定长度为12 if (packetSize == PreparedOkPacket.PACKET_SIZE && (responseHandler != null && responseHandler instanceof PrepareRequestHandler)) { handlePrepareOkPacket(data); resultStatus = RESULT_STATUS_PREPARE_RESPONSE_OK; } else { handleOkPacket(data); } break; case ErrorPacket.FIELD_COUNT: handleErrorPacket(data); break; case RequestFilePacket.FIELD_COUNT: handleRequestPacket(data); break; default: resultStatus = RESULT_STATUS_HEADER; header = data; fields = new ArrayList((int) ByteUtil.readLength(data, 4)); } break; case RESULT_STATUS_HEADER: switch (data[4]) { case ErrorPacket.FIELD_COUNT: resultStatus = RESULT_STATUS_INIT; handleErrorPacket(data); break; case EOFPacket.FIELD_COUNT: resultStatus = RESULT_STATUS_FIELD_EOF; handleFieldEofPacket(data); break; default: fields.add(data); } break; case RESULT_STATUS_FIELD_EOF: switch (data[4]) { case ErrorPacket.FIELD_COUNT: resultStatus = RESULT_STATUS_INIT; handleErrorPacket(data); break; case EOFPacket.FIELD_COUNT: resultStatus = RESULT_STATUS_INIT; handleRowEofPacket(data); break; default: handleRowPacket(data); } break; case RESULT_STATUS_PREPARE_RESPONSE_OK: handlePrepareOkPacket(data); break; default: throw new RuntimeException("unknown status!"); } } public void setResponseHandler(ResponseHandler responseHandler) { // logger.info("set response handler "+responseHandler); // if (this.responseHandler != null && responseHandler != null) { // throw new RuntimeException("reset agani!"); // } // 连接释放后如果结果状态不正确则尝试设置一下 if( responseHandler==null && resultStatus!=RESULT_STATUS_INIT ) { logger.warn("try to reset resultStatus. last responseHandler:{}, resultStatus = {}", this.responseHandler, resultStatus); resultStatus = RESULT_STATUS_INIT; } this.responseHandler = responseHandler; } private void handleLogNodeInfo(String pkgName) { Object att = source.getAttachment(); if(LOGGER.isDebugEnabled() && att!=null && att instanceof RouteResultsetNode){ RouteResultsetNode rrn = (RouteResultsetNode) att; String sql = rrn.getStatement(); if(sql!=null){ sql = sql.replaceAll("[\r\n]+", ""); } LOGGER.debug("{} MySQLConnection@{} [id={}] for node={}, sql={}", new Object[]{pkgName, source.hashCode(), source.getId(), rrn.getName(), sql}); } } /** * prepare response OK数据包处理 */ private void handlePrepareOkPacket(byte[] data) { ResponseHandler respHand = responseHandler; if (respHand != null) { respHand.okResponse(data, source); if (((PrepareRequestHandler) respHand).isLastPacket()) { this.resultStatus = RESULT_STATUS_INIT; } } else { LOGGER.error("receive OkPacket but not handle"); source.close("receive OkPacket but not handle"); } } /** * OK数据包处理 */ private void handleOkPacket(byte[] data) { ResponseHandler respHand = responseHandler; if (respHand != null) { handleLogNodeInfo("handleOkPacket"); respHand.okResponse(data, source); }else { LOGGER.error("receive OkPacket but not handle"); } } /** * ERROR数据包处理 */ private void handleErrorPacket(byte[] data) { ResponseHandler respHand = responseHandler; if (respHand != null) { handleLogNodeInfo("handleErrorPacket"); respHand.errorResponse(data, source); } else { LOGGER.error("receive ErrorPacket but no handler"); closeNoHandler(); } } /** * load data file 请求文件数据包处理 */ private void handleRequestPacket(byte[] data) { ResponseHandler respHand = responseHandler; if (respHand != null && respHand instanceof LoadDataResponseHandler) { ((LoadDataResponseHandler) respHand).requestDataResponse(data, source); } else { LOGGER.error("receive RequestPacket but no handler"); closeNoHandler(); } } /** * 字段数据包结束处理 */ private void handleFieldEofPacket(byte[] data) { ResponseHandler respHand = responseHandler; if (respHand != null) { handleLogNodeInfo("handleFieldEofPacket"); respHand.fieldEofResponse(header, fields, data, source); } else { LOGGER.error("receive FieldEofPacket but no handler"); closeNoHandler(); } } /** * 行数据包处理 */ private void handleRowPacket(byte[] data) { ResponseHandler respHand = responseHandler; if (respHand != null) { respHand.rowResponse(data, source); } else { LOGGER.error("receive RowPacket but no handler"); closeNoHandler(); } } private void closeNoHandler() { if (!source.isClosedOrQuit()) { source.close("no handler"); logger.warn("no handler bind in this con " + this + " client:" + source); } } /** * 行数据包结束处理 */ private void handleRowEofPacket(byte[] data) { if (responseHandler != null) { handleLogNodeInfo("handleRowEofPacket"); responseHandler.rowEofResponse(data, source); } else { LOGGER.error("receive RowEofPacket but no handler"); closeNoHandler(); } } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/MySQLDataSource.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql.nio; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.Socket; import java.security.NoSuchAlgorithmException; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.heartbeat.DBHeartbeat; import io.mycat.backend.heartbeat.MySQLHeartbeat; import io.mycat.backend.mysql.SecurityUtil; import io.mycat.backend.mysql.nio.handler.ResponseHandler; import io.mycat.config.Capabilities; import io.mycat.config.model.DBHostConfig; import io.mycat.config.model.DataHostConfig; import io.mycat.net.mysql.AuthPacket; import io.mycat.net.mysql.BinaryPacket; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.ErrorPacket; import io.mycat.net.mysql.HandshakePacket; import io.mycat.net.mysql.OkPacket; import io.mycat.net.mysql.QuitPacket; import io.mycat.net.mysql.Reply323Packet; /** * @author mycat */ public class MySQLDataSource extends PhysicalDatasource { private final MySQLConnectionFactory factory; public MySQLDataSource(DBHostConfig config, DataHostConfig hostConfig, boolean isReadNode) { super(config, hostConfig, isReadNode); this.factory = new MySQLConnectionFactory(); } @Override public void createNewConnection(ResponseHandler handler,String schema) throws IOException { factory.make(this, handler,schema); } private long getClientFlags() { int flag = 0; flag |= Capabilities.CLIENT_LONG_PASSWORD; flag |= Capabilities.CLIENT_FOUND_ROWS; flag |= Capabilities.CLIENT_LONG_FLAG; flag |= Capabilities.CLIENT_CONNECT_WITH_DB; // flag |= Capabilities.CLIENT_NO_SCHEMA; // flag |= Capabilities.CLIENT_COMPRESS; flag |= Capabilities.CLIENT_ODBC; // flag |= Capabilities.CLIENT_LOCAL_FILES; flag |= Capabilities.CLIENT_IGNORE_SPACE; flag |= Capabilities.CLIENT_PROTOCOL_41; flag |= Capabilities.CLIENT_INTERACTIVE; // flag |= Capabilities.CLIENT_SSL; flag |= Capabilities.CLIENT_IGNORE_SIGPIPE; flag |= Capabilities.CLIENT_TRANSACTIONS; // flag |= Capabilities.CLIENT_RESERVED; flag |= Capabilities.CLIENT_SECURE_CONNECTION; // client extension // flag |= Capabilities.CLIENT_MULTI_STATEMENTS; // flag |= Capabilities.CLIENT_MULTI_RESULTS; return flag; } private byte[] passwd(String pass, HandshakePacket hs) throws NoSuchAlgorithmException { if (pass == null || pass.length() == 0) { return null; } byte[] passwd = pass.getBytes(); int sl1 = hs.seed.length; int sl2 = hs.restOfScrambleBuff.length; byte[] seed = new byte[sl1 + sl2]; System.arraycopy(hs.seed, 0, seed, 0, sl1); System.arraycopy(hs.restOfScrambleBuff, 0, seed, sl1, sl2); return SecurityUtil.scramble411(passwd, seed); } @Override public boolean testConnection(String schema) throws IOException { boolean isConnected = true; Socket socket = null; InputStream in = null; OutputStream out = null; try { socket = new Socket(this.getConfig().getIp(), this.getConfig().getPort()); socket.setSoTimeout(1000 * 20); socket.setReceiveBufferSize( 32768 ); socket.setSendBufferSize( 32768 ); socket.setTcpNoDelay(true); socket.setKeepAlive(true); in = new BufferedInputStream(socket.getInputStream(), 32768); out = new BufferedOutputStream( socket.getOutputStream(), 32768 ); /** * Phase 1: MySQL to client. Send handshake packet. */ BinaryPacket bin1 = new BinaryPacket(); bin1.read(in); HandshakePacket handshake = new HandshakePacket(); handshake.read( bin1 ); /** * Phase 2: client to MySQL. Send auth packet. */ AuthPacket authPacket = new AuthPacket(); authPacket.packetId = 1; authPacket.clientFlags = getClientFlags(); authPacket.maxPacketSize = 1024 * 1024 * 16; authPacket.charsetIndex = handshake.serverCharsetIndex & 0xff; authPacket.user = this.getConfig().getUser();; try { authPacket.password = passwd(this.getConfig().getPassword(), handshake); } catch (NoSuchAlgorithmException e) { throw new RuntimeException(e.getMessage()); } authPacket.database = schema; authPacket.write(out); out.flush(); /** * Phase 3: MySQL to client. send OK/ERROR packet. */ BinaryPacket bin2 = new BinaryPacket(); bin2.read(in); switch (bin2.data[0]) { case OkPacket.FIELD_COUNT: break; case ErrorPacket.FIELD_COUNT: ErrorPacket err = new ErrorPacket(); err.read(bin2); isConnected = false; case EOFPacket.FIELD_COUNT: // 发送323响应认证数据包 Reply323Packet r323 = new Reply323Packet(); r323.packetId = ++bin2.packetId; String passwd = this.getConfig().getPassword(); if (passwd != null && passwd.length() > 0) { r323.seed = SecurityUtil.scramble323(passwd, new String(handshake.seed)).getBytes(); } r323.write(out); out.flush(); break; } } catch (IOException e) { isConnected = false; } finally { try { if (in != null) { in.close(); } } catch (IOException e) {} try { if (out != null) { out.write(QuitPacket.QUIT); out.flush(); out.close(); } } catch (IOException e) {} try { if (socket != null) socket.close(); } catch (IOException e) {} } return isConnected; } @Override public DBHeartbeat createHeartBeat() { return new MySQLHeartbeat(this); } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/CommitNodeHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql.nio.handler; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.backend.BackendConnection; import io.mycat.backend.mysql.nio.MySQLConnection; import io.mycat.backend.mysql.xa.TxState; import io.mycat.config.ErrorCode; import io.mycat.net.mysql.ErrorPacket; import io.mycat.route.RouteResultsetNode; import io.mycat.server.NonBlockingSession; /** * @author mycat */ public class CommitNodeHandler extends MultiNodeHandler implements ResponseHandler { private static final Logger LOGGER = LoggerFactory.getLogger(CommitNodeHandler.class); private final NonBlockingSession session; protected byte[] responseData; public CommitNodeHandler(NonBlockingSession session) { super(session); this.session = session; } public CommitNodeHandler(NonBlockingSession session, byte[] responseData) { super(session); this.session = session; this.responseData = responseData; } public void commit() { final int initCount = session.getTargetCount(); lock.lock(); try { reset(initCount); } finally { lock.unlock(); } for (RouteResultsetNode rrn : session.getTargetKeys()) { final BackendConnection conn = session.getTarget(rrn); commit(conn); } } public void commit(BackendConnection conn) { conn.setResponseHandler(CommitNodeHandler.this); boolean isClosed=conn.isClosedOrQuit(); if(isClosed) { session.getSource().writeErrMessage(ErrorCode.ER_UNKNOWN_ERROR, "receive commit,but find backend con is closed or quit"); LOGGER.error( conn+"receive commit,but fond backend con is closed or quit"); } if(conn instanceof MySQLConnection) { MySQLConnection mysqlCon = (MySQLConnection) conn; if (mysqlCon.getXaStatus() == 1) { String xaTxId = session.getXaTXID()+",'"+mysqlCon.getSchema()+"'"; String[] cmds = new String[]{"XA END " + xaTxId, "XA PREPARE " + xaTxId}; mysqlCon.execBatchCmd(cmds); } else { conn.commit(); } }else { conn.commit(); } } @Override public void connectionAcquired(BackendConnection conn) { LOGGER.error("unexpected invocation: connectionAcquired from commit"); } @Override public void okResponse(byte[] ok, BackendConnection conn) { if(conn instanceof MySQLConnection) { MySQLConnection mysqlCon = (MySQLConnection) conn; switch (mysqlCon.getXaStatus()) { case TxState.TX_STARTED_STATE: if (mysqlCon.batchCmdFinished()) { String xaTxId = session.getXaTXID()+",'"+mysqlCon.getSchema()+"'"; mysqlCon.execCmd("XA COMMIT " + xaTxId); mysqlCon.setXaStatus(TxState.TX_PREPARED_STATE); } return; case TxState.TX_PREPARED_STATE: { mysqlCon.setXaStatus(TxState.TX_INITIALIZE_STATE); break; } default: // LOGGER.error("Wrong XA status flag!"); } /* 1. 事务提交后,xa 事务结束 */ if(TxState.TX_INITIALIZE_STATE==mysqlCon.getXaStatus()){ if(session.getXaTXID()!=null){ session.setXATXEnabled(false); } } } /* 2. preAcStates 为true,事务结束后,需要设置为true。preAcStates 为ac上一个状态 */ if(session.getSource().isPreAcStates()&&!session.getSource().isAutocommit()){ session.getSource().setAutocommit(true); } // session.clearResources(false); // ServerConnection source = session.getSource(); // source.write(ok); if (decrementCountBy(1)) { if (responseData != null) { cleanAndFeedback(responseData); } else { cleanAndFeedback(ok); } } } @Override public void errorResponse(byte[] err, BackendConnection conn) { ErrorPacket errPkg = new ErrorPacket(); errPkg.read(err); String errInfo = new String(errPkg.message); session.getSource().setTxInterrupt(errInfo); // errPkg.write(session.getSource()); if (decrementCountBy(1)) { cleanAndFeedback(errPkg.writeToBytes()); } } @Override public void rowEofResponse(byte[] eof, BackendConnection conn) { LOGGER.error(new StringBuilder().append("unexpected packet for ") .append(conn).append(" bound by ").append(session.getSource()) .append(": field's eof").toString()); } @Override public void fieldEofResponse(byte[] header, List fields, byte[] eof, BackendConnection conn) { LOGGER.error(new StringBuilder().append("unexpected packet for ") .append(conn).append(" bound by ").append(session.getSource()) .append(": field's eof").toString()); } @Override public void rowResponse(byte[] row, BackendConnection conn) { LOGGER.warn(new StringBuilder().append("unexpected packet for ") .append(conn).append(" bound by ").append(session.getSource()) .append(": row data packet").toString()); } @Override public void writeQueueAvailable() { } @Override public void connectionError(Throwable e, BackendConnection conn) { if (decrementCountBy(1)) { cleanAndFeedback(createErrPkg(e.getMessage()).writeToBytes()); } } @Override public void connectionClose(BackendConnection conn, String reason) { if (decrementCountBy(1)) { cleanAndFeedback(createErrPkg(reason).writeToBytes()); } } private void cleanAndFeedback(byte[] responseData) { // clear all resources session.clearResources(false); if (session.closed()) { return; } if (this.isFail()) { createErrPkg(error).write(session.getSource()); } else { session.getSource().write(responseData); } } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/ConnectionHeartBeatHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql.nio.handler; import java.util.Collection; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.backend.BackendConnection; import io.mycat.net.mysql.ErrorPacket; /** * heartbeat check for mysql connections * * @author wuzhih * */ public class ConnectionHeartBeatHandler implements ResponseHandler { private static final Logger LOGGER = LoggerFactory .getLogger(ConnectionHeartBeatHandler.class); protected final ReentrantLock lock = new ReentrantLock(); private final ConcurrentHashMap allCons = new ConcurrentHashMap(); public void doHeartBeat(BackendConnection conn, String sql) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("do heartbeat for con " + conn); } try { HeartBeatCon hbCon = new HeartBeatCon(conn); boolean notExist = (allCons.putIfAbsent(hbCon.conn.getId(), hbCon) == null); if (notExist) { conn.setResponseHandler(this); conn.query(sql); } } catch (Exception e) { executeException(conn, e); } } /** * remove timeout connections */ public void abandTimeOuttedConns() { if (allCons.isEmpty()) { return; } Collection abandCons = new LinkedList(); long curTime = System.currentTimeMillis(); Iterator> itors = allCons.entrySet() .iterator(); while (itors.hasNext()) { HeartBeatCon hbCon = itors.next().getValue(); if (hbCon.timeOutTimestamp < curTime) { abandCons.add(hbCon.conn); itors.remove(); } } if (!abandCons.isEmpty()) { for (BackendConnection con : abandCons) { try { // if(con.isBorrowed()) con.close("heartbeat timeout "); } catch (Exception e) { LOGGER.warn("close err:" + e); } } } } @Override public void connectionAcquired(BackendConnection conn) { // not called } @Override public void connectionError(Throwable e, BackendConnection conn) { // not called } @Override public void errorResponse(byte[] data, BackendConnection conn) { removeFinished(conn); ErrorPacket err = new ErrorPacket(); err.read(data); LOGGER.warn("errorResponse " + err.errno + " " + new String(err.message)); conn.release(); } @Override public void okResponse(byte[] ok, BackendConnection conn) { boolean executeResponse = conn.syncAndExcute(); if (executeResponse) { removeFinished(conn); conn.release(); } } @Override public void rowResponse(byte[] row, BackendConnection conn) { } @Override public void rowEofResponse(byte[] eof, BackendConnection conn) { removeFinished(conn); conn.release(); } private void executeException(BackendConnection c, Throwable e) { removeFinished(c); LOGGER.warn("executeException ", e); c.close("heatbeat exception:" + e); } private void removeFinished(BackendConnection con) { Long id = ((BackendConnection) con).getId(); this.allCons.remove(id); } @Override public void writeQueueAvailable() { } @Override public void connectionClose(BackendConnection conn, String reason) { removeFinished(conn); LOGGER.warn("connection closed " + conn + " reason:" + reason); } @Override public void fieldEofResponse(byte[] header, List fields, byte[] eof, BackendConnection conn) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("received field eof from " + conn); } } } class HeartBeatCon { public final long timeOutTimestamp; public final BackendConnection conn; public HeartBeatCon(BackendConnection conn) { super(); this.timeOutTimestamp = System.currentTimeMillis() + 20 * 1000L; this.conn = conn; } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/DelegateResponseHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql.nio.handler; import java.util.List; import io.mycat.backend.BackendConnection; /** * @author mycat */ public class DelegateResponseHandler implements ResponseHandler { private final ResponseHandler target; public DelegateResponseHandler(ResponseHandler target) { if (target == null) { throw new IllegalArgumentException("delegate is null!"); } this.target = target; } @Override public void connectionAcquired(BackendConnection conn) { target.connectionAcquired(conn); } @Override public void connectionError(Throwable e, BackendConnection conn) { target.connectionError(e, conn); } @Override public void okResponse(byte[] ok, BackendConnection conn) { target.okResponse(ok, conn); } @Override public void errorResponse(byte[] err, BackendConnection conn) { target.errorResponse(err, conn); } @Override public void fieldEofResponse(byte[] header, List fields, byte[] eof, BackendConnection conn) { target.fieldEofResponse(header, fields, eof, conn); } @Override public void rowResponse(byte[] row, BackendConnection conn) { target.rowResponse(row, conn); } @Override public void rowEofResponse(byte[] eof, BackendConnection conn) { target.rowEofResponse(eof, conn); } @Override public void writeQueueAvailable() { target.writeQueueAvailable(); } @Override public void connectionClose(BackendConnection conn, String reason) { target.connectionClose(conn, reason); } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/FetchStoreNodeOfChildTableHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql.nio.handler; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.backend.BackendConnection; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.cache.CachePool; import io.mycat.config.MycatConfig; import io.mycat.net.mysql.ErrorPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.route.RouteResultsetNode; import io.mycat.server.ServerConnection; import io.mycat.server.parser.ServerParse; /** * company where id=(select company_id from customer where id=3); the one which * return data (id) is the datanode to store child table's records * * @author wuzhih * */ public class FetchStoreNodeOfChildTableHandler implements ResponseHandler { private static final Logger LOGGER = LoggerFactory .getLogger(FetchStoreNodeOfChildTableHandler.class); private String sql; private volatile String result; private volatile String dataNode; private AtomicInteger finished = new AtomicInteger(0); protected final ReentrantLock lock = new ReentrantLock(); private volatile ServerConnection sc; public String execute(String schema, String sql, List dataNodes, ServerConnection sc) { String key = schema + ":" + sql; CachePool cache = MycatServer.getInstance().getCacheService() .getCachePool("ER_SQL2PARENTID"); String result = (String) cache.get(key); if (result != null) { return result; } this.sql = sql; int totalCount = dataNodes.size(); long startTime = System.currentTimeMillis(); long endTime = startTime + 5 * 60 * 1000L; MycatConfig conf = MycatServer.getInstance().getConfig(); this.sc = sc; LOGGER.debug("find child node with sql:" + sql); for (String dn : dataNodes) { if (dataNode != null) { return dataNode; } PhysicalDBNode mysqlDN = conf.getDataNodes().get(dn); try { if (LOGGER.isDebugEnabled()) { LOGGER.debug(sc + "execute in datanode " + dn); } RouteResultsetNode node = new RouteResultsetNode(dn, ServerParse.SELECT, sql); node.setRunOnSlave(false); // 获取 子表节点，最好走master为好 /* * fix #1370 默认应该先从已经持有的连接中取连接, 否则可能因为事务隔离性看不到当前事务内更新的数据 * Tips: 通过mysqlDN.getConnection获取到的连接不是当前连接 * */ BackendConnection conn = sc.getSession2().getTarget(node); if(sc.getSession2().tryExistsCon(conn, node)) { _execute(conn, node, sc); } else { mysqlDN.getConnection(mysqlDN.getDatabase(), sc.isAutocommit(), node, this, node); } } catch (Exception e) { LOGGER.warn("get connection err " + e); } } while (dataNode == null && System.currentTimeMillis() < endTime) { try { Thread.sleep(50); } catch (InterruptedException e) { break; } if (dataNode != null || finished.get() >= totalCount) { break; } } if (dataNode != null) { cache.putIfAbsent(key, dataNode); } if(System.currentTimeMillis() > endTime) { LOGGER.error("timeout when executing fetch sql " + sql); } return dataNode; } public String execute(String schema, String sql, ArrayList dataNodes) { String key = schema + ":" + sql; CachePool cache = MycatServer.getInstance().getCacheService() .getCachePool("ER_SQL2PARENTID"); String result = (String) cache.get(key); if (result != null) { return result; } this.sql = sql; int totalCount = dataNodes.size(); long startTime = System.currentTimeMillis(); long endTime = startTime + 5 * 60 * 1000L; MycatConfig conf = MycatServer.getInstance().getConfig(); LOGGER.debug("find child node with sql:" + sql); for (String dn : dataNodes) { if (dataNode != null) { return dataNode; } PhysicalDBNode mysqlDN = conf.getDataNodes().get(dn); try { if (LOGGER.isDebugEnabled()) { LOGGER.debug("execute in datanode " + dn); } RouteResultsetNode node = new RouteResultsetNode(dn, ServerParse.SELECT, sql); node.setRunOnSlave(false); // 获取 子表节点，最好走master为好 mysqlDN.getConnection(mysqlDN.getDatabase(), true, node, this, node); // mysqlDN.getConnection(mysqlDN.getDatabase(), true, // new RouteResultsetNode(dn, ServerParse.SELECT, sql), // this, dn); } catch (Exception e) { LOGGER.warn("get connection err " + e); } try { Thread.sleep(200); } catch (InterruptedException e) { } } while (dataNode == null && System.currentTimeMillis() < endTime) { try { Thread.sleep(50); } catch (InterruptedException e) { break; } if (dataNode != null || finished.get() >= totalCount) { break; } } if (dataNode != null) { cache.putIfAbsent(key, dataNode); } return dataNode; } private void _execute(BackendConnection conn, RouteResultsetNode node, ServerConnection sc) { conn.setResponseHandler(this); try { conn.execute(node, sc, sc.isAutocommit()); } catch (IOException e) { connectionError(e, conn); } } @Override public void connectionAcquired(BackendConnection conn) { conn.setResponseHandler(this); try { conn.query(sql); } catch (Exception e) { executeException(conn, e); } } @Override public void connectionError(Throwable e, BackendConnection conn) { finished.incrementAndGet(); LOGGER.warn("connectionError " + e); } @Override public void errorResponse(byte[] data, BackendConnection conn) { finished.incrementAndGet(); ErrorPacket err = new ErrorPacket(); err.read(data); LOGGER.warn("errorResponse " + err.errno + " " + new String(err.message)); releaseConnection(conn); LOGGER.warn(this.sc + " connection release " + conn + " errorResponse" ); } @Override public void okResponse(byte[] ok, BackendConnection conn) { boolean executeResponse = conn.syncAndExcute(); if (executeResponse) { finished.incrementAndGet(); //conn.release(); releaseConnection(conn); } } @Override public void rowResponse(byte[] row, BackendConnection conn) { if (LOGGER.isDebugEnabled()) { LOGGER.debug(this.sc + "received rowResponse response," + getColumn(row) + " from " + conn); } if (result == null) { result = getColumn(row); dataNode = ((RouteResultsetNode) conn.getAttachment()).getName(); } else { LOGGER.warn("find multi data nodes for child table store, sql is: " + sql); } } private String getColumn(byte[] row) { RowDataPacket rowDataPkg = new RowDataPacket(1); rowDataPkg.read(row); byte[] columnData = rowDataPkg.fieldValues.get(0); return new String(columnData); } @Override public void rowEofResponse(byte[] eof, BackendConnection conn) { finished.incrementAndGet(); //conn.release(); releaseConnection(conn); } private void executeException(BackendConnection c, Throwable e) { finished.incrementAndGet(); LOGGER.warn("executeException " + e); c.close("exception:" + e); } @Override public void writeQueueAvailable() { } @Override public void connectionClose(BackendConnection conn, String reason) { finished.incrementAndGet(); LOGGER.warn("connection closed " + conn + " reason:" + reason); } @Override public void fieldEofResponse(byte[] header, List fields, byte[] eof, BackendConnection conn) { } private void releaseConnection(BackendConnection conn) { if(this.sc != null ) { Map target = sc.getSession2().getTargetMap(); for(BackendConnection backConn :target.values()) { if(backConn != null && backConn.equals(conn)) { return ; } } // if(sc.getSession2().tryExistsCon(conn, node)) { // _execute(conn, node, sc); // } else { // mysqlDN.getConnection(mysqlDN.getDatabase(), sc.isAutocommit(), node, this, node); // } } conn.release(); } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/GetConnectionHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql.nio.handler; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicInteger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.backend.BackendConnection; /** * wuzh * * @author mycat * */ public class GetConnectionHandler implements ResponseHandler { private final CopyOnWriteArrayList successCons; private static final Logger logger = LoggerFactory .getLogger(GetConnectionHandler.class); private final AtomicInteger finishedCount = new AtomicInteger(0); private final int total; public GetConnectionHandler( CopyOnWriteArrayList connsToStore, int totalNumber) { super(); this.successCons = connsToStore; this.total = totalNumber; } public String getStatusInfo() { return "finished "+ finishedCount.get()+" success "+successCons.size()+" target count:"+this.total; } public boolean finished() { return finishedCount.get() >= total; } @Override public void connectionAcquired(BackendConnection conn) { successCons.add(conn); finishedCount.addAndGet(1); logger.info("connected successfuly " + conn); conn.release(); } @Override public void connectionError(Throwable e, BackendConnection conn) { finishedCount.addAndGet(1); logger.warn("connect error " + conn+ e); conn.release(); } @Override public void errorResponse(byte[] err, BackendConnection conn) { logger.warn("caught error resp: " + conn + " " + new String(err)); conn.release(); } @Override public void okResponse(byte[] ok, BackendConnection conn) { logger.info("received ok resp: " + conn + " " + new String(ok)); } @Override public void fieldEofResponse(byte[] header, List fields, byte[] eof, BackendConnection conn) { } @Override public void rowResponse(byte[] row, BackendConnection conn) { } @Override public void rowEofResponse(byte[] eof, BackendConnection conn) { } @Override public void writeQueueAvailable() { } @Override public void connectionClose(BackendConnection conn, String reason) { } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/JDBCFetchStoreNodeOfChildTableHandler.java ================================================ package io.mycat.backend.mysql.nio.handler; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.jdbc.JDBCDatasource; import io.mycat.cache.CachePool; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Map; public class JDBCFetchStoreNodeOfChildTableHandler { private static final Logger LOGGER = LoggerFactory .getLogger(JDBCFetchStoreNodeOfChildTableHandler.class); public String execute(String schema, String sql, ArrayList dataNodes) { String key = schema + ":" + sql; CachePool cache = MycatServer.getInstance().getCacheService() .getCachePool("ER_SQL2PARENTID"); String result = (String) cache.get(key); if (result != null) { return result; } Map dbNodeMap = MycatServer.getInstance().getConfig().getDataNodes(); if (LOGGER.isDebugEnabled()) { LOGGER.debug("find child node with sql:" + sql); } for (String dn : dataNodes) { PhysicalDBNode physicalDBNode = dbNodeMap.get(dn); PhysicalDatasource physicalDatasource = physicalDBNode.getDbPool().getSource(); if(physicalDatasource instanceof JDBCDatasource) { JDBCDatasource jdbcDatasource = (JDBCDatasource) physicalDatasource; Connection con = null; PreparedStatement pstmt = null; ResultSet rs = null; try { con = jdbcDatasource.getDruidConnection(); String useDB = "use `" + physicalDBNode.getDatabase() + "`;"; pstmt = con.prepareStatement(useDB); pstmt.execute(); pstmt = con.prepareStatement(sql); rs = pstmt.executeQuery(); if (rs.next()) { return dn; } } catch (SQLException e) { e.printStackTrace(); } finally { try { if (con != null) { con.close(); } if (pstmt != null) { pstmt.close(); } if (rs != null) { rs.close(); } } catch (SQLException e) { e.printStackTrace(); } } } } LOGGER.error("can't find (root) parent sharding node for sql:"+ sql); return null; } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/KillConnectionHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql.nio.handler; import java.io.UnsupportedEncodingException; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.backend.BackendConnection; import io.mycat.backend.mysql.nio.MySQLConnection; import io.mycat.net.mysql.CommandPacket; import io.mycat.net.mysql.ErrorPacket; import io.mycat.net.mysql.MySQLPacket; import io.mycat.server.NonBlockingSession; /** * @author mycat */ public class KillConnectionHandler implements ResponseHandler { private static final Logger LOGGER = LoggerFactory .getLogger(KillConnectionHandler.class); private final MySQLConnection killee; private final NonBlockingSession session; public KillConnectionHandler(BackendConnection killee, NonBlockingSession session) { this.killee = (MySQLConnection) killee; this.session = session; } @Override public void connectionAcquired(BackendConnection conn) { MySQLConnection mysqlCon = (MySQLConnection) conn; conn.setResponseHandler(this); CommandPacket packet = new CommandPacket(); packet.packetId = 0; packet.command = MySQLPacket.COM_QUERY; packet.arg = new StringBuilder("KILL ").append(killee.getThreadId()) .toString().getBytes(); packet.write(mysqlCon); } @Override public void connectionError(Throwable e, BackendConnection conn) { killee.close("exception:" + e.toString()); } @Override public void okResponse(byte[] ok, BackendConnection conn) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("kill connection success connection id:" + killee.getThreadId()); } conn.release(); killee.close("killed"); } @Override public void rowEofResponse(byte[] eof, BackendConnection conn) { LOGGER.warn(new StringBuilder().append("unexpected packet for ") .append(conn).append(" bound by ").append(session.getSource()) .append(": field's eof").toString()); conn.quit(); killee.close("killed"); } @Override public void errorResponse(byte[] data, BackendConnection conn) { ErrorPacket err = new ErrorPacket(); err.read(data); String msg = null; try { msg = new String(err.message, conn.getCharset()); } catch (UnsupportedEncodingException e) { msg = new String(err.message); } LOGGER.warn("kill backend connection " + killee + " failed: " + msg + " con:" + conn); conn.release(); killee.close("exception:" + msg); } @Override public void fieldEofResponse(byte[] header, List fields, byte[] eof, BackendConnection conn) { } @Override public void rowResponse(byte[] row, BackendConnection conn) { } @Override public void writeQueueAvailable() { } @Override public void connectionClose(BackendConnection conn, String reason) { } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/LoadDataResponseHandler.java ================================================ package io.mycat.backend.mysql.nio.handler; import io.mycat.backend.BackendConnection; /** * Created by nange on 2015/3/31. */ public interface LoadDataResponseHandler { /** * 收到请求发送文件数据包的响应处理 */ void requestDataResponse(byte[] row, BackendConnection conn); } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/LockTablesHandler.java ================================================ package io.mycat.backend.mysql.nio.handler; import java.io.IOException; import java.util.List; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.backend.BackendConnection; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.config.MycatConfig; import io.mycat.net.mysql.OkPacket; import io.mycat.route.RouteResultset; import io.mycat.route.RouteResultsetNode; import io.mycat.server.NonBlockingSession; /** * lock tables 语句处理器 * @author songdabin * */ public class LockTablesHandler extends MultiNodeHandler { private static final Logger LOGGER = LoggerFactory.getLogger(LockTablesHandler.class); private final RouteResultset rrs; private final ReentrantLock lock; private final boolean autocommit; public LockTablesHandler(NonBlockingSession session, RouteResultset rrs) { super(session); this.rrs = rrs; this.autocommit = session.getSource().isAutocommit(); this.lock = new ReentrantLock(); } public void execute() throws Exception { super.reset(this.rrs.getNodes().length); MycatConfig conf = MycatServer.getInstance().getConfig(); for (final RouteResultsetNode node : rrs.getNodes()) { BackendConnection conn = session.getTarget(node); if (session.tryExistsCon(conn, node)) { _execute(conn, node); } else { // create new connection PhysicalDBNode dn = conf.getDataNodes().get(node.getName()); dn.getConnection(dn.getDatabase(), autocommit, node, this, node); } } } private void _execute(BackendConnection conn, RouteResultsetNode node) { if (clearIfSessionClosed(session)) { return; } conn.setResponseHandler(this); try { conn.execute(node, session.getSource(), autocommit); } catch (IOException e) { connectionError(e, conn); } } @Override public void connectionAcquired(BackendConnection conn) { final RouteResultsetNode node = (RouteResultsetNode) conn.getAttachment(); session.bindConnection(node, conn); _execute(conn, node); } @Override public void okResponse(byte[] data, BackendConnection conn) { boolean executeResponse = conn.syncAndExcute(); if (executeResponse) { if (clearIfSessionClosed(session)) { return; } boolean isEndPack = decrementCountBy(1); if (isEndPack) { if (this.isFail() || session.closed()) { tryErrorFinished(true); return; } OkPacket ok = new OkPacket(); ok.read(data); lock.lock(); try { ok.packetId = ++ packetId; ok.serverStatus = session.getSource().isAutocommit() ? 2:1; } finally { lock.unlock(); } ok.write(session.getSource()); } } } protected String byte2Str(byte[] data) { StringBuilder sb = new StringBuilder(); for (byte b : data) { sb.append(Byte.toString(b)); } return sb.toString(); } @Override public void fieldEofResponse(byte[] header, List fields, byte[] eof, BackendConnection conn) { LOGGER.error(new StringBuilder().append("unexpected packet for ") .append(conn).append(" bound by ").append(session.getSource()) .append(": field's eof").toString()); } @Override public void rowResponse(byte[] row, BackendConnection conn) { LOGGER.warn(new StringBuilder().append("unexpected packet for ") .append(conn).append(" bound by ").append(session.getSource()) .append(": row data packet").toString()); } @Override public void rowEofResponse(byte[] eof, BackendConnection conn) { LOGGER.error(new StringBuilder().append("unexpected packet for ") .append(conn).append(" bound by ").append(session.getSource()) .append(": row's eof").toString()); } @Override public void writeQueueAvailable() { // TODO Auto-generated method stub } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/MiddlerQueryResultHandler.java ================================================ package io.mycat.backend.mysql.nio.handler; import java.util.ArrayList; import java.util.List; import com.alibaba.druid.sql.ast.expr.SQLCharExpr; import io.mycat.backend.mysql.DataType; /** * 查询中间结果处理器 * @author huangyiming * * @param */ public class MiddlerQueryResultHandler implements MiddlerResultHandler { List reusult = new ArrayList<>(); DataType dataType; Class clazz; private SecondHandler secondHandler; public MiddlerQueryResultHandler(SecondHandler secondHandler) { this.secondHandler = secondHandler; } //确保只有一个构造函数入口 private MiddlerQueryResultHandler(){ } @Override public List getResult() { return reusult; } @Override public void add(T t ) { reusult.add(new SQLCharExpr(t==null?null:t.toString())); } @Override public String getDataType() { return dataType.name(); } @Override public void secondEexcute() { secondHandler.doExecute(getResult()); } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/MiddlerResultHandler.java ================================================ package io.mycat.backend.mysql.nio.handler; import java.util.List; import com.alibaba.druid.sql.ast.expr.SQLCharExpr; /** * 中间结果处理器 * @author huangyiming * * @param */ public interface MiddlerResultHandler { public List getResult(); public void add(T t ); public String getDataType(); public void secondEexcute(); } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/MultiNodeCoordinator.java ================================================ package io.mycat.backend.mysql.nio.handler; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.backend.BackendConnection; import io.mycat.backend.mysql.nio.MySQLConnection; import io.mycat.backend.mysql.xa.CoordinatorLogEntry; import io.mycat.backend.mysql.xa.ParticipantLogEntry; import io.mycat.backend.mysql.xa.TxState; import io.mycat.backend.mysql.xa.recovery.Repository; import io.mycat.backend.mysql.xa.recovery.impl.FileSystemRepository; import io.mycat.backend.mysql.xa.recovery.impl.InMemoryRepository; import io.mycat.config.ErrorCode; import io.mycat.net.mysql.ErrorPacket; import io.mycat.route.RouteResultsetNode; import io.mycat.server.NonBlockingSession; import io.mycat.server.sqlcmd.SQLCtrlCommand; public class MultiNodeCoordinator implements ResponseHandler { private static final Logger LOGGER = LoggerFactory .getLogger(MultiNodeCoordinator.class); public static final Repository fileRepository = new FileSystemRepository(); public static final Repository inMemoryRepository = new InMemoryRepository(); private final AtomicInteger runningCount = new AtomicInteger(0); private final AtomicInteger prepareCount = new AtomicInteger(0); protected volatile int coorXaStatus; private final AtomicInteger faileCount = new AtomicInteger(0); private volatile int nodeCount; private final NonBlockingSession session; private SQLCtrlCommand cmdHandler; private final AtomicBoolean failed = new AtomicBoolean(false); protected volatile String error; public MultiNodeCoordinator(NonBlockingSession session) { this.session = session; } /** Multi-nodes 1pc Commit Handle **/ public void executeBatchNodeCmd(SQLCtrlCommand cmdHandler) { this.cmdHandler = cmdHandler; final int initCount = session.getTargetCount(); runningCount.set(initCount); nodeCount = initCount; failed.set(false); faileCount.set(0); //XA statue init if(prepareCount.get() != 0){ //System.out.println("prepareCount :" + prepareCount.get()); } prepareCount.set(0); if(session.getXaTXID()!=null){ coorXaStatus = TxState.TX_STARTED_STATE; writeRecoverLog(initCount); } else { coorXaStatus = -1; } // 执行 int started = 0; for (RouteResultsetNode rrn : session.getTargetKeys()) { if (rrn == null) { LOGGER.error("null is contained in RoutResultsetNodes, source = " + session.getSource()); continue; } final BackendConnection conn = session.getTarget(rrn); if (conn != null) { conn.setResponseHandler(this); //process the XA_END XA_PREPARE Command if(conn instanceof MySQLConnection){ MySQLConnection mysqlCon = (MySQLConnection) conn; String xaTxId = null; if(session.getXaTXID()!=null){ xaTxId = session.getXaTXID() +",'"+ mysqlCon.getSchema()+"'"; } if (mysqlCon.getXaStatus() == TxState.TX_STARTED_STATE) { //recovery Log // participantLogEntry[started] = new ParticipantLogEntry(xaTxId,conn.getHost(),0,conn.getSchema(),((MySQLConnection) conn).getXaStatus()); String[] cmds = new String[]{"XA END " + xaTxId, "XA PREPARE " + xaTxId}; if (LOGGER.isDebugEnabled()) { LOGGER.debug("Start execute the batch cmd : "+ cmds[0] + ";" + cmds[1]+","+ "current connection:"+conn.getHost()+":"+conn.getPort()); } // prepareCount.incrementAndGet(); mysqlCon.execBatchCmd(cmds); } else { //recovery Log // participantLogEntry[started] = new ParticipantLogEntry(xaTxId,conn.getHost(),0,conn.getSchema(),((MySQLConnection) conn).getXaStatus()); cmdHandler.sendCommand(session, conn); } }else{ cmdHandler.sendCommand(session, conn); } ++started; } } if (started < nodeCount) { runningCount.set(started); LOGGER.warn("some connection failed to execute " + (nodeCount - started)); /** * assumption: only caused by front-end connection close.
* Otherwise, packet must be returned to front-end */ failed.set(true); } } private void writeRecoverLog(int initCount) { // 执行 int started = 0; //recovery nodes log ParticipantLogEntry[] participantLogEntry = new ParticipantLogEntry[initCount]; for (RouteResultsetNode rrn : session.getTargetKeys()) { if (rrn == null) { LOGGER.error("null is contained in RoutResultsetNodes, source = " + session.getSource()); continue; } final BackendConnection conn = session.getTarget(rrn); if (conn != null) { //process the XA_END XA_PREPARE Command if(conn instanceof MySQLConnection){ MySQLConnection mysqlCon = (MySQLConnection) conn; String xaTxId = null; if(session.getXaTXID()!=null){ xaTxId = session.getXaTXID() +",'"+ mysqlCon.getSchema()+"'"; } if (mysqlCon.getXaStatus() == TxState.TX_STARTED_STATE) { //recovery Log participantLogEntry[started] = new ParticipantLogEntry(xaTxId,conn.getHost(),0,conn.getSchema(),((MySQLConnection) conn).getXaStatus()); prepareCount.incrementAndGet(); } } } started++; } //xa recovery log // if(session.getXaTXID()!=null) { CoordinatorLogEntry coordinatorLogEntry = new CoordinatorLogEntry(session.getXaTXID(), false, participantLogEntry); inMemoryRepository.put(session.getXaTXID(), coordinatorLogEntry); fileRepository.writeCheckpoint(session.getXaTXID(), inMemoryRepository.getAllCoordinatorLogEntries()); // } } private boolean finished() { int val = runningCount.decrementAndGet(); return (val == 0); } @Override public void connectionError(Throwable e, BackendConnection conn) { } @Override public void connectionAcquired(BackendConnection conn) { } @Override public void errorResponse(byte[] err, BackendConnection conn) { // faileCount.incrementAndGet(); ErrorPacket errorPacket = new ErrorPacket(); errorPacket.read(err); String msg = new String(errorPacket.message); if (LOGGER.isInfoEnabled()){ LOGGER.info("======================" + "errorResponse from {} msg: {}", conn, new String(errorPacket.message)); } // to do xa prepare failure rollback if(conn instanceof MySQLConnection) { MySQLConnection mysqlCon = (MySQLConnection) conn; if(coorXaStatus == TxState.TX_STARTED_STATE) { this.setFail(msg); // 1 pc prepare failure to rollback LOGGER.error("XA prepare failure in :xa end;xa prepare; XA id is:" +session.getXaTXID() + " errmsg {},current conn:{}", msg ,conn); if(mysqlCon.batchCmdFinished()) { if(prepareCount.decrementAndGet() == 0) { this.tryErrorFinished(true); } return ; } } //replayCommit prepare statue replay commit if(coorXaStatus == TxState.TX_PREPARED_STATE) { // if(mysqlCon.getXaStatus() == TxState.TX_PREPARED_STATE) { String xaTxId = session.getXaTXID(); if (xaTxId != null) { xaTxId += ",'" + mysqlCon.getSchema() + "'"; String cmd = "XA COMMIT " + xaTxId; if (LOGGER.isInfoEnabled()) { LOGGER.info("Replay Commit execute the cmd :" + cmd + ",current host:" + mysqlCon.getHost() + ":" + mysqlCon.getPort()); } mysqlCon.execCmd(cmd); return; } } } //if some commit and some failure , print error info and return failure info and then release back connection //release connection if (this.cmdHandler.releaseConOnErr()) { session.releaseConnection(conn); } else { session.releaseConnectionIfSafe(conn, LOGGER.isDebugEnabled(),false); } // this.setFail(msg); if (this.finished()) { this.tryErrorFinished(true); } } @Override public void okResponse(byte[] ok, BackendConnection conn) { //LOGGER.info("======================" + "okResponse from {} ", conn); //process the XA Transatcion 2pc commit if(conn instanceof MySQLConnection) { MySQLConnection mysqlCon = (MySQLConnection) conn; switch (mysqlCon.getXaStatus()) { case TxState.TX_STARTED_STATE: //if there have many SQL execute wait the okResponse,will come to here one by one //should be wait all nodes ready ,then send xa commit to all nodes. if (mysqlCon.batchCmdFinished()) { String xaTxId = session.getXaTXID(); //recovery log CoordinatorLogEntry coordinatorLogEntry = inMemoryRepository.get(xaTxId); for(int i=0; i 0) { this.tryErrorFinished(true); return ; } coorXaStatus = TxState.TX_PREPARED_STATE; //进入prepare状态 for (RouteResultsetNode rrn : session.getTargetKeys()) { if (rrn == null) { LOGGER.error("null is contained in RoutResultsetNodes, source = " + session.getSource()); continue; } final BackendConnection backConn = session.getTarget(rrn); if (conn != null) { // conn.setResponseHandler(this); //process the XA_END XA_PREPARE Command if(conn instanceof MySQLConnection){ MySQLConnection backMysqlCon = (MySQLConnection) backConn; if(session.getXaTXID()!=null){ xaTxId = session.getXaTXID() +",'"+ backMysqlCon.getSchema()+"'"; } if (backMysqlCon.getXaStatus() == TxState.TX_PREPARED_STATE) { String cmd = "XA COMMIT " + xaTxId ; if (LOGGER.isDebugEnabled()) { LOGGER.debug("Start execute the cmd : "+ cmd+ " current connection:"+conn.getHost()+":"+conn.getPort()); } backMysqlCon.execCmd(cmd); } else { LOGGER.info("{} not in PREPARED_STATE", backMysqlCon); } } } } } } return; case TxState.TX_PREPARED_STATE: { //recovery log String xaTxId = session.getXaTXID(); CoordinatorLogEntry coordinatorLogEntry = inMemoryRepository.get(xaTxId); for(int i=0; i fields, byte[] eof, BackendConnection conn) { } @Override public void rowResponse(byte[] row, BackendConnection conn) { } @Override public void rowEofResponse(byte[] eof, BackendConnection conn) { } @Override public void writeQueueAvailable() { } @Override public void connectionClose(BackendConnection conn, String reason) { // faileCount.incrementAndGet(); if (LOGGER.isInfoEnabled()){ LOGGER.info("======================" + "connectionClose from {} msg: {}", conn, reason); } if(session.getXaTXID() != null) { if(conn instanceof MySQLConnection) { MySQLConnection mysqlCon = (MySQLConnection) conn; if(coorXaStatus == TxState.TX_STARTED_STATE) { this.setFail(reason); // 1 pc prepare failure to rollback LOGGER.error("XA prepare failure in :xa end;xa prepare; XA id is:" +session.getXaTXID() + " errmsg {},current conn:{}", reason ,conn); if(prepareCount.decrementAndGet() == 0) { this.tryErrorFinished(true); } return ; } //replayCommit prepare statue replay commit //todo 重新创建 连接 重新进行提交 conn 是prepare阶段的时候 if(coorXaStatus == TxState.TX_PREPARED_STATE) { String xaTxId = session.getXaTXID(); if (xaTxId != null) { xaTxId += ",'" + mysqlCon.getSchema() + "'"; String cmd = "XA COMMIT " + xaTxId; LOGGER.error("XA Comit failure in :"+cmd+" errmsg {},current conn:{}", reason ,conn); // mysqlCon.execCmd(cmd); //return; } } } } //release connection if (this.cmdHandler.releaseConOnErr()) { session.releaseConnection(conn); } else { session.releaseConnectionIfSafe(conn, LOGGER.isDebugEnabled(),false); } //设置错误信息 this.setFail("close back conn reason:" + reason); if (this.finished()) { this.tryErrorFinished(true); // ErrorPacket errPkg = new ErrorPacket(); // errPkg.errno = 1; // errPkg.message = ("close back conn reason:" + reason).getBytes(); // cmdHandler.errorResponse(session, errPkg.writeToBytes() , this.nodeCount, // this.faileCount.get()); // if (cmdHandler.isAutoClearSessionCons()) { // session.clearResources(session.getSource().isTxInterrupted()); // } } } //设置失败 private void setFail(String err){ this.error = err; faileCount.incrementAndGet(); } protected void tryErrorFinished(boolean allEnd) { if (allEnd && !session.closed()) { // xa error auto rollback if(coorXaStatus == TxState.TX_STARTED_STATE ) { LOGGER.info("{} found failure in xa ,so to rollback ,reason :{}",this, this.error); session.rollback(); return ; } // clear session resources,release all if (LOGGER.isDebugEnabled()) { LOGGER.debug(this.toString()+"error all end ,clear session resource "); } //释放连接 session.clearResources(session.getSource().isTxInterrupted()); //关闭所有的错误后端连接 清理资源 // session.closeAndClearResources(error); ErrorPacket errPkg = new ErrorPacket(); errPkg.packetId= 1; errPkg.errno = ErrorCode.ER_UNKNOWN_ERROR; errPkg.message = (this.error).getBytes(); session.setAutoCommitStatus(); cmdHandler.errorResponse(session, errPkg.writeToBytes(), this.nodeCount, this.faileCount.get()); session.getSource().clearTxInterrupt(); //if (cmdHandler.isAutoClearSessionCons()) { //} } } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/MultiNodeHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql.nio.handler; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.backend.BackendConnection; import io.mycat.config.ErrorCode; import io.mycat.net.mysql.ErrorPacket; import io.mycat.server.NonBlockingSession; import io.mycat.util.StringUtil; /** * @author mycat */ abstract class MultiNodeHandler implements ResponseHandler, Terminatable { private static final Logger LOGGER = LoggerFactory .getLogger(MultiNodeHandler.class); protected final ReentrantLock lock = new ReentrantLock(); protected final NonBlockingSession session; private AtomicBoolean isFailed = new AtomicBoolean(false); protected volatile String error; protected byte packetId; public final AtomicBoolean errorRepsponsed = new AtomicBoolean(false); public MultiNodeHandler(NonBlockingSession session) { if (session == null) { throw new IllegalArgumentException("session is null!"); } this.session = session; } public void setFail(String errMsg) { isFailed.set(true); error = errMsg; } public boolean isFail() { return isFailed.get(); } protected int nodeCount; private Runnable terminateCallBack; @Override public void terminate(Runnable terminateCallBack) { boolean zeroReached = false; lock.lock(); try { if (nodeCount > 0) { this.terminateCallBack = terminateCallBack; } else { zeroReached = true; } } finally { lock.unlock(); } if (zeroReached) { terminateCallBack.run(); } } protected boolean canClose(BackendConnection conn, boolean tryErrorFinish) { // realse this connection if safe session.releaseConnectionIfSafe(conn, LOGGER.isDebugEnabled(), false); boolean allFinished = false; if (tryErrorFinish) { allFinished = this.decrementCountBy(1); this.tryErrorFinished(allFinished); } return allFinished; } protected void decrementCountToZero() { Runnable callback; lock.lock(); try { nodeCount = 0; callback = this.terminateCallBack; this.terminateCallBack = null; } finally { lock.unlock(); } if (callback != null) { callback.run(); } } public void connectionError(Throwable e, BackendConnection conn) { setFail("backend connect: "+e); if (LOGGER.isDebugEnabled()) { LOGGER.debug(this.toString() +"on connectionError " + conn + " is has Error"+ errorRepsponsed.get() +" not receive num"+nodeCount); } final boolean canClose = decrementCountBy(1); // 需要把Throwable e的错误信息保存下来（setFail()）， 否则会导致响应 //null信息，结果mysql命令行等客户端查询结果是"Query OK"！！ // @author Uncle-pan // @since 2016-03-26 //if(canClose){ //} LOGGER.warn(this + "backend connect", e); // if(canClose) { // } this.tryErrorFinished(canClose); } public void errorResponse(byte[] data, BackendConnection conn) { session.releaseConnectionIfSafe(conn, LOGGER.isDebugEnabled(), false); ErrorPacket err = new ErrorPacket(); err.read(data); String errmsg = new String(err.message); this.setFail(errmsg); LOGGER.warn(this.toString() +"error response from " + conn + " err " + errmsg + " code:" + err.errno); this.tryErrorFinished(this.decrementCountBy(1)); } public boolean clearIfSessionClosed(NonBlockingSession session) { if (session.closed()) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("session closed ,clear resources " + session); } session.clearResources(true); this.clearResources(); return true; } else { return false; } } protected boolean decrementCountBy(int finished) { boolean zeroReached = false; Runnable callback = null; lock.lock(); try { nodeCount -= finished; zeroReached = nodeCount == 0; if (zeroReached) { callback = this.terminateCallBack; this.terminateCallBack = null; } } finally { lock.unlock(); } if (zeroReached && callback != null) { callback.run(); } return zeroReached; } protected void reset(int initCount) { nodeCount = initCount; isFailed.set(false); error = null; packetId = 0; } protected ErrorPacket createErrPkg(String errmgs) { ErrorPacket err = new ErrorPacket(); lock.lock(); try { err.packetId = ++packetId; } finally { lock.unlock(); } err.errno = ErrorCode.ER_UNKNOWN_ERROR; err.message = StringUtil.encode(errmgs, session.getSource().getCharset()); return err; } protected void tryErrorFinished(boolean allEnd) { if (allEnd && !session.closed()) { // clear session resources,release all if (LOGGER.isDebugEnabled()) { LOGGER.debug(this.toString()+"error all end ,clear session resource "); LOGGER.debug("error all end ,clear session resource "); } if (session.getSource().isAutocommit()) { session.closeAndClearResources(error); } else { session.getSource().setTxInterrupt(this.error); // clear resouces clearResources(); } if (errorRepsponsed.compareAndSet(false, true)) { //createErrPkg(this.error).write(session.getSource()); session.getSource().writeErrMessage(ErrorCode.ER_UNKNOWN_ERROR, this.error); } } } public void connectionClose(BackendConnection conn, String reason) { this.setFail("closed connection:" + reason + " con:" + conn); if (LOGGER.isDebugEnabled()) { LOGGER.debug(this.toString()+ "closed connection:" + reason + " con:" + conn); } boolean finished = false; lock.lock(); try { finished = (this.nodeCount == 0); } finally { lock.unlock(); } if (finished == false) { finished = this.decrementCountBy(1); } if (error == null) { error = "back connection closed "; } tryErrorFinished(finished); } public void clearResources() { } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/MultiNodeQueryHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql.nio.handler; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.backend.BackendConnection; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.backend.mysql.CharsetUtil; import io.mycat.backend.mysql.LoadDataUtil; import io.mycat.backend.mysql.listener.SqlExecuteStage; import io.mycat.backend.mysql.nio.MySQLConnection; import io.mycat.cache.LayerCachePool; import io.mycat.config.ErrorCode; import io.mycat.config.MycatConfig; import io.mycat.memory.unsafe.row.UnsafeRow; import io.mycat.net.mysql.BinaryRowDataPacket; import io.mycat.net.mysql.ErrorPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.OkPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.route.RouteResultset; import io.mycat.route.RouteResultsetNode; import io.mycat.server.NonBlockingSession; import io.mycat.server.ServerConnection; import io.mycat.server.parser.ServerParse; import io.mycat.sqlengine.mpp.AbstractDataNodeMerge; import io.mycat.sqlengine.mpp.ColMeta; import io.mycat.sqlengine.mpp.DataMergeService; import io.mycat.sqlengine.mpp.DataNodeMergeManager; import io.mycat.sqlengine.mpp.MergeCol; import io.mycat.statistic.stat.QueryResult; import io.mycat.statistic.stat.QueryResultDispatcher; import io.mycat.util.ResultSetUtil; import io.mycat.util.StringUtil; /** * @author mycat */ public class MultiNodeQueryHandler extends MultiNodeHandler implements LoadDataResponseHandler { private static final Logger LOGGER = LoggerFactory.getLogger(MultiNodeQueryHandler.class); private final RouteResultset rrs; private final NonBlockingSession session; // private final CommitNodeHandler icHandler; private final AbstractDataNodeMerge dataMergeSvr; private final boolean autocommit; private String priamaryKeyTable = null; private int primaryKeyIndex = -1; private int fieldCount = 0; // private final ReentrantLock lock; private long affectedRows; private long selectRows; private long insertId; private volatile boolean fieldsReturned; private int okCount; private final boolean isCallProcedure; private long startTime; private long netInBytes; private long netOutBytes; private int execCount = 0; private boolean prepared; private List fieldPackets = new ArrayList(); private int isOffHeapuseOffHeapForMerge = 1; //huangyiming add 中间处理结果是否处理完毕 private final AtomicBoolean isMiddleResultDone; /** * Limit N，M */ private int limitStart; private int limitSize; private int index = 0; private int end = 0; //huangyiming private byte[] header = null; private List fields = null; protected List errConnections; // by kaiz : 为了解决Mybatis获取由Mycat生成自增主键时，MySQL返回的Last_insert_id为最大值的问题； // 当逻辑表设置了autoIncrement='false'时，MyCAT会将ok packet当中的最小last insert id记录下来，返回给应用 // 当逻辑表设置了autoIncrement='true'时，MyCAT会将ok packet当中的最大的last insert id记录下来，然后减掉affected rows的数量后，返回给应用 public MultiNodeQueryHandler(int sqlType, RouteResultset rrs, boolean autocommit, NonBlockingSession session) { super(session); this.isMiddleResultDone = new AtomicBoolean(false); if (rrs.getNodes() == null) { throw new IllegalArgumentException("routeNode is null!"); } if (LOGGER.isDebugEnabled()) { LOGGER.debug("execute mutinode query " + rrs.getStatement()); } this.rrs = rrs; isOffHeapuseOffHeapForMerge = MycatServer.getInstance(). getConfig().getSystem().getUseOffHeapForMerge(); if (ServerParse.SELECT == sqlType && rrs.needMerge()) { /** * 使用Off Heap */ if(isOffHeapuseOffHeapForMerge == 1){ dataMergeSvr = new DataNodeMergeManager(this,rrs,isMiddleResultDone); }else { dataMergeSvr = new DataMergeService(this,rrs); } } else { dataMergeSvr = null; } isCallProcedure = rrs.isCallStatement(); this.autocommit = session.getSource().isAutocommit(); this.session = session; // this.lock = new ReentrantLock(); // this.icHandler = new CommitNodeHandler(session); this.limitStart = rrs.getLimitStart(); this.limitSize = rrs.getLimitSize(); this.end = limitStart + rrs.getLimitSize(); if (this.limitStart < 0) this.limitStart = 0; if (rrs.getLimitSize() < 0) end = Integer.MAX_VALUE; if ((dataMergeSvr != null) && LOGGER.isDebugEnabled()) { LOGGER.debug("has data merge logic "); } if ( rrs != null && rrs.getStatement() != null) { netInBytes += rrs.getStatement().getBytes().length; } this.errConnections = new ArrayList<>(); } protected void reset(int initCount) { super.reset(initCount); this.okCount = initCount; this.execCount = 0; this.netInBytes = 0; this.netOutBytes = 0; if (rrs.isLoadData()) { packetId = session.getSource().getLoadDataInfileHandler().getLastPackId(); } } private synchronized int incExecCount() { return ++this.execCount; } public NonBlockingSession getSession() { return session; } public void execute() throws Exception { final ReentrantLock lock = this.lock; lock.lock(); try { this.reset(rrs.getNodes().length); this.fieldsReturned = false; this.affectedRows = 0L; this.insertId = 0L; } finally { lock.unlock(); } MycatConfig conf = MycatServer.getInstance().getConfig(); startTime = System.currentTimeMillis(); LOGGER.debug("rrs.getRunOnSlave()-" + rrs.getRunOnSlaveDebugInfo()); //todo 增加处理如果超过最大链接的处理。是zwy 2018.07 int start = 0; try { for (final RouteResultsetNode node : rrs.getNodes()) { BackendConnection conn = session.getTarget(node); if (session.tryExistsCon(conn, node)) { if(LOGGER.isDebugEnabled()) { LOGGER.debug("node.getRunOnSlave()-" + node.getRunOnSlave()); LOGGER.debug(new StringBuilder(this.toString()).append(session.getSource()).append(rrs).toString()); } node.setRunOnSlave(rrs.getRunOnSlave()); // 实现 master/slave注解 if(LOGGER.isDebugEnabled()) { LOGGER.debug("node.getRunOnSlave()-" + node.getRunOnSlave()); } _execute(conn, node); } else { // create new connection //LOGGER.debug("node.getRunOnSlave()1-" + node.getRunOnSlave()); node.setRunOnSlave(rrs.getRunOnSlave()); // 实现 master/slave注解 //LOGGER.debug("node.getRunOnSlave()2-" + node.getRunOnSlave()); PhysicalDBNode dn = conf.getDataNodes().get(node.getName()); dn.getConnection(dn.getDatabase(), autocommit, node, this, node); // 注意该方法不仅仅是获取连接，获取新连接成功之后，会通过层层回调，最后回调到本类 的connectionAcquired // 这是通过 上面方法的 this 参数的层层传递完成的。 // connectionAcquired 进行执行操作: // session.bindConnection(node, conn); // _execute(conn, node); } start++; } }catch (Exception e) { ServerConnection source = session.getSource(); int len = rrs.getNodes().length - start; for(int i = 0 ; i < len ; i++) { //flag = this.decrementCountBy(1); this.connectionError(e, null); } LOGGER.error(new StringBuilder(this.toString()).append(source).append(rrs).toString(), e); // this.connectionError(e, null); // if(flag) { // LOGGER.error(new StringBuilder(this.toString()).append(source).append(rrs).toString(), e); //} } } private void _execute(BackendConnection conn, RouteResultsetNode node) { if (clearIfSessionClosed(session)) { return; } conn.setResponseHandler(this); try { // conn.execute(node, session.getSource(), autocommit); /** * 多节点下，非DDL的修改语句，自动开启事务；DDL语句不开启事务，原因是ddl语句会引发自动提交，如果在事务中会引起下面的问题。 * 1）如果是普通事务,直接引发自动提交； * 2）XA事务，自动提交导致ERROR 1399 (XAE07): XAER_RMFAIL: The command cannot be executed when global transaction is in the ACTIVE state * */ conn.execute(node, session.getSource(), autocommit && !node.isModifySQLExceptDDL()); } catch (IOException e) { connectionError(e, conn); } } @Override public void connectionAcquired(final BackendConnection conn) { final RouteResultsetNode node = (RouteResultsetNode) conn .getAttachment(); session.bindConnection(node, conn); if(errorRepsponsed.get()) { ServerConnection source = session.getSource(); LOGGER.warn(new StringBuilder(this.toString()).append(source).append(rrs).toString(), "connectionAcquired",conn); this.connectionClose(conn, "find error, so close this connection"); return ; } _execute(conn, node); } private boolean decrementOkCountBy(int finished) { lock.lock(); try { return --okCount == 0; } finally { lock.unlock(); } } @Override public void errorResponse(byte[] data, BackendConnection conn) { ErrorPacket errPacket = new ErrorPacket(); errPacket.read(data); lock.lock(); try { if (!isFail()) { setFail(new String(errPacket.message)); } errConnections.add(conn); conn.syncAndExcute(); if (--nodeCount == 0) { errPacket.packetId = ++packetId; processFinishWork(errPacket.writeToBytes(), false, conn); } } finally { lock.unlock(); } } /** * wangkai 后端的mysql连接被关闭调用到这里； * 2019年3月13日,异常走executeError，保证最后走handleEndPacket，保证XA顺利释放资源 */ @Override public void connectionClose(BackendConnection conn, String reason) { LOGGER.warn("backend connect" + reason + ", conn info:" + conn); ErrorPacket errPacket = new ErrorPacket(); errPacket.errno = ErrorCode.ER_ABORTING_CONNECTION; reason = "Connection {DataHost[" + conn.getHost() + ":" + conn.getPort() + "],Schema[" + conn.getSchema() + "],threadID[" + ((MySQLConnection) conn).getThreadId() + "]} was closed ,reason is [" + reason + "]"; errPacket.message = StringUtil.encode(reason, session.getSource().getCharset()); executeError(conn, errPacket); } @Override public void connectionError(Throwable e, BackendConnection conn) { LOGGER.info("backend connect", e); ErrorPacket errPacket = new ErrorPacket(); errPacket.errno = ErrorCode.ER_ABORTING_CONNECTION; String errMsg = null; if(conn == null) { errMsg = e.toString(); }else { errMsg = "Backend connect Error, Connection{DataHost[" + conn.getHost() + ":" + conn.getPort() + "],Schema[" + conn.getSchema() + "]} refused"; } errPacket.message = StringUtil.encode(errMsg, session.getSource().getCharset()); executeError(conn, errPacket); } private void executeError(BackendConnection conn, ErrorPacket errPacket) { lock.lock(); try { if (!isFail()) { setFail(new String(errPacket.message)); } if(conn != null) { errConnections.add(conn); } if (--nodeCount == 0) { errPacket.packetId = ++packetId; processFinishWork(errPacket.writeToBytes(), false, conn); } } finally { lock.unlock(); } } @Override public void okResponse(byte[] data, BackendConnection conn) { this.netOutBytes += data.length; boolean executeResponse = conn.syncAndExcute(); if (LOGGER.isDebugEnabled()) { LOGGER.debug("received ok response ,executeResponse:" + executeResponse + " from " + conn); } if (executeResponse) { ServerConnection source = session.getSource(); OkPacket ok = new OkPacket(); ok.read(data); lock.lock(); try { affectedRows += ok.affectedRows; } finally { lock.unlock(); } if (ok.hasMoreResultsExists()) { // funnyAnt:当是批量update/delete语句，提示后面还有ok包 return; } // 存储过程 boolean isCanClose2Client = (!rrs.isCallStatement()) || (rrs.isCallStatement() && !rrs.getProcedure().isResultSimpleValue()); // if(!isCallProcedure) // { // if (clearIfSessionClosed(session)) // { // return; // } else if (canClose(conn, false)) // { // return; // } // } lock.lock(); try { if (ok.insertId > 0) { if (rrs.getAutoIncrement()) { insertId = (insertId == 0) ? ok.insertId : Math.max(insertId, ok.insertId); } else { insertId = (insertId == 0) ? ok.insertId : Math.min(insertId, ok.insertId); } } } catch (Exception e) { LOGGER.error(e.getMessage(), e); } finally { lock.unlock(); } if (LOGGER.isDebugEnabled()) { LOGGER.debug(this.toString() +"on row okResponse " + conn + " "+ errorRepsponsed.get() +" "+nodeCount); } // 对于存储过程，其比较特殊，查询结果返回EndRow报文以后，还会再返回一个OK报文，才算结束 boolean isEndPacket = isCallProcedure ? decrementOkCountBy(1): decrementCountBy(1); if (isEndPacket && isCanClose2Client) { // if (this.autocommit && !session.getSource().isLocked()) {// clear all // connections // session.releaseConnections(false); // } if (this.isFail() || session.closed()) { // tryErrorFinished(true); processFinishWork(createErrPkg(this.error).writeToBytes(), false, conn); return; } lock.lock(); try { ok.packetId = ++packetId;// OK_PACKET if (rrs.isLoadData()) { ok.message = ("Records: " + affectedRows + " Deleted: 0 Skipped: 0 Warnings: 0") .getBytes();// 此处信息只是为了控制台给人看的 source.getLoadDataInfileHandler().clear(); } // 如果是全局表，去除掉重复计算的部分 if (rrs.isGlobalTable()) { affectedRows = affectedRows / rrs.getNodes().length; } ok.affectedRows = affectedRows; ok.serverStatus = source.isAutocommit() ? 2 : 1; if (insertId > 0) { ok.insertId = rrs.getAutoIncrement() ? (insertId - affectedRows + 1) : insertId; source.setLastInsertId(insertId); } // 判断是否已经报错返回给前台了 2018.07 if(source.canResponse()) { // ok.write(source); processFinishWork(ok.writeToBytes(), true, conn); } } catch (Exception e) { handleDataProcessException(e); } finally { lock.unlock(); } } // add by lian // 解决sql统计中写操作永远为0 int currentExecCount = this.incExecCount(); if (currentExecCount == rrs.getNodes().length) { source.setExecuteSql(null); //完善show @@connection.sql 监控命令.已经执行完的sql 不再显示 QueryResult queryResult = new QueryResult(session.getSource().getSchema(), session.getSource().getUser(), rrs.getSqlType(), rrs.getStatement(), selectRows, netInBytes, netOutBytes, startTime, System.currentTimeMillis(),0,source.getHost()); QueryResultDispatcher.dispatchQuery( queryResult ); } } } @Override public void rowEofResponse(final byte[] eof, BackendConnection conn) { if (LOGGER.isDebugEnabled()) { LOGGER.debug(this.toString() +"on row end reseponse " + conn + " "+ errorRepsponsed.get() +" "+nodeCount); } this.netOutBytes += eof.length; MiddlerResultHandler middlerResultHandler = session.getMiddlerResultHandler(); if (errorRepsponsed.get()) { // the connection has been closed or set to "txInterrupt" properly //in tryErrorFinished() method! If we close it here, it can // lead to tx error such as blocking rollback tx for ever. // @author Uncle-pan // @since 2016-03-25 // conn.close(this.error); return; } final ServerConnection source = session.getSource(); if (!isCallProcedure) { if (clearIfSessionClosed(session)) { return; } else if (canClose(conn, false)) { return; } } if (decrementCountBy(1)) { if (!rrs.isCallStatement()||(rrs.isCallStatement()&&rrs.getProcedure().isResultSimpleValue())) { if (this.autocommit && !session.getSource().isLocked()) {// clear all connections session.releaseConnections(false); } if (this.isFail() || session.closed()) { // tryErrorFinished(true); processFinishWork(createErrPkg(this.error).writeToBytes(), false, conn); return; } } if (dataMergeSvr != null) { //huangyiming add 数据合并前如果有中间过程则先执行数据合并再执行下一步 if(session.getMiddlerResultHandler() !=null ){ isMiddleResultDone.set(true); } try { dataMergeSvr.outputMergeResult(session, eof); } catch (Exception e) { handleDataProcessException(e); } } else { try { lock.lock(); eof[3] = ++packetId; if (LOGGER.isDebugEnabled()) { LOGGER.debug("last packet id:" + packetId); } if( middlerResultHandler ==null ){ //middlerResultHandler.secondEexcute(); if(source.canResponse()) { source.write(eof); } } } finally { lock.unlock(); } } } int currentExecCount = this.incExecCount(); if(middlerResultHandler !=null){ if (currentExecCount != rrs.getNodes().length) { return; } /*else{ middlerResultHandler.secondEexcute(); }*/ } if (currentExecCount == rrs.getNodes().length) { int resultSize = source.getWriteQueue().size()*MycatServer.getInstance().getConfig().getSystem().getBufferPoolPageSize(); source.setExecuteSql(null); //完善show @@connection.sql 监控命令.已经执行完的sql 不再显示 session.getSource().getListener().fireEvent(SqlExecuteStage.END); //TODO: add by zhuam //查询结果派发 QueryResult queryResult = new QueryResult(session.getSource().getSchema(), session.getSource().getUser(), rrs.getSqlType(), rrs.getStatement(), selectRows, netInBytes, netOutBytes, startTime, System.currentTimeMillis(),resultSize, source.getHost()); QueryResultDispatcher.dispatchQuery( queryResult ); // add huangyiming 如果是中间过程,必须等数据合并好了再进行下一步语句的拼装 if(middlerResultHandler !=null ){ while (!this.isMiddleResultDone.compareAndSet(false, true)) { Thread.yield(); } middlerResultHandler.secondEexcute(); isMiddleResultDone.set(false); } } } /** * 将汇聚结果集数据真正的发送给Mycat客户端 * @param source * @param eof * @param */ public void outputMergeResult(final ServerConnection source, final byte[] eof, Iterator iter,AtomicBoolean isMiddleResultDone) { try { lock.lock(); ByteBuffer buffer = session.getSource().allocate(); final RouteResultset rrs = this.dataMergeSvr.getRrs(); /** * 处理limit语句的start 和 end位置，将正确的结果发送给 * Mycat 客户端 */ int start = rrs.getLimitStart(); int end = start + rrs.getLimitSize(); int index = 0; if (start < 0) start = 0; if (rrs.getLimitSize() < 0) end = Integer.MAX_VALUE; if(prepared) { while (iter.hasNext()){ UnsafeRow row = iter.next(); if(index >= start){ row.packetId = ++packetId; BinaryRowDataPacket binRowPacket = new BinaryRowDataPacket(); binRowPacket.read(fieldPackets, row); buffer = binRowPacket.write(buffer, source, true); } index++; if(index == end){ break; } } } else { while (iter.hasNext()){ UnsafeRow row = iter.next(); if(index >= start){ row.packetId = ++packetId; buffer = row.write(buffer,source,true); } index++; if(index == end){ break; } } } eof[3] = ++packetId; if (LOGGER.isDebugEnabled()) { LOGGER.debug("last packet id:" + packetId); } //huangyiming add 中间过程缓存起来,isMiddleResultDone是确保合并部分执行完成后才会执行secondExecute MiddlerResultHandler middlerResultHandler = source.getSession2().getMiddlerResultHandler(); if(null != middlerResultHandler){ if(buffer.position() > 0){ buffer.flip(); byte[] data = new byte[buffer.limit()]; buffer.get(data); buffer.clear(); //如果该操作只是一个中间过程则把结果存储起来 String str = ResultSetUtil.getColumnValAsString(data, fields, 0); //真的需要数据合并的时候才合并 if(rrs.isHasAggrColumn()){ middlerResultHandler.getResult().clear(); if(str !=null){ middlerResultHandler.add(str); } } } isMiddleResultDone.set(false); }else{ ByteBuffer byteBuffer = source.writeToBuffer(eof, buffer); /** * 真正的开始把Writer Buffer的数据写入到channel 中 */ if(source.canResponse()) { source.write(byteBuffer); } } session.getSource().getListener().fireEvent(SqlExecuteStage.END); } catch (Exception e) { e.printStackTrace(); handleDataProcessException(e); } finally { dataMergeSvr.clear(); lock.unlock(); } } public void outputMergeResult(final ServerConnection source, final byte[] eof, List results) { try { lock.lock(); ByteBuffer buffer = session.getSource().allocate(); final RouteResultset rrs = this.dataMergeSvr.getRrs(); // 处理limit语句 int start = rrs.getLimitStart(); int end = start + rrs.getLimitSize(); if (start < 0) { start = 0; } if (rrs.getLimitSize() < 0) { end = results.size(); } // // 对于不需要排序的语句,返回的数据只有rrs.getLimitSize() // if (rrs.getOrderByCols() == null) { // end = results.size(); // start = 0; // } if (end > results.size()) { end = results.size(); } // for (int i = start; i < end; i++) { // RowDataPacket row = results.get(i); // if( prepared ) { // BinaryRowDataPacket binRowDataPk = new BinaryRowDataPacket(); // binRowDataPk.read(fieldPackets, row); // binRowDataPk.packetId = ++packetId; // //binRowDataPk.write(source); // buffer = binRowDataPk.write(buffer, session.getSource(), true); // } else { // row.packetId = ++packetId; // buffer = row.write(buffer, source, true); // } // } if(prepared) { for (int i = start; i < end; i++) { RowDataPacket row = results.get(i); BinaryRowDataPacket binRowDataPk = new BinaryRowDataPacket(); binRowDataPk.read(fieldPackets, row); binRowDataPk.packetId = ++packetId; //binRowDataPk.write(source); buffer = binRowDataPk.write(buffer, session.getSource(), true); } } else { for (int i = start; i < end; i++) { RowDataPacket row = results.get(i); row.packetId = ++packetId; buffer = row.write(buffer, source, true); } } eof[3] = ++packetId; if (LOGGER.isDebugEnabled()) { LOGGER.debug("last packet id:" + packetId); } if(source.canResponse()) { source.write(source.writeToBuffer(eof, buffer)); } session.getSource().getListener().fireEvent(SqlExecuteStage.END); } catch (Exception e) { handleDataProcessException(e); } finally { dataMergeSvr.clear(); lock.unlock(); } } @Override public void fieldEofResponse(byte[] header, List fields, byte[] eof, BackendConnection conn) { //10个连接有一个连接错误怎么办哦。 if (errorRepsponsed.get()|| this.isFail()) { // the connection has been closed or set to "txInterrupt" properly //in tryErrorFinished() method! If we close it here, it can // lead to tx error such as blocking rollback tx for ever. // @author Uncle-pan // @since 2016-03-25 // conn.close(this.error); return; } //huangyiming add this.header = header; this.fields = fields; MiddlerResultHandler middlerResultHandler = session.getMiddlerResultHandler(); /*if(null !=middlerResultHandler ){ return; }*/ this.netOutBytes += header.length; this.netOutBytes += eof.length; for (int i = 0, len = fields.size(); i < len; ++i) { byte[] field = fields.get(i); this.netOutBytes += field.length; } ServerConnection source = null; if (fieldsReturned) { return; } lock.lock(); try { if (fieldsReturned) { return; } fieldsReturned = true; boolean needMerg = (dataMergeSvr != null) && dataMergeSvr.getRrs().needMerge(); Set shouldRemoveAvgField = new HashSet<>(); Set shouldRenameAvgField = new HashSet<>(); if (needMerg) { Map mergeColsMap = dataMergeSvr.getRrs() .getMergeCols(); if (mergeColsMap != null) { for (Map.Entry entry : mergeColsMap .entrySet()) { String key = entry.getKey(); int mergeType = entry.getValue(); if (MergeCol.MERGE_AVG == mergeType && mergeColsMap.containsKey(key + "SUM")) { shouldRemoveAvgField.add((key + "COUNT") .toUpperCase()); shouldRenameAvgField.add((key + "SUM") .toUpperCase()); } } } } source = session.getSource(); ByteBuffer buffer = source.allocate(); fieldCount = fields.size(); if (shouldRemoveAvgField.size() > 0) { ResultSetHeaderPacket packet = new ResultSetHeaderPacket(); packet.packetId = ++packetId; packet.fieldCount = fieldCount - shouldRemoveAvgField.size(); buffer = packet.write(buffer, source, true); } else { header[3] = ++packetId; buffer = source.writeToBuffer(header, buffer); } String primaryKey = null; if (rrs.hasPrimaryKeyToCache()) { String[] items = rrs.getPrimaryKeyItems(); priamaryKeyTable = items[0]; primaryKey = items[1]; } Map columToIndx = new HashMap( fieldCount); for (int i = 0, len = fieldCount; i < len; ++i) { boolean shouldSkip = false; byte[] field = fields.get(i); if (needMerg) { FieldPacket fieldPkg = new FieldPacket(); fieldPkg.read(field); fieldPackets.add(fieldPkg); String charset = session.getSource().getCharset();// CharsetUtil.getCharset(fieldPkg.charsetIndex); String fieldName = new String(fieldPkg.name, charset).toUpperCase(); if (columToIndx != null && !columToIndx.containsKey(fieldName)) { if (shouldRemoveAvgField.contains(fieldName)) { shouldSkip = true; fieldPackets.remove(fieldPackets.size() - 1); } if (shouldRenameAvgField.contains(fieldName)) { String newFieldName = fieldName.substring(0, fieldName.length() - 3); fieldPkg.name = newFieldName.getBytes(); fieldPkg.packetId = ++packetId; shouldSkip = true; // 处理AVG字段位数和精度, AVG位数 = SUM位数 - 14 fieldPkg.length = fieldPkg.length - 14; // AVG精度 = SUM精度 + 4 fieldPkg.decimals = (byte) (fieldPkg.decimals + 4); buffer = fieldPkg.write(buffer, source, false); // 还原精度 fieldPkg.decimals = (byte) (fieldPkg.decimals - 4); } ColMeta colMeta = new ColMeta(i, fieldPkg.type); colMeta.decimals = fieldPkg.decimals; columToIndx.put(fieldName, colMeta); } } else { FieldPacket fieldPkg = new FieldPacket(); fieldPkg.read(field); fieldPackets.add(fieldPkg); fieldCount = fields.size(); if (primaryKey != null && primaryKeyIndex == -1) { // find primary key index String fieldName = new String(fieldPkg.name); if (primaryKey.equalsIgnoreCase(fieldName)) { primaryKeyIndex = i; } } } if (!shouldSkip) { field[3] = ++packetId; buffer = source.writeToBuffer(field, buffer); } } eof[3] = ++packetId; buffer = source.writeToBuffer(eof, buffer); if (null == middlerResultHandler) { //session.getSource().write(row); source.write(buffer); } if (dataMergeSvr != null) { dataMergeSvr.onRowMetaData(columToIndx, fieldCount); } } catch (Exception e) { handleDataProcessException(e); } finally { lock.unlock(); } } public void handleDataProcessException(Exception e) { if (!errorRepsponsed.get()) { this.error = e.toString(); LOGGER.warn(this.toString() +" caught exception ", e); setFail(e.toString()); //判断是否全部返回 boolean finished = false; lock.lock(); try { finished = (this.nodeCount == 0); } finally { lock.unlock(); } this.tryErrorFinished(finished); } } @Override public void rowResponse(final byte[] row, final BackendConnection conn) { if (errorRepsponsed.get()||this.isFail()) { // the connection has been closed or set to "txInterrupt" properly //in tryErrorFinished() method! If we close it here, it can // lead to tx error such as blocking rollback tx for ever. // @author Uncle-pan // @since 2016-03-25 //conn.close(error); return; } lock.lock(); try { this.selectRows++; RouteResultsetNode rNode = (RouteResultsetNode) conn.getAttachment(); String dataNode = rNode.getName(); if (dataMergeSvr != null) { // even through discarding the all rest data, we can't //close the connection for tx control such as rollback or commit. // So the "isClosedByDiscard" variable is unnecessary. // @author Uncle-pan // @since 2016-03-25 dataMergeSvr.onNewRecord(dataNode, row); MiddlerResultHandler middlerResultHandler = session.getMiddlerResultHandler(); if(null != middlerResultHandler ){ if(middlerResultHandler instanceof MiddlerQueryResultHandler){ byte[] rv = ResultSetUtil.getColumnVal(row, fields, 0); String rowValue = rv==null? "":new String(rv); middlerResultHandler.add(rowValue); } } } else { row[3] = ++packetId; RowDataPacket rowDataPkg =null; // cache primaryKey-> dataNode if (primaryKeyIndex != -1) { rowDataPkg = new RowDataPacket(fieldCount); rowDataPkg.read(row); String primaryKey = new String(rowDataPkg.fieldValues.get(primaryKeyIndex)); LayerCachePool pool = MycatServer.getInstance().getRouterservice().getTableId2DataNodeCache(); if (priamaryKeyTable != null){ pool.putIfAbsent(priamaryKeyTable.toUpperCase(), primaryKey, dataNode); } } if( prepared ) { if(rowDataPkg==null) { rowDataPkg = new RowDataPacket(fieldCount); rowDataPkg.read(row); } BinaryRowDataPacket binRowDataPk = new BinaryRowDataPacket(); binRowDataPk.read(fieldPackets, rowDataPkg); binRowDataPk.write(session.getSource()); } else { //add huangyiming MiddlerResultHandler middlerResultHandler = session.getMiddlerResultHandler(); if(null == middlerResultHandler ){ session.getSource().write(row); }else{ if(middlerResultHandler instanceof MiddlerQueryResultHandler){ String rowValue = ResultSetUtil.getColumnValAsString(row, fields, 0); middlerResultHandler.add(rowValue); } } } } } catch (Exception e) { handleDataProcessException(e); } finally { lock.unlock(); } } @Override public void clearResources() { lock.lock(); try { if (dataMergeSvr != null) { dataMergeSvr.clear(); } } finally { lock.unlock(); } } @Override public void writeQueueAvailable() { } @Override public void requestDataResponse(byte[] data, BackendConnection conn) { LoadDataUtil.requestFileDataResponse(data, conn); } public boolean isPrepared() { return prepared; } public void setPrepared(boolean prepared) { this.prepared = prepared; } /** * 最后一个OK/ERROR包返回后的后续流程： * 1）如果是隐式事务，需要自动提交 * 2）显示事务，返回OK/ERROR给客户端，需要后面继续执行rollback或commit; * 3）非事务：返回OK/ERROR给客户端 * @param data * @param isCommit 是否提交事务 * @param conn */ protected void processFinishWork(byte[] data, boolean isCommit, BackendConnection conn) { ServerConnection source = session.getSource(); source.getListener().fireEvent(SqlExecuteStage.END); boolean isImplicitTransaction = false; if (source.isAutocommit()) { for (Entry entry : session.getTargetMap().entrySet()) { BackendConnection c = entry.getValue(); if (c.isModifiedSQLExecuted() && !c.isAutocommit()) { isImplicitTransaction = true; break; } } } if (isImplicitTransaction) { // 1隐式事务:修改类语句并且autocommit=true，mycat自动开启事务，需要自动提交掉 if (nodeCount < 0) { return; } // Implicit Distributed Transaction,send commit or rollback automatically if (isCommit) { // data 包里面带有更新或改变了多少行数据的信息，需要在commit后发送给client CommitNodeHandler commitHandler = new CommitNodeHandler(session, data); commitHandler.commit(); } else { RollbackNodeHandler rollbackHandler = new RollbackNodeHandler(session, data); rollbackHandler.rollback(); } } else { // 2 非事务或者显式事务，其中显式事务需要 等待后续的commit或rollback boolean inTransaction = !source.isAutocommit(); if (!inTransaction) { // 释放可能没有被释放的连接session.target for (BackendConnection errConn : errConnections) { errConn.setResponseHandler(null); session.closeConnection(errConn, "Connection happening error can't been reused,close directly"); } } if (nodeCount == 0) { // 事务下如果已经出错设置中断 if (inTransaction && !isCommit) { source.setTxInterrupt("ROLLBACK"); } // 发送语句执行结果给client source.write(data); } } } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/NewConnectionRespHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql.nio.handler; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.backend.BackendConnection; public class NewConnectionRespHandler implements ResponseHandler{ private static final Logger LOGGER = LoggerFactory .getLogger(NewConnectionRespHandler.class); @Override public void connectionError(Throwable e, BackendConnection conn) { LOGGER.warn(conn+" connectionError "+e); } @Override public void connectionAcquired(BackendConnection conn) { // LOGGER.info("connectionAcquired "+conn); conn.release(); // NewConnectionRespHandler 因为这个是由于空闲连接数低于配置，需要新建连接，但再新建连接的时候， } @Override public void errorResponse(byte[] err, BackendConnection conn) { LOGGER.warn("caught error resp: " + conn + " " + new String(err)); conn.release(); } @Override public void okResponse(byte[] ok, BackendConnection conn) { LOGGER.info("okResponse: " + conn ); conn.release(); } @Override public void fieldEofResponse(byte[] header, List fields, byte[] eof, BackendConnection conn) { LOGGER.info("fieldEofResponse: " + conn ); } @Override public void rowResponse(byte[] row, BackendConnection conn) { LOGGER.info("rowResponse: " + conn ); } @Override public void rowEofResponse(byte[] eof, BackendConnection conn) { LOGGER.info("rowEofResponse: " + conn ); conn.release(); } @Override public void writeQueueAvailable() { } @Override public void connectionClose(BackendConnection conn, String reason) { } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/PrepareRequestHandler.java ================================================ package io.mycat.backend.mysql.nio.handler; import java.util.ArrayList; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlDeleteStatement; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlInsertStatement; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlUpdateStatement; import com.alibaba.druid.sql.parser.SQLParserUtils; import com.alibaba.druid.sql.parser.SQLStatementParser; import com.alibaba.druid.util.JdbcUtils; import io.mycat.MycatServer; import io.mycat.backend.BackendConnection; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.mysql.nio.MySQLConnection; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.TableConfig; import io.mycat.net.mysql.CommandPacket; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.ErrorPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.MySQLPacket; import io.mycat.net.mysql.PreparedOkPacket; import io.mycat.route.parser.druid.MycatSchemaStatVisitor; import io.mycat.server.ServerConnection; import io.mycat.util.StringUtil; /** * 用于取prepare语句的原数据，直接把语句透传给Mysql * https://dev.mysql.com/doc/internals/en/com-stmt-prepare-response.html#packet-COM_STMT_PREPARE_OK * * 约束： 不支持库内分表等需要改写表名的语句 * * @author funnyAnt * */ public class PrepareRequestHandler implements ResponseHandler { private static final Logger LOGGER = LoggerFactory.getLogger(PrepareRequestHandler.class); private String sql; private ServerConnection sc; private PrepareRequestCallback callbackHandler; private volatile long statementId = -1L; private List params; private List fields; private volatile boolean statementClosed; private int paramSize; private int fieldSize; private volatile boolean lastPacket; private static String SYNC_CHANNEL_STATUS_SQL = "SELECT 1";// 用于同步通道的状态信息 public PrepareRequestHandler(ServerConnection sc, String sql, PrepareRequestCallback callbackHandler) { this.sc = sc; this.sql = sql; this.callbackHandler = callbackHandler; this.params = new ArrayList<>(8); this.fields = new ArrayList<>(8); } public void execute() { try { List tables = parseTables(sql); SchemaConfig schemaConfig = MycatServer.getInstance().getConfig().getSchemas().get(sc.getSchema()); // 根据表的规则确定路由节点，路由判断规则时剔除全局表后，选择剩余第一个表的第一个节点 String routeDataNode = null; for (String table : tables) { TableConfig tc = schemaConfig.getTables().get(table.toUpperCase()); if (tc != null) { routeDataNode = tc.getDataNodes().get(0); } else { routeDataNode = schemaConfig.getDataNode(); } if (tc != null && !tc.isGlobalTable()) { break; } } if (routeDataNode == null) { doFinished(true, "can't find route node"); return; } PhysicalDBNode dn = MycatServer.getInstance().getConfig().getDataNodes().get(routeDataNode); PhysicalDatasource ds = dn.getDbPool().getSource(); if (ds.getConfig().getDbType().equalsIgnoreCase("MYSQL")) { ds.getConnection(dn.getDatabase(), true, this, null); } else { doFinished(true, "Does not support getting metadata from non-mysql database"); } } catch (Exception e) { LOGGER.info("can't get connection for sql ,error:", e); doFinished(true, e.getMessage()); } } private void doFinished(boolean failed, String errorMsg) { if (!lastPacket) { lastPacket = true; callbackHandler.callback(!failed, errorMsg, params.toArray(new FieldPacket[0]), fields.toArray(new FieldPacket[0])); } } private void closePrepareStmt(BackendConnection conn) { //https://dev.mysql.com/doc/internals/en/com-stmt-close.html if (!statementClosed) { // PreparedClosePacket packet = new PreparedClosePacket(this.statementId); CommandPacket closePreparePacket = new CommandPacket(); closePreparePacket.command = MySQLPacket.COM_STMT_CLOSE; closePreparePacket.arg = String.valueOf(this.statementId).getBytes(); statementClosed = true; closePreparePacket.write((MySQLConnection) conn); } } /** * 从sql 语句中解析出tableName * * @return */ private List parseTables(String sql) { List tables = new ArrayList<>(); try { SQLStatementParser sqlStatementParser = SQLParserUtils.createSQLStatementParser(sql, JdbcUtils.MYSQL); SQLStatement statement = sqlStatementParser.parseStatement(); if (statement instanceof MySqlUpdateStatement){ String simpleName = ((MySqlUpdateStatement) statement).getTableName().getSimpleName(); tables.add(StringUtil.removeBackquote(simpleName)); }else if (statement instanceof MySqlDeleteStatement){ String simpleName = ((MySqlDeleteStatement) statement).getTableName().getSimpleName(); tables.add(StringUtil.removeBackquote(simpleName)); }else if (statement instanceof MySqlInsertStatement){ String simpleName = ((MySqlInsertStatement) statement).getTableName().getSimpleName(); tables.add(StringUtil.removeBackquote(simpleName)); } else { MycatSchemaStatVisitor visitor = new MycatSchemaStatVisitor(); statement.accept(visitor); tables.addAll(visitor.getAliasMap().values()); } } catch (Exception e) { LOGGER.error("can not get column count", e); } return tables; } private void sendPrepareRequestCommand(BackendConnection conn) { try { CommandPacket preparePacket = new CommandPacket(); preparePacket.command = MySQLPacket.COM_STMT_PREPARE; preparePacket.arg = this.sql.getBytes(sc.getCharset()); preparePacket.write((MySQLConnection) conn); } catch (Exception e) { doFinished(true, e.getMessage()); } } public boolean isLastPacket() { return lastPacket; } @Override public void connectionError(Throwable e, BackendConnection conn) { // TODO Auto-generated method stub LOGGER.info("can't get connection for sql :" + sql); doFinished(true, e.getMessage()); } @Override public void connectionAcquired(BackendConnection conn) { // TODO Auto-generated method stub if (LOGGER.isDebugEnabled()) { LOGGER.debug("con query sql:" + sql + " to con:" + conn); } conn.setResponseHandler(this); try { if (((MySQLConnection) conn).isNeedSyncSchema()) { // 发送一个select 1语句触发同步schema；后面在rowEofResponse里面发送PrepareRequestCommand命令 conn.query(SYNC_CHANNEL_STATUS_SQL); }else { this.sendPrepareRequestCommand(conn); } } catch (Exception e) {// (UnsupportedEncodingException e) { doFinished(true, e.getMessage()); } } @Override public void errorResponse(byte[] err, BackendConnection conn) { ErrorPacket errPg = new ErrorPacket(); errPg.read(err); String errMsg = "error response errno:" + errPg.errno + ", " + new String(errPg.message) + " from of sql :" + sql + " at con:" + conn; // @see https://dev.mysql.com/doc/refman/5.6/en/error-messages-server.html // ER_SPECIFIC_ACCESS_DENIED_ERROR if (errPg.errno == 1227) { LOGGER.warn(errMsg); } else { LOGGER.info(errMsg); } doFinished(true, errMsg); // closePrepareStmt(conn); conn.release(); } @Override public void okResponse(byte[] data, BackendConnection conn) { // TODO Auto-generated method stub if(statementId == -1L) { boolean executeResponse = conn.syncAndExcute(); if (executeResponse) { // 解析包 PreparedOkPacket preparedOkPacket = new PreparedOkPacket(); preparedOkPacket.read(data); this.paramSize = preparedOkPacket.parametersNumber; this.fieldSize = preparedOkPacket.columnsNumber; this.statementId = preparedOkPacket.statementId; } }else { switch (data[4]) { case ErrorPacket.FIELD_COUNT: errorResponse(data, conn); break; case EOFPacket.FIELD_COUNT: if (fields.size() == fieldSize) { // field包后面的EOF包,整个协议结束。 doFinished(false, null); closePrepareStmt(conn); conn.release(); } break; default: // 解析field FieldPacket packet = new FieldPacket(); packet.read(data); // 数据库名称修改为mycat逻辑库 packet.db = this.sc.getSchema().getBytes(); packet.length = packet.calcPacketSize(); if (params.size() < paramSize) { params.add(packet); } else { if (fields.size() < fieldSize) { fields.add(packet); } } } } } @Override public void fieldEofResponse(byte[] header, List fields, byte[] eof, BackendConnection conn) { // TODO Auto-generated method stub } @Override public void rowResponse(byte[] row, BackendConnection conn) { // TODO Auto-generated method stub } @Override public void rowEofResponse(byte[] eof, BackendConnection conn) { // TODO Auto-generated method stub // 说明同步通道的语句SELECT 1已经成功，发送真正的prepare包。这样做的原因是包类型不一样，目前接口无法做到一次性发送 this.sendPrepareRequestCommand(conn); } @Override public void writeQueueAvailable() { // TODO Auto-generated method stub } @Override public void connectionClose(BackendConnection conn, String reason) { // TODO Auto-generated method stub doFinished(true, reason); } /** * * 用于回调返回PrepareRequest值 * */ public interface PrepareRequestCallback { void callback(boolean success, String msg, FieldPacket[] params, FieldPacket[] fields); } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/ResponseHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql.nio.handler; import java.util.List; import io.mycat.backend.BackendConnection; /** * @author mycat * @author mycat */ public interface ResponseHandler { /** * 无法获取连接 * * @param e * @param conn */ public void connectionError(Throwable e, BackendConnection conn); /** * 已获得有效连接的响应处理 */ void connectionAcquired(BackendConnection conn); /** * 收到错误数据包的响应处理 */ void errorResponse(byte[] err, BackendConnection conn); /** * 收到OK数据包的响应处理 */ void okResponse(byte[] ok, BackendConnection conn); /** * 收到字段数据包结束的响应处理 */ void fieldEofResponse(byte[] header, List fields, byte[] eof, BackendConnection conn); /** * 收到行数据包的响应处理 */ void rowResponse(byte[] row, BackendConnection conn); /** * 收到行数据包结束的响应处理 */ void rowEofResponse(byte[] eof, BackendConnection conn); /** * 写队列为空，可以写数据了 * */ void writeQueueAvailable(); /** * on connetion close event */ void connectionClose(BackendConnection conn, String reason); } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/RollbackNodeHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql.nio.handler; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.backend.BackendConnection; import io.mycat.backend.mysql.nio.MySQLConnection; import io.mycat.backend.mysql.xa.CoordinatorLogEntry; import io.mycat.backend.mysql.xa.TxState; import io.mycat.config.ErrorCode; import io.mycat.net.mysql.OkPacket; import io.mycat.route.RouteResultsetNode; import io.mycat.server.NonBlockingSession; /** * @author mycat */ public class RollbackNodeHandler extends MultiNodeHandler { private static final Logger LOGGER = LoggerFactory.getLogger(RollbackNodeHandler.class); protected byte[] responseData; public RollbackNodeHandler(NonBlockingSession session) { super(session); } public RollbackNodeHandler(NonBlockingSession session, byte[] responseData) { super(session); this.responseData = responseData; } public void rollback() { final int initCount = session.getTargetCount(); lock.lock(); try { reset(initCount); } finally { lock.unlock(); } if (session.closed()) { decrementCountToZero(); return; } int start = 0 ; boolean hasClose = false; for (final RouteResultsetNode node : session.getTargetKeys()) { if (node == null) { LOGGER.error("null is contained in RoutResultsetNodes, source = " + session.getSource()); hasClose = true; break; } final BackendConnection conn = session.getTarget(node); if (conn != null) { boolean isClosed=conn.isClosedOrQuit(); if(isClosed) { hasClose = true; // break; } else { start ++; } } } //有连接已被关闭 直接关闭所有的连接， 非xa模式下 ，xa prepare状态下需要rollback if(hasClose && session.getXaTXID() == null) { LOGGER.warn("find close back conn close ,so close all back connection" + session.getSource()); session.setAutoCommitStatus(); //一定是先恢复状态 在写消息 this.setFail("receive rollback,but find backend con is closed or quit"); this.tryErrorFinished(true); return ; } // 执行 //modify by zwy // 执行 lock.lock(); try { reset(start); } finally { lock.unlock(); } boolean writeCheckPoint = false; for (final RouteResultsetNode node : session.getTargetKeys()) { if (node == null) { LOGGER.error("null is contained in RoutResultsetNodes, source = " + session.getSource()); continue; } final BackendConnection conn = session.getTarget(node); if (conn != null) { boolean isClosed=conn.isClosedOrQuit(); if(isClosed) { this.setFail("receive rollback,but find backend con is closed or quit"); // session.getSource().writeErrMessage(ErrorCode.ER_UNKNOWN_ERROR, // "receive rollback,but find backend con is closed or quit"); LOGGER.error( conn+"receive rollback,but fond backend con is closed or quit"); continue; } if (LOGGER.isDebugEnabled()) { LOGGER.debug("rollback job run for " + conn); } if (clearIfSessionClosed(session)) { return; } conn.setResponseHandler(RollbackNodeHandler.this); //support the XA rollback //to do to write xa recover log and judge xa statue to judge if send xa end if(session.getXaTXID()!=null && conn instanceof MySQLConnection) { MySQLConnection mysqlCon = (MySQLConnection) conn; //recovery log String xaTxId = session.getXaTXID(); CoordinatorLogEntry coordinatorLogEntry = MultiNodeCoordinator.inMemoryRepository.get(xaTxId); if(coordinatorLogEntry != null) { writeCheckPoint = true; //已经prepare的修改recover log for(int i=0; i fields, byte[] eof, BackendConnection conn) { LOGGER.error(new StringBuilder().append("unexpected packet for ") .append(conn).append(" bound by ").append(session.getSource()) .append(": field's eof").toString()); } @Override public void rowResponse(byte[] row, BackendConnection conn) { LOGGER.error(new StringBuilder().append("unexpected packet for ") .append(conn).append(" bound by ").append(session.getSource()) .append(": field's eof").toString()); } @Override public void writeQueueAvailable() { } public void connectionClose(BackendConnection conn, String reason) { this.setFail("closed connection:" + reason + " con:" + conn); boolean finished = false; if (finished == false) { finished = this.decrementCountBy(1); } if (error == null) { error = "back connection closed "; } if(finished) { session.setAutoCommitStatus(); tryErrorFinished(finished); } } protected void tryErrorFinished(boolean allEnd) { if (allEnd && !session.closed()) { // clear session resources,release all if (LOGGER.isDebugEnabled()) { LOGGER.debug("error all end ,clear session resource "); } // 关闭所有的错误后端连接 清理资源 session.closeAndClearResources(error); // 避免高并发 重新在清空一次 session.getSource().clearTxInterrupt(); // createErrPkg(this.error).write(session.getSource()); if (responseData != null) { LOGGER.error(this.error); session.getSource().write(responseData); } else session.getSource().writeErrMessage(ErrorCode.ER_UNKNOWN_ERROR, this.error); } } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/RollbackReleaseHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql.nio.handler; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.backend.BackendConnection; /** * @author mycat */ public class RollbackReleaseHandler implements ResponseHandler { private static final Logger logger = LoggerFactory .getLogger(RollbackReleaseHandler.class); public RollbackReleaseHandler() { } @Override public void connectionAcquired(BackendConnection conn) { logger.error("unexpected invocation: connectionAcquired from rollback-release"); } @Override public void connectionError(Throwable e, BackendConnection conn) { } @Override public void errorResponse(byte[] err, BackendConnection conn) { conn.quit(); } @Override public void okResponse(byte[] ok, BackendConnection conn) { logger.debug("autocomit is false,but no commit or rollback ,so mycat rollbacked backend conn "+conn); conn.release(); } @Override public void fieldEofResponse(byte[] header, List fields, byte[] eof, BackendConnection conn) { } @Override public void rowResponse(byte[] row, BackendConnection conn) { } @Override public void rowEofResponse(byte[] eof, BackendConnection conn) { } @Override public void writeQueueAvailable() { } @Override public void connectionClose(BackendConnection conn, String reason) { } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/SecondHandler.java ================================================ package io.mycat.backend.mysql.nio.handler; import java.util.List; /** * 查询分解后的第二部处理 * @author huangyiming * */ public interface SecondHandler { public void doExecute(List params); } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/SecondQueryHandler.java ================================================ package io.mycat.backend.mysql.nio.handler; import java.util.List; public class SecondQueryHandler implements SecondHandler { public MiddlerResultHandler middlerResultHandler; public SecondQueryHandler(MiddlerResultHandler middlerResultHandler){ this.middlerResultHandler = middlerResultHandler; } @Override public void doExecute(List params) { // TODO Auto-generated method stub } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/SimpleLogHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql.nio.handler; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.backend.BackendConnection; public class SimpleLogHandler implements ResponseHandler{ private static final Logger LOGGER = LoggerFactory .getLogger(SimpleLogHandler.class); @Override public void connectionError(Throwable e, BackendConnection conn) { LOGGER.warn(conn+" connectionError "+e); } @Override public void connectionAcquired(BackendConnection conn) { LOGGER.info("connectionAcquired "+conn); } @Override public void errorResponse(byte[] err, BackendConnection conn) { LOGGER.warn("caught error resp: " + conn + " " + bytesToHex(err)); } @Override public void okResponse(byte[] ok, BackendConnection conn) { LOGGER.info("okResponse: " + conn + "," + bytesToHex(ok) ); } @Override public void fieldEofResponse(byte[] header, List fields, byte[] eof, BackendConnection conn) { LOGGER.info("fieldEofResponse : " + conn ); LOGGER.info("SimpleLogHandler.fieldEofResponse header: " + bytesToHex(header) ); for(byte[] field : fields) { LOGGER.info("SimpleLogHandler.fieldEofResponse fields: " + bytesToHex(field) ); } LOGGER.info("SimpleLogHandler.fieldEofResponse eof: " + bytesToHex(eof) ); } @Override public void rowResponse(byte[] row, BackendConnection conn) { LOGGER.info("rowResponse: " + conn ); System.out.println("SimpleLogHandler.rowResponse: " + bytesToHex(row) ); } @Override public void rowEofResponse(byte[] eof, BackendConnection conn) { LOGGER.info("rowEofResponse: " + conn ); LOGGER.info("SimpleLogHandler.rowEofResponse: " + bytesToHex(eof) ); } @Override public void writeQueueAvailable() { } @Override public void connectionClose(BackendConnection conn, String reason) { } public static String bytesToHex(byte[] bytes) { StringBuilder sb = new StringBuilder(); for (byte b : bytes) { sb.append(String.format("%02x ", b)); } return sb.toString(); } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/SingleNodeHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql.nio.handler; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Strings; import io.mycat.MycatServer; import io.mycat.backend.BackendConnection; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.backend.mysql.LoadDataUtil; import io.mycat.backend.mysql.listener.SqlExecuteStage; import io.mycat.config.ErrorCode; import io.mycat.config.MycatConfig; import io.mycat.config.model.SchemaConfig; import io.mycat.net.mysql.BinaryRowDataPacket; import io.mycat.net.mysql.ErrorPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.OkPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.route.RouteResultset; import io.mycat.route.RouteResultsetNode; import io.mycat.server.NonBlockingSession; import io.mycat.server.ServerConnection; import io.mycat.server.parser.ServerParse; import io.mycat.server.parser.ServerParseShow; import io.mycat.server.response.ShowFullTables; import io.mycat.server.response.ShowTables; import io.mycat.statistic.stat.QueryResult; import io.mycat.statistic.stat.QueryResultDispatcher; import io.mycat.util.ResultSetUtil; import io.mycat.util.StringUtil; /** * @author mycat */ public class SingleNodeHandler implements ResponseHandler, Terminatable, LoadDataResponseHandler { private static final Logger LOGGER = LoggerFactory.getLogger(SingleNodeHandler.class); private final RouteResultsetNode node; private final RouteResultset rrs; private final NonBlockingSession session; // only one thread access at one time no need lock private volatile byte packetId; protected volatile ByteBuffer buffer; private volatile boolean isRunning; private Runnable terminateCallBack; protected long startTime; protected long netInBytes; protected long netOutBytes; private long selectRows; protected long affectedRows; protected final AtomicBoolean errorRepsponsed = new AtomicBoolean(false); private boolean prepared; protected int fieldCount; protected List fieldPackets = new ArrayList(); private volatile boolean isDefaultNodeShowTable; private volatile boolean isDefaultNodeShowFullTable; private Set shardingTablesSet; private byte[] header = null; private List fields = null; public SingleNodeHandler(RouteResultset rrs, NonBlockingSession session) { this.rrs = rrs; this.node = rrs.getNodes()[0]; if (node == null) { throw new IllegalArgumentException("routeNode is null!"); } if (session == null) { throw new IllegalArgumentException("session is null!"); } this.session = session; ServerConnection source = session.getSource(); String schema = source.getSchema(); if (schema != null && ServerParse.SHOW == rrs.getSqlType()) { SchemaConfig schemaConfig = MycatServer.getInstance().getConfig().getSchemas().get(schema); int type = ServerParseShow.tableCheck(rrs.getStatement(), 0); isDefaultNodeShowTable = (ServerParseShow.TABLES == type && !Strings.isNullOrEmpty(schemaConfig.getDataNode())); isDefaultNodeShowFullTable = (ServerParseShow.FULLTABLES == type && !Strings.isNullOrEmpty(schemaConfig.getDataNode())); if (isDefaultNodeShowTable) { shardingTablesSet = ShowTables.getTableSet(source, rrs.getStatement()); } else if (isDefaultNodeShowFullTable) { shardingTablesSet = ShowFullTables.getTableSet(source, rrs.getStatement()); } } if ( rrs != null && rrs.getStatement() != null) { netInBytes += rrs.getStatement().getBytes().length; } } public NonBlockingSession getSession() { return this.session; } public RouteResultsetNode getRouteResultsetNode() { return this.node; } public RouteResultset getRouteResultset(){ return this.rrs; } public ByteBuffer getBuffer() { return this.buffer; } @Override public void terminate(Runnable callback) { boolean zeroReached = false; if (isRunning) { terminateCallBack = callback; } else { zeroReached = true; } if (zeroReached) { callback.run(); } } protected void endRunning() { Runnable callback = null; if (isRunning) { isRunning = false; callback = terminateCallBack; terminateCallBack = null; } if (callback != null) { callback.run(); } } private void recycleResources() { ByteBuffer buf = buffer; if (buf != null) { session.getSource().recycle(buffer); buffer = null; } } public void execute() throws Exception { startTime=System.currentTimeMillis(); ServerConnection sc = session.getSource(); this.isRunning = true; if (rrs.isLoadData()) { this.packetId = session.getSource().getLoadDataInfileHandler().getLastPackId(); } else { this.packetId = 0; } final BackendConnection conn = session.getTarget(node); LOGGER.debug("rrs.getRunOnSlave() " + rrs.getRunOnSlaveDebugInfo()); node.setRunOnSlave(rrs.getRunOnSlave()); // 实现 master/slave注解 LOGGER.debug("node.getRunOnSlave() " + node.getRunOnSlaveDebugInfo()); try { if (session.tryExistsCon(conn, node)) { _execute(conn); } else { // create new connection MycatConfig conf = MycatServer.getInstance().getConfig(); LOGGER.debug("node.getRunOnSlave() " + node.getRunOnSlave()); node.setRunOnSlave(rrs.getRunOnSlave()); // 实现 master/slave注解 LOGGER.debug("node.getRunOnSlave() " + node.getRunOnSlave()); PhysicalDBNode dn = conf.getDataNodes().get(node.getName()); dn.getConnection(dn.getDatabase(), sc.isAutocommit(), node, this, node); } }catch (Exception e) { ServerConnection source = session.getSource(); LOGGER.warn(new StringBuilder().append(source).append(rrs).toString(), e); //设置错误 connectionError(e, null); } } @Override public void connectionAcquired(final BackendConnection conn) { session.bindConnection(node, conn); _execute(conn); } private void _execute(BackendConnection conn) { if (session.closed()) { endRunning(); session.clearResources(true); return; } conn.setResponseHandler(this); try { conn.execute(node, session.getSource(), session.getSource() .isAutocommit()); } catch (Exception e1) { executeException(conn, e1); return; } } private void executeException(BackendConnection c, Exception e) { ErrorPacket err = new ErrorPacket(); err.packetId = ++packetId; err.errno = ErrorCode.ERR_FOUND_EXCEPTION; String message = e.toString(); LOGGER.error(message); err.message = StringUtil.encode(message, session.getSource().getCharset()); this.backConnectionErr(err, c); } @Override public void connectionError(Throwable e, BackendConnection conn) { endRunning(); // ErrorPacket err = new ErrorPacket(); // err.packetId = ++packetId; // err.errno = ErrorCode.ER_NEW_ABORTING_CONNECTION; // err.message = StringUtil.encode(e.getMessage(), session.getSource().getCharset()); ServerConnection source = session.getSource(); // source.write(err.write(allocBuffer(), source, true)); //modify by zwy 2018.07 if(errorRepsponsed.compareAndSet(false, true)) { // source.setTxInterrupt(e.getMessage()); source.writeErrMessage(ErrorCode.ER_NEW_ABORTING_CONNECTION, e.getMessage()); } source.getListener().fireEvent(SqlExecuteStage.END); } @Override public void errorResponse(byte[] data, BackendConnection conn) { ErrorPacket err = new ErrorPacket(); err.read(data); err.packetId = ++packetId; backConnectionErr(err, conn); } private void backConnectionErr(ErrorPacket errPkg, BackendConnection conn) { endRunning(); ServerConnection source = session.getSource(); String errUser = source.getUser(); String errHost = source.getHost(); int errPort = source.getLocalPort(); String errmgs = " errno:" + errPkg.errno + " " + new String(errPkg.message); LOGGER.warn("execute sql err :" + errmgs + " con:" + conn + " frontend host:" + errHost + "/" + errPort + "/" + errUser); session.releaseConnectionIfSafe(conn, LOGGER.isDebugEnabled(), false); source.setTxInterrupt(errmgs); /** * TODO: 修复全版本BUG * * BUG复现： * 1、MysqlClient: SELECT 9223372036854775807 + 1; * 2、MyCatServer: ERROR 1690 (22003): BIGINT value is out of range in '(9223372036854775807 + 1)' * 3、MysqlClient: ERROR 2013 (HY000): Lost connection to MySQL server during query * * Fixed后 * 1、MysqlClient: SELECT 9223372036854775807 + 1; * 2、MyCatServer: ERROR 1690 (22003): BIGINT value is out of range in '(9223372036854775807 + 1)' * 3、MysqlClient: ERROR 1690 (22003): BIGINT value is out of range in '(9223372036854775807 + 1)' * */ // 由于 pakcetId != 1 造成的问题 //todo 统一调用writeErr errPkg.packetId = 1; //errPkg.write(source); //modify by zwy if (errorRepsponsed.compareAndSet(false, true)) { source.writeErrMessage(errPkg.errno, new String(errPkg.message)); } recycleResources(); source.getListener().fireEvent(SqlExecuteStage.END); } /** * insert/update/delete * * okResponse()：读取data字节数组，组成一个OKPacket，并调用ok.write(source)将结果写入前端连接FrontendConnection的写缓冲队列writeQueue中， * 真正发送给应用是由对应的NIOSocketWR从写队列中读取ByteBuffer并返回的 */ @Override public void okResponse(byte[] data, BackendConnection conn) { // this.netOutBytes += data.length; boolean executeResponse = conn.syncAndExcute(); if (executeResponse) { ServerConnection source = session.getSource(); OkPacket ok = new OkPacket(); ok.read(data); this.affectedRows += ok.affectedRows; if (ok.hasMoreResultsExists()) { // funnyAnt:当是批量update/delete语句，提示后面还有ok包 return; } boolean isCanClose2Client =(!rrs.isCallStatement()) ||(rrs.isCallStatement() &&!rrs.getProcedure().isResultSimpleValue()); if (rrs.isLoadData()) { // byte lastPackId = source.getLoadDataInfileHandler().getLastPackId(); ok.packetId = ++packetId;// OK_PACKET source.getLoadDataInfileHandler().clear(); } else if (isCanClose2Client) { ok.packetId = ++packetId;// OK_PACKET } if (isCanClose2Client) { session.releaseConnectionIfSafe(conn, LOGGER.isDebugEnabled(), false); endRunning(); } ok.serverStatus = source.isAutocommit() ? 2 : 1; recycleResources(); if (isCanClose2Client) { source.setLastInsertId(ok.insertId); //modify by zwy 2018.07 if(!errorRepsponsed.get() && !session.closed() && source.canResponse()) { ok.write(source); } } source.setExecuteSql(null); source.getListener().fireEvent(SqlExecuteStage.END); // add by lian // 解决sql统计中写操作永远为0 QueryResult queryResult = new QueryResult(session.getSource().getSchema(), session.getSource().getUser(), rrs.getSqlType(), rrs.getStatement(), affectedRows, netInBytes, netOutBytes, startTime, System.currentTimeMillis(),0, source.getHost()); QueryResultDispatcher.dispatchQuery( queryResult ); } } /** * select * * 行结束标志返回时触发，将EOF标志写入缓冲区，最后调用source.write(buffer)将缓冲区放入前端连接的写缓冲队列中，等待NIOSocketWR将其发送给应用 */ @Override public void rowEofResponse(byte[] eof, BackendConnection conn) { this.netOutBytes += eof.length; ServerConnection source = session.getSource(); conn.recordSql(source.getHost(), source.getSchema(), node.getStatement()); // 判断是调用存储过程的话不能在这里释放链接 if (!rrs.isCallStatement()||(rrs.isCallStatement()&&rrs.getProcedure().isResultSimpleValue())) { session.releaseConnectionIfSafe(conn, LOGGER.isDebugEnabled(), false); endRunning(); } eof[3] = ++packetId; buffer = source.writeToBuffer(eof, allocBuffer()); int resultSize = source.getWriteQueue().size()*MycatServer.getInstance().getConfig().getSystem().getBufferPoolPageSize(); resultSize=resultSize+buffer.position(); MiddlerResultHandler middlerResultHandler = session.getMiddlerResultHandler(); if(middlerResultHandler !=null ){ middlerResultHandler.secondEexcute(); } else{ //modify by zwy 2018.07 if(!errorRepsponsed.get()&& !session.closed()&& source.canResponse()) { source.write(buffer); } } source.setExecuteSql(null); source.getListener().fireEvent(SqlExecuteStage.END); //TODO: add by zhuam //查询结果派发 QueryResult queryResult = new QueryResult(session.getSource().getSchema(), session.getSource().getUser(), rrs.getSqlType(), rrs.getStatement(), affectedRows, netInBytes, netOutBytes, startTime, System.currentTimeMillis(),resultSize, source.getHost()); QueryResultDispatcher.dispatchQuery( queryResult ); } /** * lazy create ByteBuffer only when needed * * @return */ protected ByteBuffer allocBuffer() { if (buffer == null) { buffer = session.getSource().allocate(); } return buffer; } /** * select * * 元数据返回时触发，将header和元数据内容依次写入缓冲区中 */ @Override public void fieldEofResponse(byte[] header, List fields, byte[] eof, BackendConnection conn) { this.header = header; this.fields = fields; MiddlerResultHandler middlerResultHandler = session.getMiddlerResultHandler(); if(null !=middlerResultHandler ){ return; } this.netOutBytes += header.length; for (int i = 0, len = fields.size(); i < len; ++i) { byte[] field = fields.get(i); this.netOutBytes += field.length; } header[3] = ++packetId; ServerConnection source = session.getSource(); buffer = source.writeToBuffer(header, allocBuffer()); for (int i = 0, len = fields.size(); i < len; ++i) { byte[] field = fields.get(i); field[3] = ++packetId; // 保存field信息 FieldPacket fieldPk = new FieldPacket(); fieldPk.read(field); fieldPackets.add(fieldPk); buffer = source.writeToBuffer(field, buffer); } fieldCount = fieldPackets.size(); eof[3] = ++packetId; buffer = source.writeToBuffer(eof, buffer); if (isDefaultNodeShowTable) { for (String name : shardingTablesSet) { RowDataPacket row = new RowDataPacket(1); row.add(StringUtil.encode(name.toLowerCase(), source.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, source, true); } } else if (isDefaultNodeShowFullTable) { for (String name : shardingTablesSet) { RowDataPacket row = new RowDataPacket(1); row.add(StringUtil.encode(name.toLowerCase(), source.getCharset())); row.add(StringUtil.encode("BASE TABLE", source.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, source, true); } } } /** * select * * 行数据返回时触发，将行数据写入缓冲区中 */ @Override public void rowResponse(byte[] row, BackendConnection conn) { //已经有错误了直接不处理返回 modify by zwy2018.07 if(errorRepsponsed.get()) { return; } this.netOutBytes += row.length; this.selectRows++; if (isDefaultNodeShowTable || isDefaultNodeShowFullTable) { RowDataPacket rowDataPacket = new RowDataPacket(1); rowDataPacket.read(row); String table = StringUtil.decode(rowDataPacket.fieldValues.get(0), session.getSource().getCharset()); if (shardingTablesSet.contains(table.toUpperCase())) { return; } } row[3] = ++packetId; if ( prepared ) { RowDataPacket rowDataPk = new RowDataPacket(fieldCount); rowDataPk.read(row); BinaryRowDataPacket binRowDataPk = new BinaryRowDataPacket(); binRowDataPk.read(fieldPackets, rowDataPk); binRowDataPk.packetId = rowDataPk.packetId; // binRowDataPk.write(session.getSource()); /* * [fix bug] : 这里不能直接将包写到前端连接, * 因为在fieldEofResponse()方法结束后buffer还没写出, * 所以这里应该将包数据顺序写入buffer(如果buffer满了就写出),然后再将buffer写出 */ buffer = binRowDataPk.write(buffer, session.getSource(), true); } else { MiddlerResultHandler middlerResultHandler = session.getMiddlerResultHandler(); if(null ==middlerResultHandler ){ buffer = session.getSource().writeToBuffer(row, allocBuffer()); }else{ if(middlerResultHandler instanceof MiddlerQueryResultHandler){ byte[] rv = ResultSetUtil.getColumnVal(row, fields, 0); String rowValue = rv==null?"":new String(rv); middlerResultHandler.add(rowValue); } } } } @Override public void writeQueueAvailable() { } @Override public void connectionClose(BackendConnection conn, String reason) { ErrorPacket err = new ErrorPacket(); err.packetId = ++packetId; err.errno = ErrorCode.ER_ERROR_ON_CLOSE; LOGGER.error(reason); err.message = StringUtil.encode(reason, session.getSource() .getCharset()); this.backConnectionErr(err, conn); } public void clearResources() { } @Override public void requestDataResponse(byte[] data, BackendConnection conn) { LoadDataUtil.requestFileDataResponse(data, conn); } public boolean isPrepared() { return prepared; } public void setPrepared(boolean prepared) { this.prepared = prepared; } @Override public String toString() { return "SingleNodeHandler [node=" + node + ", packetId=" + packetId + "]"; } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/Terminatable.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.backend.mysql.nio.handler; /** * @author mycat */ public interface Terminatable { void terminate(Runnable runnable); } ================================================ FILE: src/main/java/io/mycat/backend/mysql/nio/handler/UnLockTablesHandler.java ================================================ package io.mycat.backend.mysql.nio.handler; import java.util.List; import java.util.Map; import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.backend.BackendConnection; import io.mycat.net.mysql.OkPacket; import io.mycat.route.RouteResultsetNode; import io.mycat.server.NonBlockingSession; import io.mycat.server.parser.ServerParse; /** * unlock tables 语句处理器 * @author songdabin * */ public class UnLockTablesHandler extends MultiNodeHandler implements ResponseHandler { private static final Logger LOGGER = LoggerFactory.getLogger(UnLockTablesHandler.class); private final NonBlockingSession session; private final boolean autocommit; private final String srcStatement; public UnLockTablesHandler(NonBlockingSession session, boolean autocommit, String sql) { super(session); this.session = session; this.autocommit = autocommit; this.srcStatement = sql; } public void execute() { Map lockedConns = session.getTargetMap(); Set dnSet = lockedConns.keySet(); this.reset(lockedConns.size()); // 客户端直接发送unlock tables命令，由于之前未发送lock tables语句，无法获取后端绑定的连接，此时直接返回OK包 if (lockedConns.size() == 0) { LOGGER.warn("find no locked backend connection!"+session.getSource()); OkPacket ok = new OkPacket(); ok.packetId = ++ packetId; ok.packetLength = 7; // unlock table 命令返回MySQL协议包长度为7 ok.serverStatus = session.getSource().isAutocommit() ? 2:1; ok.write(session.getSource()); return; } for (RouteResultsetNode dataNode : dnSet) { RouteResultsetNode node = new RouteResultsetNode(dataNode.getName(), ServerParse.UNLOCK, srcStatement); BackendConnection conn = lockedConns.get(dataNode); if (clearIfSessionClosed(session)) { return; } conn.setResponseHandler(this); try { conn.execute(node, session.getSource(), autocommit); } catch (Exception e) { connectionError(e, conn); } } } @Override public void connectionError(Throwable e, BackendConnection conn) { super.connectionError(e, conn); } @Override public void connectionAcquired(BackendConnection conn) { LOGGER.error("unexpected invocation: connectionAcquired from unlock tables"); } @Override public void errorResponse(byte[] err, BackendConnection conn) { super.errorResponse(err, conn); } @Override public void okResponse(byte[] data, BackendConnection conn) { boolean executeResponse = conn.syncAndExcute(); if (executeResponse) { boolean isEndPack = decrementCountBy(1); session.releaseConnection(conn); if (isEndPack) { if (this.isFail() || session.closed()) { tryErrorFinished(true); return; } OkPacket ok = new OkPacket(); ok.read(data); lock.lock(); try { ok.packetId = ++ packetId; ok.serverStatus = session.getSource().isAutocommit() ? 2:1; } finally { lock.unlock(); } ok.write(session.getSource()); } } } @Override public void fieldEofResponse(byte[] header, List fields, byte[] eof, BackendConnection conn) { LOGGER.error(new StringBuilder().append("unexpected packet for ") .append(conn).append(" bound by ").append(session.getSource()) .append(": field's eof").toString()); } @Override public void rowResponse(byte[] row, BackendConnection conn) { LOGGER.warn(new StringBuilder().append("unexpected packet for ") .append(conn).append(" bound by ").append(session.getSource()) .append(": row data packet").toString()); } @Override public void rowEofResponse(byte[] eof, BackendConnection conn) { LOGGER.error(new StringBuilder().append("unexpected packet for ") .append(conn).append(" bound by ").append(session.getSource()) .append(": row's eof").toString()); } @Override public void writeQueueAvailable() { // TODO Auto-generated method stub } @Override public void connectionClose(BackendConnection conn, String reason) { // TODO Auto-generated method stub } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/xa/CoordinatorLogEntry.java ================================================ package io.mycat.backend.mysql.xa; import io.mycat.util.TimeUtil; import java.io.Serializable; /** * Created by zhangchao on 2016/10/17. */ public class CoordinatorLogEntry implements Serializable { private static final long serialVersionUID = -919666492191340531L; public final String id; // public final boolean wasCommitted; public long createTime; public final ParticipantLogEntry[] participants; public CoordinatorLogEntry(String coordinatorId, ParticipantLogEntry[] participantDetails) { this(coordinatorId, false, participantDetails, null, TimeUtil.currentTimeMillis()); } public CoordinatorLogEntry(String coordinatorId, boolean wasCommitted, ParticipantLogEntry[] participants) { createTime = TimeUtil.currentTimeMillis(); this.id = coordinatorId; // this.wasCommitted = wasCommitted; this.participants = participants; } public CoordinatorLogEntry(String coordinatorId, boolean wasCommitted, ParticipantLogEntry[] participants, String superiorCoordinatorId, long creteTime) { this.createTime = creteTime; this.id = coordinatorId; // this.wasCommitted = wasCommitted; this.participants = participants; } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/xa/Deserializer.java ================================================ package io.mycat.backend.mysql.xa; import io.mycat.backend.mysql.xa.recovery.DeserialisationException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; /** * Created by zhangchao on 2016/10/17. */ public class Deserializer { private static final String JSON_ARRAY_END = "]"; private static final String JSON_ARRAY_START = "["; private static final String OBJECT_START= "{"; private static final String OBJECT_END= "}"; List tokenize(String content) { List result = new ArrayList(); int endObject = content.indexOf(OBJECT_END); while(endObject >0){ String object = content.substring(0,endObject+1); result.add(object); content = content.substring(endObject+1); endObject = content.indexOf(OBJECT_END); } return result; } String extractArrayPart(String content) { if(!content.contains(JSON_ARRAY_START) && !content.contains(JSON_ARRAY_END)) { //no array... return ""; } //else int start=content.indexOf(JSON_ARRAY_START); int end=content.indexOf(JSON_ARRAY_END); return content.substring(start+1, end); } public CoordinatorLogEntry fromJSON(String coordinatorLogEntryStr) throws DeserialisationException { try { String jsonContent = coordinatorLogEntryStr.trim(); validateJSONContent(jsonContent); Map header = extractHeader(jsonContent); String coordinatorId = header.get("id"); String arrayContent = extractArrayPart(jsonContent); List elements = tokenize(arrayContent); ParticipantLogEntry[] participantLogEntries = new ParticipantLogEntry[elements.size()]; for (int i = 0; i < participantLogEntries.length; i++) { participantLogEntries[i]=recreateParticipantLogEntry(coordinatorId,elements.get(i)); } CoordinatorLogEntry actual = new CoordinatorLogEntry(header.get("id"),Boolean.valueOf(header.get("wasCommitted")), participantLogEntries,header.get("superiorCoordinatorId"), Long.valueOf(header.get("createTime"))); return actual; } catch (Exception unexpectedEOF) { throw new DeserialisationException(coordinatorLogEntryStr); } } private void validateJSONContent(String coordinatorLogEntryStr) throws DeserialisationException { if (!coordinatorLogEntryStr.startsWith(OBJECT_START)){ throw new DeserialisationException(coordinatorLogEntryStr); } if (!coordinatorLogEntryStr.endsWith(OBJECT_END)){ throw new DeserialisationException(coordinatorLogEntryStr); } } private Map extractHeader(String coordinatorLogEntryStr) { Map header = new HashMap(2); String[] attributes = coordinatorLogEntryStr.split(","); for (String attribute : attributes) { String[] pair = attribute.split(":"); header.put(pair[0].replaceAll("\\{", "").replace("\"", ""), pair[1].replace("\"", "")); } return header; } ParticipantLogEntry recreateParticipantLogEntry(String coordinatorId, String participantLogEntry) { participantLogEntry = participantLogEntry.replaceAll("\\{", "").replaceAll("\\}", ""); Map content = new HashMap(5); String[] attributes = participantLogEntry.split(","); for (String attribute : attributes) { String[] pair = attribute.split(":"); if(pair.length>1){ content.put(pair[0].replace("\"", ""), pair[1].replace("\"", "")); } } ParticipantLogEntry actual = new ParticipantLogEntry(coordinatorId, content.get("uri"), Long.valueOf(content.get("expires")), content.get("resourceName"), Integer.parseInt(content.get("state"))); return actual; } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/xa/LogFileLock.java ================================================ package io.mycat.backend.mysql.xa; import io.mycat.backend.mysql.xa.recovery.LogException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; /** * Created by zhangchao on 2016/10/17. */ public class LogFileLock { public static final Logger logger = LoggerFactory .getLogger(LogFileLock.class); private static final String FILE_SEPARATOR = String.valueOf(File.separatorChar); private File lockfileToPreventDoubleStartup_; private FileOutputStream lockfilestream_ = null; private FileLock lock_ = null; private String dir; private String fileName; public LogFileLock(String dir, String fileName) { if(!dir.endsWith(FILE_SEPARATOR)) { dir += FILE_SEPARATOR; } this.dir = dir; this.fileName = fileName; } public void acquireLock() throws LogException { try { File parent = new File(dir); if(!parent.exists()) { parent.mkdir(); } lockfileToPreventDoubleStartup_ = new File(dir, fileName + ".lck"); lockfilestream_ = new FileOutputStream(lockfileToPreventDoubleStartup_); lock_ = lockfilestream_.getChannel().tryLock(); lockfileToPreventDoubleStartup_.deleteOnExit(); } catch (OverlappingFileLockException failedToGetLock) { // happens on windows lock_ = null; } catch (IOException failedToGetLock) { // happens on windows lock_ = null; } if (lock_ == null) { logger.error("ERROR: the specified log seems to be in use already: " + fileName + " in " + dir + ". Make sure that no other instance is running, or kill any pending process if needed."); throw new LogException("Log already in use? " + fileName + " in "+ dir); } } public void releaseLock() { try { if (lock_ != null) { lock_.release(); } if (lockfilestream_ != null) lockfilestream_.close(); } catch (IOException e) { logger.warn("Error releasing file lock: " + e.getMessage()); } finally { lock_ = null; } if (lockfileToPreventDoubleStartup_ != null) { lockfileToPreventDoubleStartup_.delete(); lockfileToPreventDoubleStartup_ = null; } } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/xa/ParticipantLogEntry.java ================================================ package io.mycat.backend.mysql.xa; import java.io.Serializable; /** * Created by zhangchao on 2016/10/17. */ public class ParticipantLogEntry implements Serializable { private static final long serialVersionUID = 1728296701394899871L; /** * The ID of the global transaction as known by the transaction core. */ public String coordinatorId; /** * Identifies the participant within the global transaction. */ public String uri; /** * When does this participant expire (expressed in millis since Jan 1, 1970)? */ public long expires; /** * Best-known state of the participant. */ public int txState; /** * For diagnostic purposes, null if not relevant. */ public String resourceName; public ParticipantLogEntry(String coordinatorId, String uri, long expires, String resourceName, int txState) { this.coordinatorId = coordinatorId; this.uri = uri; this.expires = expires; this.resourceName = resourceName; this.txState = txState; } @Override public boolean equals(Object other) { boolean ret = false; if (other instanceof ParticipantLogEntry) { ParticipantLogEntry o = (ParticipantLogEntry) other; if (o.coordinatorId.equals(coordinatorId) && o.uri.equals(uri)) ret = true; } return ret; } @Override public int hashCode() { return coordinatorId.hashCode(); } @Override public String toString() { return "ParticipantLogEntry [id=" + coordinatorId + ", uri=" + uri + ", expires=" + expires + ", state=" + txState + ", resourceName=" + resourceName + "]"; } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/xa/Serializer.java ================================================ package io.mycat.backend.mysql.xa; /** * Created by zhangchao on 2016/10/17. */ public class Serializer { private static final String PROPERTY_SEPARATOR = ","; private static final String QUOTE = "\""; private static final String END_ARRAY = "]"; private static final String START_ARRAY = "["; private static final String START_OBJECT = "{"; private static final String END_OBJECT = "}"; private static final String LINE_SEPARATOR = System.getProperty("line.separator"); public String toJSON(CoordinatorLogEntry coordinatorLogEntry) { StringBuilder strBuilder = new StringBuilder(600); strBuilder.append(START_OBJECT); strBuilder.append(QUOTE).append("id").append(QUOTE).append(":").append(QUOTE).append(coordinatorLogEntry.id).append(QUOTE); strBuilder.append(PROPERTY_SEPARATOR); strBuilder.append(QUOTE).append("createTime").append(QUOTE).append(":").append(QUOTE).append(coordinatorLogEntry.createTime).append(QUOTE); strBuilder.append(PROPERTY_SEPARATOR); //strBuilder.append(QUOTE).append("wasCommitted").append(QUOTE).append(":").append(coordinatorLogEntry.wasCommitted); //strBuilder.append(PROPERTY_SEPARATOR); String prefix = ""; if(coordinatorLogEntry.participants.length>0){ strBuilder.append(QUOTE).append("participants").append(QUOTE); strBuilder.append(":"); strBuilder.append(START_ARRAY); for(ParticipantLogEntry participantLogEntry :coordinatorLogEntry.participants){ if(participantLogEntry==null){continue;} strBuilder.append(prefix); prefix = PROPERTY_SEPARATOR; strBuilder.append(START_OBJECT); strBuilder.append(QUOTE).append("uri").append(QUOTE).append(":").append(QUOTE).append(participantLogEntry.uri).append(QUOTE); strBuilder.append(PROPERTY_SEPARATOR); strBuilder.append(QUOTE).append("state").append(QUOTE).append(":").append(QUOTE).append(participantLogEntry.txState).append(QUOTE); strBuilder.append(PROPERTY_SEPARATOR); strBuilder.append(QUOTE).append("expires").append(QUOTE).append(":").append(participantLogEntry.expires); if (participantLogEntry.resourceName!=null) { strBuilder.append(PROPERTY_SEPARATOR); strBuilder.append(QUOTE).append("resourceName").append(QUOTE).append(":").append(QUOTE).append(participantLogEntry.resourceName).append(QUOTE); } strBuilder.append(END_OBJECT); } // for (ParticipantLogEntry participantLogEntry : coordinatorLogEntry.participants) { // // } strBuilder.append(END_ARRAY); } strBuilder.append(END_OBJECT); strBuilder.append(LINE_SEPARATOR); return strBuilder.toString(); } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/xa/TxState.java ================================================ package io.mycat.backend.mysql.xa; /** * Created by zhangchao on 2016/10/13. */ public class TxState { /** XA INIT STATUS **/ public static final int TX_INITIALIZE_STATE = 0; /** XA STARTED STATUS **/ public static final int TX_STARTED_STATE = 1; /** XA is prepared **/ public static final int TX_PREPARED_STATE = 2; /** XA is commited **/ public static final int TX_COMMITED_STATE = 3; /** XA is rollbacked **/ public static final int TX_ROLLBACKED_STATE = 4; } ================================================ FILE: src/main/java/io/mycat/backend/mysql/xa/VersionedFile.java ================================================ package io.mycat.backend.mysql.xa; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FilenameFilter; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.channels.FileChannel; /** * Created by zhangchao on 2016/10/17. */ public class VersionedFile { private static final String FILE_SEPARATOR = String.valueOf(File.separatorChar); private String baseDir; private String suffix; private String baseName; //state attributes below private long version; private FileInputStream inputStream; private RandomAccessFile randomAccessFile; /** * Creates a new instance based on the given name parameters. * The actual complete name(s) of the physical file(s) will be based on a version number * inserted in between, to identify versions. * * @param baseDir The base folder. * @param baseName The base name for of the file path/name. * @param suffix The suffix to append to the complete file name. */ public VersionedFile ( String baseDir , String baseName , String suffix ) { if(!baseDir.endsWith(FILE_SEPARATOR)) { baseDir += FILE_SEPARATOR; } this.baseDir = baseDir; this.suffix = suffix; this.baseName = baseName; resetVersion(); } private void resetVersion() { this.version = extractLastValidVersionNumberFromFileNames(); } private long extractLastValidVersionNumberFromFileNames() { long version = -1; File cd = new File ( getBaseDir() ); String[] names = cd.list ( new FilenameFilter() { public boolean accept ( File dir , String name ) { return (name.startsWith ( getBaseName() ) && name .endsWith ( getSuffix() )); } } ); if ( names!= null ) { for ( int i = 0; i < names.length; i++ ) { long sfx = extractVersion ( names[i] ); if ( version < 0 || sfx < version ) version = sfx; } } return version; } private long extractVersion ( String name ) { long ret = 0; int lastpos = name.lastIndexOf ( '.' ); int startpos = getBaseName().length (); String suffix = name.substring ( startpos, lastpos ); try { ret = Long.valueOf( suffix ); } catch ( NumberFormatException e ) { IllegalArgumentException err = new IllegalArgumentException ( "Error extracting version from file: " + name+" in " + getBaseDir() ); err.initCause ( e ); throw err; } return ret; } private String getBackupVersionFileName() { return getBaseUrl() + (version - 1) + getSuffix(); } public String getCurrentVersionFileName() { return getBaseUrl() + version + getSuffix(); } public String getBaseUrl() { return baseDir + baseName; } public String getBaseDir() { return this.baseDir; } public String getBaseName() { return this.baseName; } public String getSuffix() { return this.suffix; } /** * Opens the last valid version for reading. * * @return A stream to read the last valid contents * of the file: either the backup version (if present) * or the current (and only) version if no backup is found. * * @throws IllegalStateException If a newer version was opened for writing. * @throws FileNotFoundException If no last version was found. */ public FileInputStream openLastValidVersionForReading() throws IllegalStateException, FileNotFoundException { if ( randomAccessFile != null ) throw new IllegalStateException ( "Already started writing." ); inputStream = new FileInputStream ( getCurrentVersionFileName() ); return inputStream; } /** * Opens a new version for writing to. Note that * this new version is tentative and cannot be read * by {@link #openLastValidVersionForReading()} until * {@link #discardBackupVersion()} is called. * * @return A stream for writing to. * @throws IllegalStateException If called more than once * without a close in between. * @throws IOException If the file cannot be opened for writing. */ public FileOutputStream openNewVersionForWriting() throws IOException { openNewVersionForNioWriting(); return new FileOutputStream(randomAccessFile.getFD()); } /** * Opens a new version for writing to. Note that * this new version is tentative and cannot be read * by {@link #openLastValidVersionForReading()} until * {@link #discardBackupVersion()} is called. * * @return A file for writing to. * @throws IOException * * @throws IllegalStateException If called more than once * without a close in between. * @throws FileNotFoundException If the file cannot be opened for writing. * @throws IOException */ public FileChannel openNewVersionForNioWriting() throws FileNotFoundException { if ( randomAccessFile != null ) throw new IllegalStateException ( "Already writing a new version." ); //version++; randomAccessFile = new RandomAccessFile(getCurrentVersionFileName(), "rw"); return randomAccessFile.getChannel(); } /** * Discards the backup version (if any). * After calling this method, the newer version * produced after calling {@link #openNewVersionForWriting()} * becomes valid for reading next time when * {@link #openLastValidVersionForReading()} is called. * * Note: it is the caller's responsibility to make sure that * all new data has been flushed to disk before calling this method! * * @throws IllegalStateException If {@link #openNewVersionForWriting()} has not been called yet. * @throws IOException If the previous version exists but could no be deleted. */ public void discardBackupVersion() throws IllegalStateException, IOException { if ( randomAccessFile == null ) throw new IllegalStateException ( "No new version yet!" ); String fileName = getBackupVersionFileName(); File temp = new File ( fileName ); if ( temp.exists() && !temp.delete() ) throw new IOException ( "Failed to delete backup version: " + fileName ); } /** * Closes any open resources and resets the file for reading again. * @throws IOException If the output stream could not be closed. */ public void close() throws IOException { resetVersion(); if ( inputStream != null ) { try { inputStream.close(); } catch (IOException e) { //don't care and won't happen: closing an input stream //does nothing says the JDK javadoc! } finally { inputStream = null; } } if ( randomAccessFile != null ) { try { if ( randomAccessFile.getFD().valid() ) randomAccessFile.close(); } finally { randomAccessFile = null; } } } public long getSize() { long res = -1; File f = new File ( getCurrentVersionFileName() ); res = f.length(); return res; } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/xa/XACommitCallback.java ================================================ package io.mycat.backend.mysql.xa; import io.mycat.backend.mysql.nio.handler.MultiNodeCoordinator; import io.mycat.sqlengine.SQLQueryResult; import io.mycat.sqlengine.SQLQueryResultListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Map; /** * Created by zhangwy on 2018/12/16. */ public class XACommitCallback implements SQLQueryResultListener>> { private static final Logger LOGGER = LoggerFactory.getLogger(XACommitCallback.class); private final String xaId; private final ParticipantLogEntry participantLogEntry; public XACommitCallback(String xaId, ParticipantLogEntry participantLogEntry) { this.xaId = xaId; this.participantLogEntry = participantLogEntry; } public void onResult(SQLQueryResult> result) { // SQLQueryResult> queryRestl=new SQLQueryResult>(this.result,!failed, dataNode,errorMsg); if (result.isSuccess()) { //recovery log LOGGER.debug("onResult success on xa commit {},{}", xaId, participantLogEntry.resourceName); CoordinatorLogEntry coordinatorLogEntry = MultiNodeCoordinator.inMemoryRepository.get(xaId); for (int i = 0; i < coordinatorLogEntry.participants.length; i++) { if (coordinatorLogEntry.participants[i].resourceName.equals(participantLogEntry.resourceName)) { coordinatorLogEntry.participants[i].txState = TxState.TX_COMMITED_STATE; } } MultiNodeCoordinator.inMemoryRepository.put(xaId, coordinatorLogEntry); MultiNodeCoordinator.fileRepository.writeCheckpoint(xaId, MultiNodeCoordinator.inMemoryRepository.getAllCoordinatorLogEntries()); } else { String errorMsg = result.getErrMsg(); LOGGER.error(errorMsg); if (errorMsg.indexOf("Unknown XID") > -1) { //todo unknow xaId how do CoordinatorLogEntry coordinatorLogEntry = MultiNodeCoordinator.inMemoryRepository.get(xaId); for (int i = 0; i < coordinatorLogEntry.participants.length; i++) { if (coordinatorLogEntry.participants[i].resourceName.equals(participantLogEntry.resourceName)) { coordinatorLogEntry.participants[i].txState = TxState.TX_COMMITED_STATE; } } MultiNodeCoordinator.inMemoryRepository.put(xaId, coordinatorLogEntry); MultiNodeCoordinator.fileRepository.writeCheckpoint(xaId, MultiNodeCoordinator.inMemoryRepository.getAllCoordinatorLogEntries()); } } LOGGER.debug("[CALLBACK][XA COMMIT] when Mycat start"); } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/xa/XARollbackCallback.java ================================================ package io.mycat.backend.mysql.xa; import io.mycat.backend.mysql.nio.handler.MultiNodeCoordinator; import io.mycat.sqlengine.SQLQueryResult; import io.mycat.sqlengine.SQLQueryResultListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Map; /** * Created by zhangchao on 2016/10/18. */ public class XARollbackCallback implements SQLQueryResultListener>> { private static final Logger LOGGER = LoggerFactory.getLogger(XARollbackCallback.class); private final String xaId; private final ParticipantLogEntry participantLogEntry; public XARollbackCallback(String xaId, ParticipantLogEntry participantLogEntry) { this.xaId = xaId; this.participantLogEntry = participantLogEntry; } public void onResult(SQLQueryResult> result) { // SQLQueryResult> queryRestl=new SQLQueryResult>(this.result,!failed, dataNode,errorMsg); if(result.isSuccess()) { LOGGER.debug("onResult success on xa rollback {},{}", xaId, participantLogEntry.resourceName); //recovery log CoordinatorLogEntry coordinatorLogEntry = MultiNodeCoordinator.inMemoryRepository.get(xaId); for(int i=0; i -1) { //todo unknow xaId CoordinatorLogEntry coordinatorLogEntry = MultiNodeCoordinator.inMemoryRepository.get(xaId); for(int i=0; i findAllCommittingCoordinatorLogEntries() ; Collection getAllCoordinatorLogEntries() ; void writeCheckpoint(String id, Collection checkpointContent) ; void close(); } ================================================ FILE: src/main/java/io/mycat/backend/mysql/xa/recovery/impl/FileSystemRepository.java ================================================ package io.mycat.backend.mysql.xa.recovery.impl; import java.io.BufferedReader; import java.io.EOFException; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.ObjectStreamException; import java.io.StreamCorruptedException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.backend.mysql.xa.CoordinatorLogEntry; import io.mycat.backend.mysql.xa.Deserializer; import io.mycat.backend.mysql.xa.Serializer; import io.mycat.backend.mysql.xa.VersionedFile; import io.mycat.backend.mysql.xa.recovery.DeserialisationException; import io.mycat.backend.mysql.xa.recovery.Repository; import io.mycat.config.MycatConfig; import io.mycat.config.model.SystemConfig; /** * Created by zhangchao on 2016/10/13. */ public class FileSystemRepository implements Repository{ public static final Logger logger = LoggerFactory .getLogger(FileSystemRepository.class); private VersionedFile file; private FileChannel rwChannel = null; private Map writeStorage = new HashMap(); public FileSystemRepository() { init(); } @Override public void init(){ // ConfigProperties configProperties = Configuration.getConfigProperties(); // String baseDir = configProperties.getLogBaseDir(); // String baseName = configProperties.getLogBaseName(); MycatConfig mycatconfig = MycatServer.getInstance().getConfig(); SystemConfig systemConfig = mycatconfig.getSystem(); String baseDir =systemConfig.getXARecoveryLogBaseDir(); String baseName = systemConfig.getXARecoveryLogBaseName(); logger.debug("baseDir " + baseDir); logger.debug("baseName " + baseName); //Judge whether exist the basedir createBaseDir(baseDir); file = new VersionedFile(baseDir, baseName, ".log"); } private Serializer serializer = new Serializer(); /*没有被调用*/ @Override public void put(String id, CoordinatorLogEntry coordinatorLogEntry) { // try { // initChannelIfNecessary(); // write(coordinatorLogEntry, true); // } catch (IOException e) { // logger.error(e.getMessage(),e); // } } private synchronized void initChannelIfNecessary() throws FileNotFoundException { if (rwChannel == null) { rwChannel = file.openNewVersionForNioWriting(); } } private int write(CoordinatorLogEntry coordinatorLogEntry, boolean flushImmediately) throws IOException { String str = serializer.toJSON(coordinatorLogEntry); //缓存一下 writeStorage.put(coordinatorLogEntry.id, str); // logger.info(str); byte[] buffer = str.getBytes(); ByteBuffer buff = ByteBuffer.wrap(buffer); writeToFile(buff, flushImmediately); return buffer.length; } private synchronized void writeToFile(ByteBuffer buff, boolean force) throws IOException { rwChannel.write(buff); rwChannel.force(force); } @Override public CoordinatorLogEntry get(String coordinatorId) { throw new UnsupportedOperationException(); } @Override public Collection findAllCommittingCoordinatorLogEntries() { throw new UnsupportedOperationException(); } @Override public Collection getAllCoordinatorLogEntries() { FileInputStream fis = null; try { fis = file.openLastValidVersionForReading(); } catch (FileNotFoundException firstStart) { // the file could not be opened for reading; // merely return the default empty vector } if (fis != null) { return readFromInputStream(fis); } //else return Collections.emptyList(); } public static Collection readFromInputStream( InputStream in) { Map coordinatorLogEntries = new HashMap(); BufferedReader br = null; try { InputStreamReader isr = new InputStreamReader(in); br = new BufferedReader(isr); coordinatorLogEntries = readContent(br); } catch (Exception e) { logger.error("Error in recover", e); } finally { closeSilently(br); } return coordinatorLogEntries.values(); } static Map readContent(BufferedReader br) throws IOException { Map coordinatorLogEntries = new HashMap(); try { String line; while ((line = br.readLine()) != null) { CoordinatorLogEntry coordinatorLogEntry = deserialize(line); coordinatorLogEntries.put(coordinatorLogEntry.id, coordinatorLogEntry); } } catch (EOFException unexpectedEOF) { logger.info( "Unexpected EOF - logfile not closed properly last time?", unexpectedEOF); // merely return what was read so far... } catch (StreamCorruptedException unexpectedEOF) { logger.info( "Unexpected EOF - logfile not closed properly last time?", unexpectedEOF); // merely return what was read so far... } catch (ObjectStreamException unexpectedEOF) { logger.info( "Unexpected EOF - logfile not closed properly last time?", unexpectedEOF); // merely return what was read so far... } catch (DeserialisationException unexpectedEOF) { logger.info("Unexpected EOF - logfile not closed properly last time? " + unexpectedEOF); } return coordinatorLogEntries; } private static void closeSilently(BufferedReader fis) { try { if (fis != null) fis.close(); } catch (IOException io) { logger.warn("Fail to close logfile after reading - ignoring"); } } private static Deserializer deserializer = new Deserializer(); private static CoordinatorLogEntry deserialize(String line) throws DeserialisationException { return deserializer.fromJSON(line); } @Override public void close() { try { closeOutput(); } catch (Exception e) { logger.warn("Error closing file - ignoring", e); } } protected void closeOutput() throws IllegalStateException { try { if (file != null) { file.close(); } } catch (IOException e) { throw new IllegalStateException("Error closing previous output", e); } } @Override public synchronized void writeCheckpoint(String id, Collection checkpointContent) { try { if(rwChannel == null) { initChannelIfNecessary(); } // closeOutput(); // rwChannel = file.openNewVersionForNioWriting(); //判断xaId这条记录是否被修改 boolean isUpdate = true; for (CoordinatorLogEntry coordinatorLogEntry : checkpointContent) { if(coordinatorLogEntry.id.equals(id)) { isUpdate = checkForUpdate(id, coordinatorLogEntry); break; } } if(isUpdate == false ){ return ; } //清空所有的缓存 writeStorage.clear(); rwChannel.position(0); long writeSize = 0 ; for (CoordinatorLogEntry coordinatorLogEntry : checkpointContent) { writeSize += write(coordinatorLogEntry, false); } // logger.info("xaId {} writeCheckpoint {}",id, writeStorage.get(id)); rwChannel.truncate(writeSize); rwChannel.force(true); file.discardBackupVersion(); } catch (FileNotFoundException firstStart) { // the file could not be opened for reading; // merely return the default empty vector } catch (Exception e) { logger.error("Failed to write checkpoint", e); } } private boolean checkForUpdate(String id, CoordinatorLogEntry coordinatorLogEntry) { String backCoordinatorLogEntryStr = writeStorage.get(id); String str = serializer.toJSON(coordinatorLogEntry); if(null == backCoordinatorLogEntryStr || !str.equals(backCoordinatorLogEntryStr)) { writeStorage.put(id, str); return true; } return false; } /** * create the log base dir * @param baseDir */ public void createBaseDir(String baseDir){ File baseDirFolder = new File (baseDir); if (!baseDirFolder.exists()){ baseDirFolder.mkdirs(); } } } ================================================ FILE: src/main/java/io/mycat/backend/mysql/xa/recovery/impl/InMemoryRepository.java ================================================ package io.mycat.backend.mysql.xa.recovery.impl; import io.mycat.backend.mysql.nio.handler.MultiNodeCoordinator; import io.mycat.backend.mysql.xa.CoordinatorLogEntry; import io.mycat.backend.mysql.xa.ParticipantLogEntry; import io.mycat.backend.mysql.xa.TxState; import io.mycat.backend.mysql.xa.recovery.Repository; import java.util.Collection; import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; /** * Created by zhangchao on 2016/10/18. */ public class InMemoryRepository implements Repository { private Map storage = new ConcurrentHashMap(); private boolean closed = true; long count = 0 ; @Override public void init() { closed=false; } @Override public synchronized void put(String id, CoordinatorLogEntry coordinatorLogEntry) { count++ ; if(count > 1000){ count = 0; clear(id); } storage.put(id, coordinatorLogEntry); } private void clear(String id) { Collection checkpointContent = storage.values();; for (CoordinatorLogEntry coordinatorLogEntry : checkpointContent) { ParticipantLogEntry[] participants = coordinatorLogEntry.participants; boolean hasAllFinish = true; for(int i = 0 ; i < participants.length; i++) { if(participants[i].txState != TxState.TX_ROLLBACKED_STATE && participants[i].txState != TxState.TX_COMMITED_STATE) { hasAllFinish = false; break; } } if(hasAllFinish && !id.equals(coordinatorLogEntry.id)) { storage.remove(coordinatorLogEntry.id); // ((FileSystemRepository)MultiNodeCoordinator.fileRepository).writeStorage.remove(id); } } } @Override public synchronized CoordinatorLogEntry get(String coordinatorId) { return storage.get(coordinatorId); } @Override public synchronized Collection findAllCommittingCoordinatorLogEntries() { // Set res = new HashSet(); // Collection allCoordinatorLogEntry = storage.values(); // for (CoordinatorLogEntry coordinatorLogEntry : allCoordinatorLogEntry) { // if(coordinatorLogEntry.getResultingState() == TxState.TX_PREPARED_STATE){ // res.add(coordinatorLogEntry); // } // } // return res; return null; } @Override public void close() { storage.clear(); closed=true; } @Override public Collection getAllCoordinatorLogEntries() { return storage.values(); } @Override public void writeCheckpoint( String id, Collection checkpointContent) { storage.clear(); for (CoordinatorLogEntry coordinatorLogEntry : checkpointContent) { storage.put(coordinatorLogEntry.id, coordinatorLogEntry); } } public boolean isClosed() { return closed; } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/PostgreSQLBackendConnection.java ================================================ package io.mycat.backend.postgresql; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.nio.channels.NetworkChannel; import java.nio.channels.SocketChannel; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import io.mycat.backend.jdbc.ShowVariables; import io.mycat.backend.mysql.CharsetUtil; import io.mycat.backend.mysql.nio.MySQLConnectionHandler; import io.mycat.backend.mysql.nio.handler.ResponseHandler; import io.mycat.backend.postgresql.packet.Query; import io.mycat.backend.postgresql.packet.Terminate; import io.mycat.backend.postgresql.utils.PIOUtils; import io.mycat.backend.postgresql.utils.PacketUtils; import io.mycat.backend.postgresql.utils.PgSqlApaterUtils; import io.mycat.config.Isolations; import io.mycat.net.BackendAIOConnection; import io.mycat.route.RouteResultsetNode; import io.mycat.server.ServerConnection; import io.mycat.server.parser.ServerParse; import io.mycat.util.exception.UnknownTxIsolationException; /************************************************************* * PostgreSQL Native Connection impl * * @author Coollf * */ public class PostgreSQLBackendConnection extends BackendAIOConnection { public static enum BackendConnectionState { closed, connected, connecting } private static class StatusSync { private final Boolean autocommit; private final Integer charsetIndex; private final String schema; private final AtomicInteger synCmdCount; private final Integer txtIsolation; private final boolean xaStarted; private Boolean txReadonly; public StatusSync(boolean xaStarted, String schema, Integer charsetIndex, Integer txtIsolation, Boolean autocommit, int synCount, Boolean txReadonly) { super(); this.xaStarted = xaStarted; this.schema = schema; this.charsetIndex = charsetIndex; this.txtIsolation = txtIsolation; this.autocommit = autocommit; this.synCmdCount = new AtomicInteger(synCount); this.txReadonly = txReadonly; } public boolean synAndExecuted(PostgreSQLBackendConnection conn) { int remains = synCmdCount.decrementAndGet(); if (remains == 0) {// syn command finished this.updateConnectionInfo(conn); conn.metaDataSyned = true; return false; } else if (remains < 0) { return true; } return false; } private void updateConnectionInfo(PostgreSQLBackendConnection conn) { conn.xaStatus = (xaStarted) ? 1 : 0; if (schema != null) { conn.schema = schema; conn.oldSchema = conn.schema; } if (charsetIndex != null) { conn.setCharset(CharsetUtil.getCharset(charsetIndex)); } if (txtIsolation != null) { conn.txIsolation = txtIsolation; } if (autocommit != null) { conn.autocommit = autocommit; } if (txReadonly != null) { conn.txReadonly = txReadonly; } } } private static final Query _COMMIT = new Query("commit"); private static final Query _ROLLBACK = new Query("rollback"); private static void getCharsetCommand(StringBuilder sb, int clientCharIndex) { sb.append("SET names '").append(CharsetUtil.getCharset(clientCharIndex).toUpperCase()).append("';"); } /** * 获取 更改事物级别sql * * @param * @param txIsolation */ private static void getTxIsolationCommand(StringBuilder sb, int txIsolation) { switch (txIsolation) { case Isolations.READ_UNCOMMITTED: sb.append("SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;"); return; case Isolations.READ_COMMITTED: sb.append("SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED;"); return; case Isolations.REPEATED_READ: sb.append("SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ;"); return; case Isolations.SERIALIZABLE: sb.append("SET SESSION TRANSACTION ISOLATION LEVEL SERIALIZABLE;"); return; default: throw new UnknownTxIsolationException("txIsolation:" + txIsolation); } } private Object attachment; private volatile boolean autocommit=true; private volatile boolean txReadonly=false; private volatile boolean borrowed; protected volatile String charset = "utf8"; /*** * 当前事物ID */ private volatile String currentXaTxId; /** * 来自子接口 */ private volatile boolean fromSlaveDB; /**** * PG是否在事物中 */ private volatile boolean inTransaction = false; private AtomicBoolean isQuit = new AtomicBoolean(false); private volatile long lastTime; /** * 元数据同步 */ private volatile boolean metaDataSyned = true; private volatile boolean modifiedSQLExecuted = false; private volatile String oldSchema; /** * 密码 */ private volatile String password; /** * 数据源配置 */ private PostgreSQLDataSource pool; /*** * 响应handler */ private volatile ResponseHandler responseHandler; /*** * 对应数据库空间 */ private volatile String schema; // PostgreSQL服务端密码 private volatile int serverSecretKey; private volatile BackendConnectionState state = BackendConnectionState.connecting; private volatile StatusSync statusSync; private volatile int txIsolation; /*** * 用户名 */ private volatile String user; private volatile int xaStatus; public PostgreSQLBackendConnection(NetworkChannel channel, boolean fromSlaveDB) { super(channel); this.fromSlaveDB = fromSlaveDB; } @Override public void commit() { ByteBuffer buf = this.allocate(); _COMMIT.write(buf); this.write(buf); } @Override public void execute(RouteResultsetNode rrn, ServerConnection sc, boolean autocommit) throws IOException { int sqlType = rrn.getSqlType(); String orgin = rrn.getStatement(); if (LOGGER.isDebugEnabled()) { LOGGER.debug("{}查询任务。。。。{}", id, rrn.getStatement()); LOGGER.debug(orgin); } //FIX BUG https://github.com/MyCATApache/Mycat-Server/issues/1185 if (sqlType == ServerParse.SELECT || sqlType == ServerParse.SHOW) { if (sqlType == ServerParse.SHOW) { //此处进行部分SHOW 语法适配 String _newSql = PgSqlApaterUtils.apater(orgin); if(_newSql.trim().substring(0,4).equalsIgnoreCase("show")){//未能适配成功 ShowVariables.execute(sc, orgin, this); return; } } else if ("SELECT CONNECTION_ID()".equalsIgnoreCase(orgin)) { ShowVariables.justReturnValue(sc, String.valueOf(sc.getId()), this); return; } } if (!modifiedSQLExecuted && rrn.isModifySQL()) { modifiedSQLExecuted = true; } String xaTXID = null; if(sc.getSession2().getXaTXID()!=null){ xaTXID = sc.getSession2().getXaTXID() +",'"+getSchema()+"'"; } synAndDoExecute(xaTXID, rrn, sc.getCharsetIndex(), sc.getTxIsolation(), autocommit, sc.isTxReadonly()); } @Override public Object getAttachment() { return attachment; } private void getAutocommitCommand(StringBuilder sb, boolean autoCommit) { if (autoCommit) { sb.append(/*"SET autocommit=1;"*/"");//Fix bug 由于 PG9.0 开始不支持此选项，默认是为自动提交逻辑。 } else { sb.append("begin transaction;"); } } private void getTxReadonly(StringBuilder sb, boolean txReadonly) { if (txReadonly) { sb.append("SET SESSION TRANSACTION READ ONLY;"); } else { sb.append("SET SESSION TRANSACTION READ WRITE;"); } } @Override public long getLastTime() { return lastTime; } public String getPassword() { return password; } public PostgreSQLDataSource getPool() { return pool; } public ResponseHandler getResponseHandler() { return responseHandler; } @Override public String getSchema() { return this.schema; } public int getServerSecretKey() { return serverSecretKey; } public BackendConnectionState getState() { return state; } @Override public int getTxIsolation() { return txIsolation; } public String getUser() { return user; } @Override public boolean isAutocommit() { return autocommit; } @Override public boolean isTxReadonly() { return txReadonly; } @Override public int getSqlSelectLimit() { // todo return -1; } @Override public boolean isBorrowed() { return borrowed; } @Override public boolean isClosedOrQuit() { return isClosed() || isQuit.get(); } @Override public boolean isFromSlaveDB() { return fromSlaveDB; } public boolean isInTransaction() { return inTransaction; } @Override public boolean isModifiedSQLExecuted() { return modifiedSQLExecuted; } @Override public void onConnectFailed(Throwable t) { if (handler instanceof MySQLConnectionHandler) { } } @Override public void onConnectfinish() { LOGGER.debug("连接后台真正完成"); try { SocketChannel chan = (SocketChannel) this.channel; ByteBuffer buf = PacketUtils.makeStartUpPacket(user, schema); buf.flip(); chan.write(buf); } catch (Exception e) { LOGGER.error("Connected PostgreSQL Send StartUpPacket ERROR", e); throw new RuntimeException(e); } } protected final int getPacketLength(ByteBuffer buffer, int offset) { // Pg 协议获取包长度的方法和mysql 不一样 return PIOUtils.redInteger4(buffer, offset + 1) + 1; } /********** * 此查询用于心跳检查和获取连接后的健康检查 */ @Override public void query(String query) throws UnsupportedEncodingException { RouteResultsetNode rrn = new RouteResultsetNode("default", ServerParse.SELECT, query); synAndDoExecute(null, rrn, this.charsetIndex, this.txIsolation, true, this.txReadonly); } @Override public void quit() { if (isQuit.compareAndSet(false, true) && !isClosed()) { if (state == BackendConnectionState.connected) {// 断开 与PostgreSQL连接 Terminate terminate = new Terminate(); ByteBuffer buf = this.allocate(); terminate.write(buf); write(buf); } else { close("normal"); } } } /******* * 记录sql执行信息 */ @Override public void recordSql(String host, String schema, String statement) { LOGGER.debug(String.format("executed sql: host=%s,schema=%s,statement=%s", host, schema, statement)); } @Override public void release() { if (!metaDataSyned) {/* * indicate connection not normalfinished ,and * we can't know it's syn status ,so close it */ LOGGER.warn("can't sure connection syn result,so close it " + this); this.responseHandler = null; this.close("syn status unkown "); return; } metaDataSyned = true; attachment = null; statusSync = null; modifiedSQLExecuted = false; setResponseHandler(null); pool.releaseChannel(this); } @Override public void rollback() { ByteBuffer buf = this.allocate(); _ROLLBACK.write(buf); this.write(buf); } @Override public void setAttachment(Object attachment) { this.attachment = attachment; } @Override public void setBorrowed(boolean borrowed) { this.borrowed = borrowed; } public void setInTransaction(boolean inTransaction) { this.inTransaction = inTransaction; } @Override public void setLastTime(long currentTimeMillis) { this.lastTime = currentTimeMillis; } public void setPassword(String password) { this.password = password; } public void setPool(PostgreSQLDataSource pool) { this.pool = pool; } @Override public boolean setResponseHandler(ResponseHandler commandHandler) { this.responseHandler = commandHandler; return true; } @Override public void setSchema(String newSchema) { String curSchema = schema; if (curSchema == null) { this.schema = newSchema; this.oldSchema = newSchema; } else { this.oldSchema = curSchema; this.schema = newSchema; } } public void setServerSecretKey(int serverSecretKey) { this.serverSecretKey = serverSecretKey; } public void setState(BackendConnectionState state) { this.state = state; } public void setUser(String user) { this.user = user; } private void synAndDoExecute(String xaTxID, RouteResultsetNode rrn, int clientCharSetIndex, int clientTxIsoLation, boolean clientAutoCommit, boolean clientTxReadonly) { String xaCmd = null; boolean conAutoComit = this.autocommit; boolean conTxReadonly = this.txReadonly; String conSchema = this.schema; // never executed modify sql,so auto commit boolean expectAutocommit = !modifiedSQLExecuted || isFromSlaveDB() || clientAutoCommit; if (!expectAutocommit && xaTxID != null && xaStatus == 0) { clientTxIsoLation = Isolations.SERIALIZABLE; xaCmd = "XA START " + xaTxID + ';'; currentXaTxId = xaTxID; } int schemaSyn = conSchema.equals(oldSchema) ? 0 : 1; int charsetSyn = (this.charsetIndex == clientCharSetIndex) ? 0 : 1; int txIsoLationSyn = (txIsolation == clientTxIsoLation) ? 0 : 1; int autoCommitSyn = (conAutoComit == expectAutocommit) ? 0 : 1; int txReadonlySyn = (conTxReadonly == clientTxReadonly) ? 0 : 1; int synCount = schemaSyn + charsetSyn + txIsoLationSyn + autoCommitSyn + txReadonlySyn; if (synCount == 0) { String sql = rrn.getStatement(); Query query = new Query(PgSqlApaterUtils.apater(sql)); ByteBuffer buf = this.allocate();// XXX 此处处理问题 query.write(buf); this.write(buf); return; } // TODO COOLLF 此处大锅待实现. 相关 事物, 切换 库,自动提交等功能实现 StringBuilder sb = new StringBuilder(); if (charsetSyn == 1) { getCharsetCommand(sb, clientCharSetIndex); } if (txIsoLationSyn == 1) { getTxIsolationCommand(sb, clientTxIsoLation); } if (autoCommitSyn == 1) { getAutocommitCommand(sb, expectAutocommit); } if (txReadonlySyn == 1) { getTxReadonly(sb, clientTxReadonly); } if (xaCmd != null) { sb.append(xaCmd); } if (LOGGER.isDebugEnabled()) { LOGGER.debug("con need syn ,total syn cmd " + synCount + " commands " + sb.toString() + "schema change:" + ("" != null) + " con:" + this); } metaDataSyned = false; statusSync = new StatusSync(xaCmd != null, conSchema, clientCharSetIndex, clientTxIsoLation, expectAutocommit, synCount, clientTxReadonly); String sql = sb.append(PgSqlApaterUtils.apater(rrn.getStatement())).toString(); if(LOGGER.isDebugEnabled()){ LOGGER.debug("con={}, SQL={}", this, sql); } Query query = new Query(sql); ByteBuffer buf = allocate();// 申请ByetBuffer query.write(buf); this.write(buf); metaDataSyned = true; } public void close(String reason) { if (!isClosed.get()) { isQuit.set(true); super.close(reason); pool.connectionClosed(this); if (this.responseHandler != null) { this.responseHandler.connectionClose(this, reason); responseHandler = null; } } } @Override public void closeWithoutRsp(String reason) { // TODO Auto-generated method stub this.responseHandler = null; close(reason); } @Override public boolean syncAndExcute() { StatusSync sync = this.statusSync; if (sync != null) { boolean executed = sync.synAndExecuted(this); if (executed) { statusSync = null; } return executed; } return true; } @Override public String toString() { return "PostgreSQLBackendConnection [id=" + id + ", host=" + host + ", port=" + port + ", localPort=" + localPort + "]"; } @Override public void query(String sql, int charsetIndex) { try { query(sql); } catch (UnsupportedEncodingException e) { e.printStackTrace(); LOGGER.debug("UnsupportedEncodingException :"+ e.getMessage()); } } @Override public void disableRead() { // TODO Auto-generated method stub } @Override public void enableRead() { // TODO Auto-generated method stub } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/PostgreSQLBackendConnectionFactory.java ================================================ package io.mycat.backend.postgresql; import io.mycat.MycatServer; import io.mycat.backend.mysql.nio.handler.ResponseHandler; import io.mycat.config.model.DBHostConfig; import io.mycat.net.NIOConnector; import io.mycat.net.factory.BackendConnectionFactory; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.channels.AsynchronousSocketChannel; import java.nio.channels.CompletionHandler; import java.nio.channels.NetworkChannel; public class PostgreSQLBackendConnectionFactory extends BackendConnectionFactory { @SuppressWarnings({ "unchecked", "rawtypes" }) public PostgreSQLBackendConnection make(PostgreSQLDataSource pool, ResponseHandler handler, final String schema) throws IOException { final DBHostConfig dsc = pool.getConfig(); NetworkChannel channel = this.openSocketChannel(MycatServer .getInstance().isAIO()); final PostgreSQLBackendConnection c = new PostgreSQLBackendConnection( channel, pool.isReadNode()); MycatServer.getInstance().getConfig().setSocketParams(c, false); // 设置NIOHandler c.setHandler(new PostgreSQLBackendConnectionHandler(c)); c.setHost(dsc.getIp()); c.setPort(dsc.getPort()); c.setUser(dsc.getUser()); c.setPassword(dsc.getPassword()); c.setSchema(schema); c.setPool(pool); c.setResponseHandler(handler); c.setIdleTimeout(pool.getConfig().getIdleTimeout()); if (channel instanceof AsynchronousSocketChannel) { ((AsynchronousSocketChannel) channel).connect( new InetSocketAddress(dsc.getIp(), dsc.getPort()), c, (CompletionHandler) MycatServer.getInstance() .getConnector()); } else { ((NIOConnector) MycatServer.getInstance().getConnector()) .postConnect(c); } return c; } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/PostgreSQLBackendConnectionHandler.java ================================================ package io.mycat.backend.postgresql; import io.mycat.MycatServer; import io.mycat.backend.mysql.nio.handler.ResponseHandler; import io.mycat.backend.postgresql.PostgreSQLBackendConnection.BackendConnectionState; import io.mycat.backend.postgresql.packet.AuthenticationPacket; import io.mycat.backend.postgresql.packet.AuthenticationPacket.AuthType; import io.mycat.backend.postgresql.packet.BackendKeyData; import io.mycat.backend.postgresql.packet.CommandComplete; import io.mycat.backend.postgresql.packet.CopyInResponse; import io.mycat.backend.postgresql.packet.CopyOutResponse; import io.mycat.backend.postgresql.packet.DataRow; import io.mycat.backend.postgresql.packet.EmptyQueryResponse; import io.mycat.backend.postgresql.packet.ErrorResponse; import io.mycat.backend.postgresql.packet.NoticeResponse; import io.mycat.backend.postgresql.packet.NotificationResponse; import io.mycat.backend.postgresql.packet.ParameterStatus; import io.mycat.backend.postgresql.packet.PasswordMessage; import io.mycat.backend.postgresql.packet.PostgreSQLPacket; import io.mycat.backend.postgresql.packet.ReadyForQuery; import io.mycat.backend.postgresql.packet.ReadyForQuery.TransactionState; import io.mycat.backend.postgresql.packet.RowDescription; import io.mycat.backend.postgresql.utils.PacketUtils; import io.mycat.backend.postgresql.utils.PgPacketApaterUtils; import io.mycat.buffer.BufferArray; import io.mycat.config.ErrorCode; import io.mycat.net.handler.BackendAsyncHandler; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.ErrorPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.OkPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.fastjson.JSON; public class PostgreSQLBackendConnectionHandler extends BackendAsyncHandler { static class SelectResponse { private List dataRows = new ArrayList<>(); private RowDescription description; public SelectResponse(RowDescription description) { this.description = description; } public void addDataRow(DataRow packet) { this.dataRows.add(packet); } public List getDataRows() { return dataRows; } public RowDescription getDescription() { return description; } public void setDataRows(List dataRows) { this.dataRows = dataRows; } } private static final Logger LOGGER = LoggerFactory .getLogger(PostgreSQLBackendConnection.class); private static final int RESULT_STATUS_INIT = 0; private byte packetId = 1; /***** * 每个后台响应有唯一的连接 */ private final PostgreSQLBackendConnection source; /** * 响应数据 */ private volatile SelectResponse response = null; /** * 响应状态 */ private int resultStatus; public PostgreSQLBackendConnectionHandler(PostgreSQLBackendConnection source) { this.source = source; } /*** * 进行连接处理 * * @param con * @param buf * @param start * @param readedLength */ private void doConnecting(PostgreSQLBackendConnection con, ByteBuffer buf, int start, int readedLength) { try { List packets = PacketUtils.parsePacket(buf, 0, readedLength); LOGGER.debug(JSON.toJSONString(packets)); if (!packets.isEmpty() && packets.get(0) instanceof AuthenticationPacket) { // pg认证信息 AuthenticationPacket packet = (AuthenticationPacket) packets .get(0); AuthType aut = packet.getAuthType(); if (aut != AuthType.Ok) { PasswordMessage pak = new PasswordMessage( con.getUser(), con.getPassword(), aut, ((AuthenticationPacket) packet).getSalt()); ByteBuffer buffer = con.allocate(); //allocate(pak.getLength() + 1); pak.write(buffer); con.write(buffer); } else {// 登入成功了.... for (int i = 1; i < packets.size(); i++) { PostgreSQLPacket _p = packets.get(i); if (_p instanceof BackendKeyData) { con.setServerSecretKey(((BackendKeyData) _p) .getSecretKey()); } } LOGGER.debug("SUCCESS Connected TO PostgreSQL , con id is {}",con.getId()); con.setState(BackendConnectionState.connected); con.getResponseHandler().connectionAcquired(con);// 连接已经可以用来 } } } catch (IOException e) { LOGGER.error("error",e); } } /*** * 进行业务处理 * * @param con * @param buf * @param start * @param readedLength */ private void doHandleBusinessMsg(PostgreSQLBackendConnection con, ByteBuffer buf, int start, int readedLength) { try { List packets = PacketUtils.parsePacket(buf, 0, readedLength); if (packets == null || packets.isEmpty()) { return ; //throw new RuntimeException("数据包解析出错"); } for (PostgreSQLPacket packet : packets) { if (packet instanceof ErrorResponse) { doProcessErrorResponse(con, (ErrorResponse) packet); } else if (packet instanceof RowDescription) { response = new SelectResponse((RowDescription) packet); } else if (packet instanceof DataRow) { response.addDataRow((DataRow) packet); } else if (packet instanceof ParameterStatus) { doProcessParameterStatus(con, (ParameterStatus) packet); } else if (packet instanceof CommandComplete) { doProcessCommandComplete(con, (CommandComplete) packet, response); } else if (packet instanceof NoticeResponse) { doProcessNoticeResponse(con, (NoticeResponse) packet); } else if (packet instanceof ReadyForQuery) { doProcessReadyForQuery(con, (ReadyForQuery) packet); } else if (packet instanceof NotificationResponse) { doProcessNotificationResponse(con, (NotificationResponse) packet); } else if (packet instanceof CopyInResponse) { doProcessCopyInResponse(con, (CopyInResponse) packet); } else if (packet instanceof CopyOutResponse) { doProcessCopyOutResponse(con, (CopyOutResponse) packet); } else if (packet instanceof EmptyQueryResponse) { doProcessEmptyQueryResponse(con, (EmptyQueryResponse) packet); } } } catch (Exception e) { LOGGER.error("处理出异常了", e); ErrorPacket err = new ErrorPacket(); err.packetId = ++packetId; err.message = ("内部服务器处理出错!" + e.getMessage()).getBytes(); err.errno = ErrorCode.ERR_NOT_SUPPORTED; ResponseHandler respHand = con.getResponseHandler(); if (respHand != null) { respHand.errorResponse(err.writeToBytes(), con); } else { LOGGER.error("{},respHand 为空",this); } } } /*************** * 处理简单查询结果 ,每一个查询都是一件 CommandComplete 为结束 * @param con PostgreSQL 后端连接 * @param response * @param commandComplete */ private void doProcessBusinessQuery(PostgreSQLBackendConnection con, SelectResponse response, CommandComplete commandComplete) { RowDescription rowHd = response.getDescription(); List fieldPks = PgPacketApaterUtils .rowDescConvertFieldPacket(rowHd); List rowDatas = new ArrayList<>(); for (DataRow dataRow : response.getDataRows()) { rowDatas.add(PgPacketApaterUtils .rowDataConvertRowDataPacket(dataRow)); } BufferArray bufferArray = MycatServer.getInstance().getBufferPool() .allocateArray(); ResultSetHeaderPacket headerPkg = new ResultSetHeaderPacket(); headerPkg.fieldCount = fieldPks.size(); headerPkg.packetId = ++packetId; headerPkg.write(bufferArray); byte[] header = bufferArray.writeToByteArrayAndRecycle(); List fields = new ArrayList(fieldPks.size()); Iterator itor = fieldPks.iterator(); while (itor.hasNext()) { bufferArray = MycatServer.getInstance().getBufferPool() .allocateArray(); FieldPacket curField = itor.next(); curField.packetId = ++packetId; curField.write(bufferArray); byte[] field = bufferArray.writeToByteArrayAndRecycle(); fields.add(field); itor.remove(); } bufferArray = MycatServer.getInstance().getBufferPool().allocateArray(); EOFPacket eofPckg = new EOFPacket(); eofPckg.packetId = ++packetId; eofPckg.write(bufferArray); byte[] eof = bufferArray.writeToByteArrayAndRecycle(); if (con.getResponseHandler() != null) { con.getResponseHandler().fieldEofResponse(header, fields, eof, con); } else { LOGGER.error("响应句柄为空"); } // output row for (RowDataPacket curRow : rowDatas) { bufferArray = MycatServer.getInstance().getBufferPool() .allocateArray(); curRow.packetId = ++packetId; curRow.write(bufferArray); byte[] row = bufferArray.writeToByteArrayAndRecycle(); con.getResponseHandler().rowResponse(row, con); } // end row bufferArray = MycatServer.getInstance().getBufferPool().allocateArray(); eofPckg = new EOFPacket(); eofPckg.packetId = ++packetId; eofPckg.write(bufferArray); eof = bufferArray.writeToByteArrayAndRecycle(); if (con.getResponseHandler() != null) { con.getResponseHandler().rowEofResponse(eof, con); } else { LOGGER.error("响应句柄为空"); } } private void doProcessCommandComplete(PostgreSQLBackendConnection con, CommandComplete commandComplete, SelectResponse response) { if (commandComplete.isSelectComplete()) { if (response == null) { throw new RuntimeException( "the select proess err ,the SelectResponse is empty"); } doProcessBusinessQuery(con, response, commandComplete); } else { OkPacket okPck = new OkPacket(); okPck.affectedRows =commandComplete.getAffectedRows(); okPck.insertId =commandComplete.getInsertId(); okPck.packetId = ++packetId; okPck.message = commandComplete.getCommandResponse().getBytes(); con.getResponseHandler().okResponse(okPck.writeToBytes(), con); } } private void doProcessCopyInResponse(PostgreSQLBackendConnection con, CopyInResponse packet) { // TODO(复制数据暂时不需要) } private void doProcessCopyOutResponse(PostgreSQLBackendConnection con, CopyOutResponse packet) { // TODO(复制数据暂时不需要) } private void doProcessEmptyQueryResponse(PostgreSQLBackendConnection con, EmptyQueryResponse packet) { // TODO(现阶段无空白sql) } /*** * 处理查询出错数据包 * * @param con * @param errorResponse */ private void doProcessErrorResponse(PostgreSQLBackendConnection con, ErrorResponse errorResponse) { LOGGER.debug("查询出错了!"); ErrorPacket err = new ErrorPacket(); err.packetId = ++packetId; err.message = errorResponse.getErrMsg().trim().replaceAll("\0", " ") .getBytes(); err.errno = ErrorCode.ER_UNKNOWN_ERROR; con.getResponseHandler().errorResponse(err.writeToBytes(), con); } /****** * 执行成功但是又警告信息 * * @param con * @param noticeResponse */ private void doProcessNoticeResponse(PostgreSQLBackendConnection con, NoticeResponse noticeResponse) { // TODO (通知提醒信息) } private void doProcessNotificationResponse(PostgreSQLBackendConnection con, NotificationResponse notificationResponse) { // TODO(后台参数改变通知) } private void doProcessParameterStatus(PostgreSQLBackendConnection con, ParameterStatus parameterStatus) { // TODO(设置参数响应) } /**** * PostgreSQL 已经处理完成一个任务等等下一个任务 * @param con * @param readyForQuery */ private void doProcessReadyForQuery(PostgreSQLBackendConnection con, ReadyForQuery readyForQuery) { if (con.isInTransaction() != (readyForQuery.getState() == TransactionState.IN)) {// 设置连接的后台事物状态 con.setInTransaction((readyForQuery.getState() == TransactionState.IN)); } } @Override public void handle(byte[] data) { offerData(data, source.getProcessor().getExecutor()); } /* * 真正处理 数据库发过来的数据 * * * * @see io.mycat.net.handler.BackendAsyncHandler#handleData(byte[]) */ @Override protected void handleData(byte[] data) { ByteBuffer theBuf = null; try { theBuf = source.allocate(); theBuf.put(data); switch (source.getState()) { case connecting: { doConnecting(source, theBuf, 0, data.length); return; } case connected: { try { doHandleBusinessMsg(source, theBuf , 0, data.length); } catch (Exception e) { LOGGER.warn("caught err of con " + source, e); } return; } default: LOGGER.warn("not handled connecton state err " + source.getState() + " for con " + source); break; } } catch (Exception e) { LOGGER.error("读取数据包出错",e); }finally{ if(theBuf!=null){ source.recycle(theBuf); } } } @Override protected void offerDataError() { resultStatus = RESULT_STATUS_INIT; throw new RuntimeException("offer data error!"); } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/PostgreSQLDataSource.java ================================================ package io.mycat.backend.postgresql; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.heartbeat.DBHeartbeat; import io.mycat.backend.mysql.nio.handler.ResponseHandler; import io.mycat.backend.postgresql.heartbeat.PostgreSQLHeartbeat; import io.mycat.config.model.DBHostConfig; import io.mycat.config.model.DataHostConfig; import java.io.IOException; /******************* * PostgreSQL 后端数据源实现 * @author Coollf * */ public class PostgreSQLDataSource extends PhysicalDatasource { private final PostgreSQLBackendConnectionFactory factory; public PostgreSQLDataSource(DBHostConfig config, DataHostConfig hostConfig, boolean isReadNode) { super(config, hostConfig, isReadNode); this.factory = new PostgreSQLBackendConnectionFactory(); } @Override public DBHeartbeat createHeartBeat() { return new PostgreSQLHeartbeat(this); } @Override public void createNewConnection(ResponseHandler handler, String schema) throws IOException { factory.make(this, handler, schema); } @Override public boolean testConnection(String schema) throws IOException { // TODO Auto-generated method stub return true; } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/heartbeat/PostgreSQLDetector.java ================================================ package io.mycat.backend.postgresql.heartbeat; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.heartbeat.DBHeartbeat; import io.mycat.backend.heartbeat.MySQLHeartbeat; import io.mycat.backend.postgresql.PostgreSQLDataSource; import io.mycat.config.model.DataHostConfig; import io.mycat.sqlengine.OneRawSQLQueryResultHandler; import io.mycat.sqlengine.SQLJob; import io.mycat.sqlengine.SQLQueryResult; import io.mycat.sqlengine.SQLQueryResultListener; import io.mycat.util.TimeUtil; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; public class PostgreSQLDetector implements SQLQueryResultListener>> { private static final String[] MYSQL_SLAVE_STAUTS_COLMS = new String[] { "Seconds_Behind_Master", "Slave_IO_Running", "Slave_SQL_Running" }; private PostgreSQLHeartbeat heartbeat; private final AtomicBoolean isQuit; private volatile long heartbeatTimeout; private volatile long lastSendQryTime; private volatile SQLJob sqlJob; private long lasstReveivedQryTime; public PostgreSQLDetector(PostgreSQLHeartbeat heartbeat) { this.heartbeat = heartbeat; this.isQuit = new AtomicBoolean(false); } @Override public void onResult(SQLQueryResult> result) { if (result.isSuccess()) { int balance = heartbeat.getSource().getDbPool().getBalance(); PhysicalDatasource source = heartbeat.getSource(); Map resultResult = result.getResult(); if (source.getHostConfig().isShowSlaveSql() && (source.getHostConfig().getSwitchType() == DataHostConfig.SYN_STATUS_SWITCH_DS || PhysicalDBPool.BALANCE_NONE != balance)) { String Slave_IO_Running = resultResult != null ? resultResult .get("Slave_IO_Running") : null; String Slave_SQL_Running = resultResult != null ? resultResult .get("Slave_SQL_Running") : null; if (Slave_IO_Running != null && Slave_IO_Running.equals(Slave_SQL_Running) && Slave_SQL_Running.equals("Yes")) { heartbeat.setDbSynStatus(DBHeartbeat.DB_SYN_NORMAL); String Seconds_Behind_Master = resultResult .get("Seconds_Behind_Master"); if (null != Seconds_Behind_Master && !"".equals(Seconds_Behind_Master)) { heartbeat.setSlaveBehindMaster(Integer .valueOf(Seconds_Behind_Master)); } } else if (source.isSalveOrRead()) { MySQLHeartbeat.LOGGER .warn("found MySQL master/slave Replication err !!! " + heartbeat.getSource().getConfig()); heartbeat.setDbSynStatus(DBHeartbeat.DB_SYN_ERROR); } } heartbeat.setResult(PostgreSQLHeartbeat.OK_STATUS, this, null); } else { heartbeat.setResult(PostgreSQLHeartbeat.ERROR_STATUS, this, null); } lasstReveivedQryTime = System.currentTimeMillis(); } public PostgreSQLHeartbeat getHeartbeat() { return heartbeat; } public long getHeartbeatTimeout() { return heartbeatTimeout; } public void heartbeat() { lastSendQryTime = System.currentTimeMillis(); PostgreSQLDataSource ds = heartbeat.getSource(); String databaseName = ds.getDbPool().getSchemas()[0]; String[] fetchColms = {}; if (heartbeat.getSource().getHostConfig().isShowSlaveSql()) { fetchColms = MYSQL_SLAVE_STAUTS_COLMS; } OneRawSQLQueryResultHandler resultHandler = new OneRawSQLQueryResultHandler( fetchColms, this); sqlJob = new SQLJob(heartbeat.getHeartbeatSQL(), databaseName, resultHandler, ds); sqlJob.run(); } public void close(String msg) { SQLJob curJob = sqlJob; if (curJob != null && !curJob.isFinished()) { curJob.teminate(msg); sqlJob = null; } } public boolean isHeartbeatTimeout() { return TimeUtil.currentTimeMillis() > Math.max(lastSendQryTime, lasstReveivedQryTime) + heartbeatTimeout; } public long getLastSendQryTime() { return lastSendQryTime; } public long getLasstReveivedQryTime() { return lasstReveivedQryTime; } public void quit() { } public boolean isQuit() { return isQuit.get(); } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/heartbeat/PostgreSQLHeartbeat.java ================================================ package io.mycat.backend.postgresql.heartbeat; import java.text.SimpleDateFormat; import java.util.Date; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.heartbeat.DBHeartbeat; import io.mycat.backend.postgresql.PostgreSQLDataSource; import io.mycat.config.model.DataHostConfig; public class PostgreSQLHeartbeat extends DBHeartbeat { private static final int MAX_RETRY_COUNT = 5; public static final Logger LOGGER = LoggerFactory.getLogger(PostgreSQLHeartbeat.class); private PostgreSQLDataSource source; private ReentrantLock lock; private int maxRetryCount; private PostgreSQLDetector detector; public PostgreSQLHeartbeat(PostgreSQLDataSource source) { this.source = source; this.lock = new ReentrantLock(false); this.maxRetryCount = MAX_RETRY_COUNT; this.status = INIT_STATUS; this.heartbeatSQL = source.getHostConfig().getHearbeatSQL(); } @Override public void start() { final ReentrantLock lock = this.lock; lock.lock(); try { isStop.compareAndSet(true, false); super.status = DBHeartbeat.OK_STATUS; } finally { lock.unlock(); } } @Override public void stop() { final ReentrantLock lock = this.lock; lock.lock(); try { if (isStop.compareAndSet(false, true)) { if (isChecking.get()) { // nothing } else { PostgreSQLDetector detector = this.detector; if (detector != null) { detector.quit(); isChecking.set(false); } } } } finally { lock.unlock(); } } @Override public String getLastActiveTime() { PostgreSQLDetector detector = this.detector; if (detector == null) { return null; } long t = detector.getLasstReveivedQryTime(); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); return sdf.format(new Date(t)); } @Override public long getTimeout() { PostgreSQLDetector detector = this.detector; if (detector == null) { return -1L; } return detector.getHeartbeatTimeout(); } @Override public void heartbeat() { final ReentrantLock lock = this.lock; lock.lock(); try { if (isChecking.compareAndSet(false, true)) { PostgreSQLDetector detector = this.detector; if (detector == null || detector.isQuit()) { try { detector = new PostgreSQLDetector(this); detector.heartbeat(); } catch (Exception e) { LOGGER.warn(source.getConfig().toString(), e); setResult(ERROR_STATUS, detector, null); return; } this.detector = detector; } else { detector.heartbeat(); } } else { PostgreSQLDetector detector = this.detector; if (detector != null) { if (detector.isQuit()) { isChecking.compareAndSet(true, false); } else if (detector.isHeartbeatTimeout()) { setResult(TIMEOUT_STATUS, detector, null); } } } } finally { lock.unlock(); } } public PostgreSQLDataSource getSource() { return source; } public void setResult(int result, PostgreSQLDetector detector, Object attr) { this.isChecking.set(false); switch (result) { case OK_STATUS: setOk(detector); break; case ERROR_STATUS: setError(detector); break; case TIMEOUT_STATUS: setTimeout(detector); break; } if (this.status != OK_STATUS) { switchSourceIfNeed("heartbeat error"); } } private void switchSourceIfNeed(String reason) { int switchType = source.getHostConfig().getSwitchType(); if (switchType == DataHostConfig.NOT_SWITCH_DS) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("not switch datasource ,for switchType is " + DataHostConfig.NOT_SWITCH_DS); return; } return; } PhysicalDBPool pool = this.source.getDbPool(); int curDatasourceHB = pool.getSource().getHeartbeat().getStatus(); // read node can't switch ,only write node can switch if (pool.getWriteType() == PhysicalDBPool.WRITE_ONLYONE_NODE && !source.isReadNode() && curDatasourceHB != DBHeartbeat.OK_STATUS && pool.getSources().length > 1) { synchronized (pool) { // try to see if need switch datasource curDatasourceHB = pool.getSource().getHeartbeat().getStatus(); if (curDatasourceHB != DBHeartbeat.INIT_STATUS && curDatasourceHB != DBHeartbeat.OK_STATUS) { int curIndex = pool.getActivedIndex(); int nextId = pool.next(curIndex); PhysicalDatasource[] allWriteNodes = pool.getSources(); while (true) { if (nextId == curIndex) { break; } PhysicalDatasource theSource = allWriteNodes[nextId]; DBHeartbeat theSourceHB = theSource.getHeartbeat(); int theSourceHBStatus = theSourceHB.getStatus(); if (theSourceHBStatus == DBHeartbeat.OK_STATUS) { if (switchType == DataHostConfig.SYN_STATUS_SWITCH_DS) { if (Integer.valueOf(0).equals(theSourceHB.getSlaveBehindMaster())) { LOGGER.info("try to switch datasource ,slave is synchronized to master " + theSource.getConfig()); pool.switchSourceOrVoted(nextId, true, reason); break; } else { LOGGER.warn( "ignored datasource ,slave is not synchronized to master , slave behind master :" + theSourceHB.getSlaveBehindMaster() + " " + theSource.getConfig()); } } else { // normal switch LOGGER.info("try to switch datasource ,not checked slave synchronize status " + theSource.getConfig()); pool.switchSourceOrVoted(nextId, true, reason); break; } } nextId = pool.next(nextId); } } } } } private void setTimeout(PostgreSQLDetector detector) { this.isChecking.set(false); status = DBHeartbeat.TIMEOUT_STATUS; } private void setError(PostgreSQLDetector detector) { // should continues check error status if ( this.errorCount.incrementAndGet() < maxRetryCount) { if (detector != null && !detector.isQuit()) { heartbeat(); // error count not enough, heart beat again } // return; } else { if (detector != null) { detector.quit(); } this.status = ERROR_STATUS; this.errorCount.set(0);; } } private void setOk(PostgreSQLDetector detector) { recorder.set(detector.getLasstReveivedQryTime() - detector.getLastSendQryTime()); switch (status) { case DBHeartbeat.TIMEOUT_STATUS: this.status = DBHeartbeat.INIT_STATUS; this.errorCount.set(0);; if (isStop.get()) { detector.quit(); } else { heartbeat();// timeout, heart beat again } break; case DBHeartbeat.OK_STATUS: break; default: this.status = OK_STATUS; this.errorCount.set(0);; } if (isStop.get()) { detector.quit(); } } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/package-info.java ================================================ /** * @author Coollf * */ package io.mycat.backend.postgresql; /* postgresql mycat 相关支持 config demo ============================================================================================================ select 1 ============================================================================================================= */ ================================================ FILE: src/main/java/io/mycat/backend/postgresql/packet/AuthenticationPacket.java ================================================ package io.mycat.backend.postgresql.packet; import java.nio.ByteBuffer; import io.mycat.backend.postgresql.utils.PIOUtils; public class AuthenticationPacket extends PostgreSQLPacket { public static enum AuthType { Ok(0), KerberosV5(2), CleartextPassword(3), CryptPassword(4), MD5Password(5), SCMCredential(6); private int value; AuthType(int v) { this.value = v; } public int getValue() { return value; } public static AuthType valueOf(int v) { if (v == Ok.value) { return Ok; } if (v == KerberosV5.value) { return KerberosV5; } if (v == CleartextPassword.value) { return CleartextPassword; } if (v == MD5Password.value) { return MD5Password; } if (v == SCMCredential.value) { return SCMCredential; } return null; } } /*** * 标记 */ private char marker = PacketMarker.B_Auth.getValue(); /**** * 数据包长度 */ private int length; /*** * 盐粒 */ private byte[] salt; private AuthType authType; public AuthType getAuthType() { return authType; } @Override public int getLength() { return length; } @Override public char getMarker() { return marker; } public byte[] getSalt() { return salt; } public void setSalt(byte[] salt) { this.salt = salt; } public static AuthenticationPacket parse(ByteBuffer buffer, int offset){ if (buffer.get(offset) != PacketMarker.B_Auth.getValue()) { throw new IllegalArgumentException("this packetData not is AuthenticationPacket"); } AuthenticationPacket packet = new AuthenticationPacket(); packet.length = PIOUtils.redInteger4(buffer, offset + 1); packet.authType = AuthType.valueOf(PIOUtils.redInteger4(buffer, offset + 1 + 4)); if (packet.authType == AuthType.MD5Password) { packet.salt = PIOUtils.redByteArray(buffer, offset + 1 + 4 + 4, 4); } return packet; } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/packet/BackendKeyData.java ================================================ package io.mycat.backend.postgresql.packet; import java.nio.ByteBuffer; /** * 后端数据包信息 * * @author Coollf * */ // BackendKeyData (B) // Byte1('K') // 标识该消息是一个取消键字数据。 如果前端希望能够在稍后发出 CancelRequest 消息， 那么它必须保存这个值。 // // Int32(12) // 以字节记的消息内容的长度，包括长度本身。 // // Int32 // 后端的进程号（PID）。 // // Int32 // 此后端的密钥（secret key ）。 public class BackendKeyData extends PostgreSQLPacket { /** * 长度 */ private int length; /*** * 进程ID */ private int pid; /*** * 此后端的密钥（secret key ） */ private int secretKey; public int getPid() { return pid; } public int getSecretKey() { return secretKey; } @Override public int getLength() { return length; } @Override public char getMarker() { return PacketMarker.B_BackendKey.getValue(); } /*** * 解析数据包 * * @param buffer * @param offset * @return * @throws IllegalArgumentException */ public static BackendKeyData parse(ByteBuffer buffer, int offset) { if (buffer.get(offset) != PacketMarker.B_BackendKey.getValue()) { throw new IllegalArgumentException("this packet not is BackendKeyData"); } BackendKeyData pac = new BackendKeyData(); pac.length = buffer.getInt(offset + 1); pac.pid = buffer.getInt(offset + 1 + 4); pac.secretKey = buffer.getInt(offset + 1 + 4 + 4); return pac; } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/packet/Bind.java ================================================ package io.mycat.backend.postgresql.packet; // Bind (F) // Byte1('B') // 标识该信息是一个绑定命令。 // // Int32 // 以字节记的消息内容的长度，包括长度本身。 // // String // 目标入口的名字（空字串则选取未命名的入口）。 // // String // 源准备好语句的名字（空字串则选取未命名的准备好语句）。 // // Int16 // // 后面跟着的参数格式代码的数目（在下面的 C 中说明）。 这个数值可以是零，表示没有参数，或者是参数都使用缺省格式（文本）； 或者是一，这种情况下声明的格式代码应用于所有参数；或者它可以等于实际数目的参数。 // // Int16[C] // 参数格式代码。目前每个都必须是零（文本）或者一（二进制）。 // // Int16 // 后面跟着的参数值的数目（可能为零）。这些必须和查询需要的参数个数匹配。 // // 然后，每个参数都会出现下面的字段对： // // Int32 // 参数值的长度，以字节记（这个长度并不包含长度本身）。可以为零。 一个特殊的情况是，-1 表示一个 NULL 参数值。在 NULL 的情况下， 后面不会跟着数值字节。 // // Byten // 参数值，格式是关联的格式代码标明的。n 是上面的长度。 // // 在最后一个参数之后，出现下面的字段： // // Int16 // // 后面跟着的结果字段格式代码数目（下面的 R 描述）。 这个数目可以是零表示没有结果字段，或者结果字段都使用缺省格式（文本）； 或者是一，这种情况下声明格式代码应用于所有结果字段（如果有的话）；或者它可以等于查询的结果字段的实际数目。 // // Int16[R] // 结果字段格式代码。目前每个必须是零（文本）或者一（二进制）。 public class Bind extends PostgreSQLPacket { public static class DataParameter { /** * 字段值的长度，以字节记（这个长度不包括它自己）。 可以为零。一个特殊的情况是，-1 表示一个 NULL 的字段值。 在 NULL * 的情况下就没有跟着数据字段。 */ private int length; private byte[] data; private boolean isNull; /** * @return the length */ public int getLength() { return length; } /** * @param length the length to set */ public void setLength(int length) { this.length = length; } /** * @return the data */ public byte[] getData() { return data; } /** * @param data the data to set */ public void setData(byte[] data) { this.data = data; } /** * @return the isNull */ public boolean isNull() { return isNull; } /** * @param isNull the isNull to set */ public void setNull(boolean isNull) { this.isNull = isNull; } } private char marker; private int length; private String name; private String sql; private short parameterProtocolNumber; private DataProtocol[] parameterProtocol; private short parameterNumber; private DataParameter[] parameter; private short resultNumber; private DataProtocol[] resultProtocol; @Override public int getLength() { return length; } @Override public char getMarker() { return marker; } /** * @return the name */ public String getName() { return name; } /** * @return the sql */ public String getSql() { return sql; } /** * @return the parameterProtocolNumber */ public short getParameterProtocolNumber() { return parameterProtocolNumber; } /** * @return the parameterProtocol */ public DataProtocol[] getParameterProtocol() { return parameterProtocol; } /** * @return the parameterNumber */ public short getParameterNumber() { return parameterNumber; } /** * @return the parameter */ public DataParameter[] getParameter() { return parameter; } /** * @return the resultNumber */ public short getResultNumber() { return resultNumber; } /** * @return the resultProtocol */ public DataProtocol[] getResultProtocol() { return resultProtocol; } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/packet/BindComplete.java ================================================ package io.mycat.backend.postgresql.packet; import java.nio.ByteBuffer; import io.mycat.backend.postgresql.utils.PIOUtils; // BindComplete (B) // Byte1('2') // 标识消息为一个绑定结束标识符。 // // Int32(4) // 以字节记的消息长度，包括长度本身。 /*** * 绑定预编译sql成功 * @author Coollf * */ public class BindComplete extends PostgreSQLPacket { private char marker = PacketMarker.B_BindComplete.getValue(); private int length; @Override public int getLength() { return length; } @Override public char getMarker() { return marker; } public static BindComplete parse(ByteBuffer buffer, int offset) { if ((char) buffer.get(offset) != PacketMarker.B_BindComplete.getValue()) { throw new IllegalArgumentException( "this packet not is BindComplete"); } BindComplete parse = new BindComplete(); parse.length = PIOUtils.redInteger4(buffer, offset + 1); return parse; } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/packet/CancelRequest.java ================================================ package io.mycat.backend.postgresql.packet; import java.nio.ByteBuffer; import io.mycat.backend.postgresql.utils.PIOUtils; // CancelRequest (F) // Int32(16) // 以字节计的消息长度。包括长度本身。 // // Int32(80877102) // 取消请求代码。选这个值是为了在高16位包含 1234， 低16位包含 5678。（为避免混乱，这个代码必须与协议版本号不同．） // // Int32 // 目标后端的进程号（PID）。 // // Int32 // 目标后端的密钥（secret key）。 /*** * 取消请求 * * @author Coollf * */ public class CancelRequest extends PostgreSQLPacket { private int length = 16; private int cancelCode = 80877102; private int pid; private int secretKey; public CancelRequest(int pid, int secretKey) { this.pid = pid; this.secretKey = secretKey; } @Override public int getLength() { return length; } @Override public char getMarker() { return 0; } public void write(ByteBuffer buffer) { PIOUtils.SendInteger4(length, buffer); PIOUtils.SendInteger4(cancelCode, buffer); PIOUtils.SendInteger4(pid, buffer); PIOUtils.SendInteger4(secretKey, buffer); } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/packet/CommandComplete.java ================================================ package io.mycat.backend.postgresql.packet; import java.nio.ByteBuffer; import io.mycat.backend.postgresql.utils.PIOUtils; // CommandComplete (B) // Byte1('C') // 标识此消息是一个命令结束响应。 // // Int32 // 以字节记的消息内容的长度，包括长度本身。 // // String // 命令标记。它通常是一个单字，标识那个命令完成。 // // 对于INSERT命令，标记是INSERT oid rows， 这里的rows是插入的行数。oid 在row为 1 并且目标表有 OID 的时候是插入行的对象 ID； 否则oid就是 0。 // // 对于DELETE 命令，标记是 DELETE rows， 这里的 rows 是删除的行数。 // // 对于 UPDATE 命令，标记是 UPDATE rows 这里的 rows 是更新的行数。 // // 对于 MOVE 命令，标记是 MOVE rows，这里的 rows 是游标未知改变的行数。 // // 对于 FETCH 命令，标记是 FETCH rows，这里的 rows 是从游标中检索出来的行数。 public class CommandComplete extends PostgreSQLPacket { private int length; /** * 命令 */ private String commandResponse; // 存储状态。 public int getAffectedRows() { return affectedRows; } public void setAffectedRows(int affectedRows) { this.affectedRows = affectedRows; } public int getInsertId() { return insertId; } public void setInsertId(int insertId) { this.insertId = insertId; } // 修改影响条数 private int affectedRows = 0; // 插入ID private int insertId = 0; @Override public int getLength() { return length; } public boolean isDDLComplete() { return commandResponse != null && (commandResponse.startsWith("INSERT") || commandResponse.startsWith("DELETE") || commandResponse.startsWith("UPDATE")); } public boolean isTranComplete() { return commandResponse != null && (commandResponse.startsWith("ROLLBACK") || commandResponse.startsWith("COMMIT")); } public boolean isSelectComplete() { return commandResponse != null && (commandResponse.startsWith("SELECT")); } public int getRows() { if (!isDDLComplete()) { return 0; } if (commandResponse != null) { String[] s = commandResponse.split(" +"); if (s.length == 0) { return 0; } try { return Integer.valueOf(s[s.length - 1].trim()); } catch (Exception e) { throw new RuntimeException(e); } } return 0; } @Override public char getMarker() { return PacketMarker.B_CommandComplete.getValue(); } public static CommandComplete parse(ByteBuffer buffer, int offset) { if (buffer.get(offset) != PacketMarker.B_CommandComplete.getValue()) { throw new IllegalArgumentException("this packetData not is CommandComplete"); } CommandComplete packet = new CommandComplete(); packet.length = PIOUtils.redInteger4(buffer, offset + 1); packet.commandResponse = new String(PIOUtils.redByteArray(buffer, offset + 1 + 4, packet.length - 4), UTF8) .trim(); if (packet.commandResponse.startsWith("INSERT")) { String vs[] = packet.commandResponse.replace("INSERT", "").trim().split(" +"); packet.insertId = parseInt(vs[0]); packet.affectedRows =parseInt(vs[1]); } else if (packet.commandResponse.startsWith("UPDATE")) { packet.affectedRows = parseInt(packet.commandResponse.replace("UPDATE", "").trim()); }else if(packet.commandResponse.startsWith("DELETE")){ packet.affectedRows = parseInt(packet.commandResponse.replace("DELETE", "").trim()); } return packet; } private static int parseInt(String value) { try{ return Integer.parseInt(value); }catch (Exception e) { e.printStackTrace(); } return 0; } public String getCommandResponse() { return commandResponse; } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/packet/CopyInResponse.java ================================================ package io.mycat.backend.postgresql.packet; import java.nio.ByteBuffer; // CopyInResponse (B) // Byte1('G') // 标识这条消息是一条 Start Copy In （开始拷贝进入）响应消息。 前端现在必须发送一条拷贝入数据。（如果还没准备好做这些事情， 那么发送一条 CopyFail 消息）。 // // Int32 // 以字节记的消息内容的长度，包括长度本身。 // // Int8 // 0 表示全部的 COPY 格式都是文本的（数据行由换行符分隔，字段由分隔字符分隔等等）。 1 表示全部 COPY 格式都是二进制的（类似 DataRow 格式）。 参阅 COPY 获取更多信息。 // // Int16 // 数据中要拷贝的字段数（由下面的 N 解释）。 // // Int16[N] // 每个字段将要用的格式代码，目前每个都必须是零（文本）或者一（二进制）。 如果全部拷贝格式都是文本的，那么所有的都必须是零。 import io.mycat.backend.postgresql.utils.PIOUtils; /*** * 拷贝数据开始 * * @author Coollf * */ public class CopyInResponse extends PostgreSQLPacket { /** * 标示 */ private char marker = PacketMarker.B_CopyInResponse.getValue(); /** * 长度 */ private int length; /** * 拷贝协议, 0 文本, 1 二进制 */ private DataProtocol protocol; /*** * 拷贝的数据字段数 */ private short dataLength; /** * @return the protocol */ public DataProtocol getProtocol() { return protocol; } /** * @return the dataLength */ public short getDataLength() { return dataLength; } /** * @return the columnType */ public DataProtocol[] getColumnType() { return columnType; } /** * 要拷贝数据列的类型 Int16[N] */ private DataProtocol[] columnType; @Override public int getLength() { return length; } @Override public char getMarker() { return marker; } public static CopyInResponse parse(ByteBuffer buffer, int offset) { if (buffer.get(offset) != PacketMarker.B_CopyInResponse.getValue()) { throw new IllegalArgumentException( "this packetData not is CopyInResponse"); } int _offset = offset + 1; CopyInResponse pack = new CopyInResponse(); pack.length = PIOUtils.redInteger4(buffer, _offset); _offset += 4; pack.protocol = DataProtocol.valueOf(PIOUtils.redInteger1(buffer, _offset)); _offset += 1; pack.dataLength = PIOUtils.redInteger2(buffer, _offset); _offset += 2; pack.columnType = new DataProtocol[pack.dataLength]; for (int i = 0; i < pack.columnType.length; i++) { pack.columnType[i] = DataProtocol.valueOf(PIOUtils.redInteger2( buffer, _offset)); _offset += 2; } return pack; } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/packet/CopyOutResponse.java ================================================ package io.mycat.backend.postgresql.packet; import java.nio.ByteBuffer; import io.mycat.backend.postgresql.utils.PIOUtils; // CopyOutResponse (B) // Byte1('H') // 标识这条消息是一条 Start Copy Out （开始拷贝进出）响应消息。 这条消息后面将跟着一条拷贝出数据消息。 // // Int32 // 以字节记的消息内容的长度，包括它自己。 // // Int8 // 0 表示全部拷贝格式都是文本（数据行由换行符分隔， 字段由分隔字符分隔等等）。1 表示所有拷贝格式都是二进制的（类似于 DataRow 格式）。参阅 COPY 获取更多信息。 // // Int16 // 要拷贝的数据的字段的数目（在下面的 N 说明）。 // // Int16[N] // 每个字段要试用的格式代码。目前每个都必须是零（文本）或者一（二进制）。 如果全部的拷贝格式都是文本，那么所有的都必须是零。 public class CopyOutResponse extends PostgreSQLPacket { /** * 标示 */ private char marker = PacketMarker.B_CopyOutResponse.getValue(); /** * 长度 */ private int length; /** * @return the marker */ public char getMarker() { return marker; } /** * @return the length */ public int getLength() { return length; } /** * @return the protocol */ public DataProtocol getProtocol() { return protocol; } /** * @return the dataLength */ public short getDataLength() { return dataLength; } /** * @return the columnType */ public DataProtocol[] getColumnType() { return columnType; } /** * 拷贝协议, 0 文本, 1 二进制 */ private DataProtocol protocol; /*** * 拷贝的数据字段数 */ private short dataLength; /** * 要拷贝数据列的类型 Int16[N] */ private DataProtocol[] columnType; public static CopyOutResponse parse(ByteBuffer buffer, int offset) { if (buffer.get(offset) != PacketMarker.B_CopyOutResponse.getValue()) { throw new IllegalArgumentException( "this packetData not is CopyInResponse"); } int _offset = offset + 1; CopyOutResponse pack = new CopyOutResponse(); pack.length = PIOUtils.redInteger4(buffer, _offset); _offset += 4; pack.protocol = DataProtocol.valueOf(PIOUtils.redInteger1(buffer, _offset)); _offset += 1; pack.dataLength = PIOUtils.redInteger2(buffer, _offset); _offset += 2; pack.columnType = new DataProtocol[pack.dataLength]; for (int i = 0; i < pack.columnType.length; i++) { pack.columnType[i] = DataProtocol.valueOf(PIOUtils.redInteger2( buffer, _offset)); _offset += 2; } return pack; } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/packet/DataRow.java ================================================ package io.mycat.backend.postgresql.packet; import java.nio.ByteBuffer; import io.mycat.backend.postgresql.utils.PIOUtils; // DataRow (B) // Byte1('D') // 标识这个消息是一个数据行。 // // Int32 // 以字节记的消息内容的长度，包括长度自身。 // // Int16 // 后面跟着的字段值的个数（可能是零）。 // // 然后，每个字段都会出现下面的数据域对： // // Int32 // 字段值的长度，以字节记（这个长度不包括它自己）。 可以为零。一个特殊的情况是，-1 表示一个 NULL 的字段值。 在 NULL 的情况下就没有跟着数据字段。 // // Byten // 一个字段的数值，以相关的格式代码表示的格式展现。 n 是上面的长度。 public class DataRow extends PostgreSQLPacket { public static class DataColumn { /** * 字段值的长度，以字节记（这个长度不包括它自己）。 可以为零。一个特殊的情况是，-1 表示一个 NULL 的字段值。 在 NULL * 的情况下就没有跟着数据字段。 */ private int length; private byte[] data; private boolean isNull; /** * @return the isNull */ public boolean isNull() { return isNull; } /** * @return the length */ public int getLength() { return length; } /** * @param length * the length to set */ public void setLength(int length) { this.length = length; } /** * @return the data */ public byte[] getData() { return data; } /** * @param data * the data to set */ public void setData(byte[] data) { this.data = data; } } /** * 标准 */ private char marker = PacketMarker.B_DataRow.getValue(); /** * 长度 */ private int length; /** * 列数 */ private short columnNumber; /** * @return the columnNumber */ public short getColumnNumber() { return columnNumber; } /** * @return the columns */ public DataColumn[] getColumns() { return columns; } /** * 数据列 */ private DataColumn[] columns; @Override public int getLength() { return length; } @Override public char getMarker() { return marker; } public static DataRow parse(ByteBuffer buffer, int offset) { if (buffer.get(offset) != PacketMarker.B_DataRow.getValue()) { throw new IllegalArgumentException("this packetData not is DataRow"); } int _offset = offset + 1; DataRow pack = new DataRow(); pack.length = PIOUtils.redInteger4(buffer, _offset); _offset += 4; pack.columnNumber = PIOUtils.redInteger2(buffer, _offset); _offset += 2; pack.columns = new DataColumn[pack.columnNumber]; for (int i = 0; i < pack.columns.length; i++) { DataColumn col = new DataColumn(); col.length = PIOUtils.redInteger4(buffer, _offset); _offset += 4; if (col.length == -1) { // 数据为空 col.isNull = true; } else { col.data = PIOUtils.redByteArray(buffer, _offset, col.length); _offset += col.length; } pack.columns[i] = col; } return pack; } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/packet/EmptyQueryResponse.java ================================================ package io.mycat.backend.postgresql.packet; import java.nio.ByteBuffer; import io.mycat.backend.postgresql.utils.PIOUtils; // EmptyQueryResponse (B) // Byte1('I') // 标识这条消息是对一个空查询字串的响应。 （这个消息替换了 CommandComplete。） // // Int32(4) // 以字节记的消息内容长度，包括它自己。 /******* * 空查询响应 * @author Coollf * */ public class EmptyQueryResponse extends PostgreSQLPacket { private char marker = PacketMarker.B_EmptyQueryResponse.getValue(); private int length; @Override public int getLength() { return length; } @Override public char getMarker() { return marker; } public static EmptyQueryResponse parse(ByteBuffer buffer, int offset) { if (buffer.get(offset) != PacketMarker.B_EmptyQueryResponse.getValue()) { throw new IllegalArgumentException( "this packetData not is EmptyQueryResponse"); } int _offset = offset + 1; EmptyQueryResponse pack = new EmptyQueryResponse(); pack.length = PIOUtils.redInteger4(buffer, _offset); return pack; } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/packet/ErrorResponse.java ================================================ package io.mycat.backend.postgresql.packet; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; //ErrorResponse (B) //Byte1('E') //标识消息是一条错误。 // //Int32 //以字节记的消息内容的长度，包括长度本身。 // //消息体由一个或多个标识出来的字段组成，后面跟着一个字节零作为终止符。 字段可以以任何顺序出现。对于每个字段都有下面的东西： // //Byte1 //一个标识字段类型的代码；如果为零，这就是消息终止符并且不会跟着有字串。 目前定义的字段类型在 Section 43.5 列出。 因为将来可能增加更多的字段类型，所以前端应该不声不响地忽略不认识类型的字段。 // //String //字段值。 public class ErrorResponse extends PostgreSQLPacket { /********* * 解析错误包 * * @param buffer * @param offset * @return * @throws UnsupportedEncodingException * @throws IllegalAccessException */ public static ErrorResponse parse(ByteBuffer buffer, int offset) throws IllegalArgumentException { if ((char) buffer.get(offset) != PacketMarker.B_Error.getValue()) { throw new IllegalArgumentException("this packet not is ErrorResponse"); } ErrorResponse err = new ErrorResponse(); err.length = buffer.getInt(offset + 1); err.mark = buffer.get(offset + 1 + 4); if (err.mark != 0) { byte[] str = new byte[err.length - (4+4)]; for(int i =0;i params; // 协议参数 @Override public int getLength() { return 0; } @Override @Deprecated public char getMarker() { return marker; } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/packet/Terminate.java ================================================ package io.mycat.backend.postgresql.packet; import java.nio.ByteBuffer; // // Terminate (F) // Byte1('X') // 标识消息是一个终止消息。 // // Int32(4) // 以字节记的消息内容的长度，包括长度自身。 import io.mycat.backend.postgresql.utils.PIOUtils; /*** * 终止命令 * * @author Coollf * */ public class Terminate extends PostgreSQLPacket { private int length = 4; @Override public int getLength() { return length; } @Override public char getMarker() { return PacketMarker.F_Terminate.getValue(); } public void write(ByteBuffer buffer) { PIOUtils.SendChar(getMarker(), buffer); PIOUtils.SendInteger4(getLength(), buffer); } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/utils/MD5Digest.java ================================================ /*------------------------------------------------------------------------- * * Copyright (c) 2003-2014, PostgreSQL Global Development Group * * *------------------------------------------------------------------------- */ package io.mycat.backend.postgresql.utils; /** * MD5-based utility function to obfuscate passwords before network * transmission. * * @author Jeremy Wohl */ import java.security.MessageDigest; public class MD5Digest { private MD5Digest() { } /* * Encodes user/password/salt information in the following way: * MD5(MD5(password + user) + salt) * * @param user The connecting user. * @param password The connecting user's password. * @param salt A four-salt sent by the server. * * @return A 35-byte array, comprising the string "md5" and an MD5 digest. */ public static byte[] encode(byte user[], byte password[], byte salt[]) { MessageDigest md; byte[] temp_digest, pass_digest; byte[] hex_digest = new byte[35]; try { md = MessageDigest.getInstance("MD5"); md.update(password); md.update(user); temp_digest = md.digest(); bytesToHex(temp_digest, hex_digest, 0); md.update(hex_digest, 0, 32); md.update(salt); pass_digest = md.digest(); bytesToHex(pass_digest, hex_digest, 3); hex_digest[0] = (byte) 'm'; hex_digest[1] = (byte) 'd'; hex_digest[2] = (byte) '5'; } catch (Exception e) { ; // "MessageDigest failure; " + e } return hex_digest; } /* * Turn 16-byte stream into a human-readable 32-byte hex string */ private static void bytesToHex(byte[] bytes, byte[] hex, int offset) { final char lookup[] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; int i, c, j, pos = offset; for (i = 0; i < 16; i++) { c = bytes[i] & 0xFF; j = c >> 4; hex[pos++] = (byte) lookup[j]; j = (c & 0xF); hex[pos++] = (byte) lookup[j]; } } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/utils/PIOUtils.java ================================================ package io.mycat.backend.postgresql.utils; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.Charset; /***** * PostgreSQL io工具类 * * @author Coollf * */ public class PIOUtils { public final static Charset UTF8 = Charset.forName("utf-8"); /** * Sends a 4-byte integer to the back end * * @param val * the integer to be sent * @exception IOException * if an I/O error occurs */ public static void SendInteger4(int val, ByteBuffer buffer) { byte[] _int4buf = new byte[4]; _int4buf[0] = (byte) (val >>> 24); _int4buf[1] = (byte) (val >>> 16); _int4buf[2] = (byte) (val >>> 8); _int4buf[3] = (byte) (val); buffer.put(_int4buf); } public static int redInteger4(ByteBuffer buffer, int offset) { return buffer.getInt(offset); } /*** * 读取数据 * @param buffer * @param offset * @return */ public static short redInteger2(ByteBuffer buffer, int offset) { return buffer.getShort(offset); } /** * Sends a 2-byte integer (short) to the back end * * @param val * the integer to be sent * @exception IOException * if an I/O error occurs or val cannot be * encoded in 2 bytes */ public static void SendInteger2(int val, ByteBuffer buffer) throws IOException { if (val < Short.MIN_VALUE || val > Short.MAX_VALUE) { throw new IOException( "Tried to send an out-of-range integer as a 2-byte value: " + val); } byte[] _int2buf = new byte[2]; _int2buf[0] = (byte) (val >>> 8); _int2buf[1] = (byte) val; buffer.put(_int2buf); } public static void Send(byte[] encodedParam, ByteBuffer buffer) { buffer.put(encodedParam); } public static void SendChar(int i, ByteBuffer buffer) { buffer.put((byte) i); } /*** * 读取数组信息 * * @param buffer * @param offset * @param length * @return */ public static byte[] redByteArray(ByteBuffer buffer, int offset, int length) { byte[] dst = new byte[length]; for (int i = 0; i < length; i++) { dst[i] = buffer.get(offset + i); } return dst; } public static void SendString(String string, ByteBuffer buffer) { buffer.put(string.getBytes(UTF8)); } public static String redString(ByteBuffer buffer, int offset, Charset charset) throws IOException { ByteArrayOutputStream out =new ByteArrayOutputStream(); for(int i=offset ;i< buffer.limit();i++){ if(((char)buffer.get(i)) == '\0'){ break; } out.write(new byte[]{buffer.get(i)}); } return new String(out.toByteArray(),charset); } /** * 读取1byte数据 * @param buffer * @param _offset * @return */ public static byte redInteger1(ByteBuffer buffer, int _offset) { return buffer.get(_offset); } public static void SendByte(byte b, ByteBuffer buffer) { buffer.put(b); } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/utils/PacketUtils.java ================================================ package io.mycat.backend.postgresql.utils; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.TimeZone; import io.mycat.backend.postgresql.packet.AuthenticationPacket; import io.mycat.backend.postgresql.packet.BackendKeyData; import io.mycat.backend.postgresql.packet.CommandComplete; import io.mycat.backend.postgresql.packet.CopyInResponse; import io.mycat.backend.postgresql.packet.CopyOutResponse; import io.mycat.backend.postgresql.packet.DataRow; import io.mycat.backend.postgresql.packet.EmptyQueryResponse; import io.mycat.backend.postgresql.packet.ErrorResponse; import io.mycat.backend.postgresql.packet.NoticeResponse; import io.mycat.backend.postgresql.packet.ParameterStatus; import io.mycat.backend.postgresql.packet.ParseComplete; import io.mycat.backend.postgresql.packet.PostgreSQLPacket; import io.mycat.backend.postgresql.packet.ReadyForQuery; import io.mycat.backend.postgresql.packet.RowDescription; public class PacketUtils { public static List parsePacket(ByteBuffer buffer,int offset,int readLength) throws IOException{ final ByteBuffer bytes = buffer; List pgs = new ArrayList<>(); while(offset < readLength){ char MAKE = (char)bytes.get(offset); PostgreSQLPacket pg = null; switch (MAKE) { case 'R': pg = AuthenticationPacket.parse(bytes, offset); break; case 'E': pg = ErrorResponse.parse(bytes, offset); break; case 'K': pg = BackendKeyData.parse(bytes, offset); break; case 'S': pg = ParameterStatus.parse(bytes, offset); break; case 'Z': pg = ReadyForQuery.parse(bytes, offset); break; case 'N': pg = NoticeResponse.parse(bytes, offset); break; case 'C': pg = CommandComplete.parse(bytes, offset); break; case 'T': pg = RowDescription.parse(bytes, offset); break; case 'D': pg = DataRow.parse(bytes, offset); break; case 'I': pg = EmptyQueryResponse.parse(bytes, offset); break; case 'G': pg = CopyInResponse.parse(bytes, offset); break; case 'H': pg = CopyOutResponse.parse(bytes, offset); break; case '1': pg = ParseComplete.parse(bytes, offset); break; default: throw new RuntimeException("Unknown packet"); } if (pg != null) { offset = offset + pg.getLength() + 1; pgs.add(pg); } } return pgs; } @Deprecated private static List parsePacket(byte[] bytes, int offset, int readLength) throws IOException { List pgs = new ArrayList<>(); while (offset < readLength) { char MAKE = (char) bytes[offset]; PostgreSQLPacket pg = null; switch (MAKE) { case 'R': pg = AuthenticationPacket.parse(ByteBuffer.wrap(bytes), offset); break; case 'E': pg = ErrorResponse.parse(ByteBuffer.wrap(bytes), offset); break; case 'K': pg = BackendKeyData.parse(ByteBuffer.wrap(bytes), offset); break; case 'S': pg = ParameterStatus.parse(ByteBuffer.wrap(bytes), offset); break; case 'Z': pg = ReadyForQuery.parse(ByteBuffer.wrap(bytes), offset); break; case 'N': pg = NoticeResponse.parse(ByteBuffer.wrap(bytes), offset); break; case 'C': pg = CommandComplete.parse(ByteBuffer.wrap(bytes), offset); break; case 'T': pg = RowDescription.parse(ByteBuffer.wrap(bytes), offset); break; case 'D': pg = DataRow.parse(ByteBuffer.wrap(bytes), offset); break; case 'I': pg = EmptyQueryResponse.parse(ByteBuffer.wrap(bytes), offset); break; case 'G': pg = CopyInResponse.parse(ByteBuffer.wrap(bytes), offset); break; case 'H': pg = CopyOutResponse.parse(ByteBuffer.wrap(bytes), offset); break; case '1': pg = ParseComplete.parse(ByteBuffer.wrap(bytes), offset); break; default: throw new RuntimeException("Unknown packet"); } if (pg != null) { offset = offset + pg.getLength() + 1; pgs.add(pg); } } return pgs; } /** * Convert Java time zone to postgres time zone. All others stay the same * except that GMT+nn changes to GMT-nn and vise versa. * * @return The current JVM time zone in postgresql format. */ public static String createPostgresTimeZone() { String tz = TimeZone.getDefault().getID(); if (tz.length() <= 3 || !tz.startsWith("GMT")) { return tz; } char sign = tz.charAt(3); String start; if (sign == '+') { start = "GMT-"; } else if (sign == '-') { start = "GMT+"; } else { // unknown type return tz; } return start + tz.substring(4); } public static ByteBuffer makeStartUpPacket(String user, String database) throws IOException { List paramList = new ArrayList(); String appName = "MyCat-Server"; paramList.add(new String[] { "user", user }); paramList.add(new String[] { "database", database }); paramList.add(new String[] { "client_encoding", "UTF8" }); paramList.add(new String[] { "DateStyle", "ISO" }); paramList.add(new String[] { "TimeZone", createPostgresTimeZone() }); paramList.add(new String[] { "extra_float_digits", "3" }); paramList.add(new String[] { "application_name", appName }); String[][] params = paramList.toArray(new String[0][]); StringBuilder details = new StringBuilder(); for (int i = 0; i < params.length; ++i) { if (i != 0) { details.append(", "); } details.append(params[i][0]); details.append("="); details.append(params[i][1]); } /* * Precalculate message length and encode params. */ int length = 4 + 4; byte[][] encodedParams = new byte[params.length * 2][]; for (int i = 0; i < params.length; ++i) { encodedParams[i * 2] = params[i][0].getBytes("UTF-8"); encodedParams[i * 2 + 1] = params[i][1].getBytes("UTF-8"); length += encodedParams[i * 2].length + 1 + encodedParams[i * 2 + 1].length + 1; } length += 1; // Terminating \0 ByteBuffer buffer = ByteBuffer.allocate(length); /* * Send the startup message. */ PIOUtils.SendInteger4(length, buffer); PIOUtils.SendInteger2(3, buffer); // protocol major PIOUtils.SendInteger2(0, buffer); // protocol minor for (byte[] encodedParam : encodedParams) { PIOUtils.Send(encodedParam, buffer); PIOUtils.SendChar(0, buffer); } PIOUtils.Send(new byte[] { 0 }, buffer); return buffer; } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/utils/PgPacketApaterUtils.java ================================================ package io.mycat.backend.postgresql.utils; import io.mycat.backend.postgresql.packet.DataRow; import io.mycat.backend.postgresql.packet.DataRow.DataColumn; import io.mycat.backend.postgresql.packet.PostgreSQLPacket.DateType; import io.mycat.backend.postgresql.packet.RowDescription; import io.mycat.backend.postgresql.packet.RowDescription.ColumnDescription; import io.mycat.config.Fields; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.RowDataPacket; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; /********* * 数据包适配 * @author Coollf * */ public class PgPacketApaterUtils { private static final Charset UTF8 = Charset.forName("utf-8"); /** * 列标示转换成Mysql的数据 * @param description * @return */ public static List rowDescConvertFieldPacket(RowDescription description){ List fieldPks = new ArrayList(description.getColumnNumber()); for(ColumnDescription c: description.getColumns()){ FieldPacket fieldPk = new FieldPacket(); fieldPk.name = c.getColumnName().trim().getBytes(UTF8); fieldPk.type = convertFieldType(c.getColumnType()); fieldPks.add(fieldPk); } //TODO 等待实现 return fieldPks; } /*** * 将pg的sql类型转换成 * @param columnType * @return */ private static int convertFieldType(DateType columnType) { if(columnType == DateType.timestamp_){ return Fields.FIELD_TYPE_TIMESTAMP; } if(columnType == DateType.int2_ || columnType == DateType.int4_ || columnType == DateType.int8_ ){ return Fields.FIELD_TYPE_INT24; } if(columnType == DateType.decimal_){ return Fields.FIELD_TYPE_NEW_DECIMAL; } if(columnType == DateType.UNKNOWN){ } return Fields.FIELD_TYPE_VARCHAR; } /*** * 行数据转换成mysql的数据 * @param dataRow * @return */ public static RowDataPacket rowDataConvertRowDataPacket(DataRow dataRow){ RowDataPacket curRow = new RowDataPacket(dataRow.getColumnNumber()); for(DataColumn c: dataRow.getColumns()){ curRow.add(c.getData()); } return curRow; } } ================================================ FILE: src/main/java/io/mycat/backend/postgresql/utils/PgSqlApaterUtils.java ================================================ package io.mycat.backend.postgresql.utils; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; public class PgSqlApaterUtils { /** * 查询表结构 */ private static final String SHOW_TABLE_STATUS_SQL_PREFIX ="SHOW TABLE STATUS LIKE"; /******* * 展示建表语句 */ private static final String SHOW_CREATE_TABLE_SQL_PREFIX = "SHOW CREATE TABLE"; /** * 表机构信息 ,包含主键 */ private static final String SHOW_COLUMNS_SQL_PREFIX ="SHOW COLUMNS FROM"; public static String apater(String sql){ sql = sql.replaceAll("`","\""); final String SQL = sql.toUpperCase().replaceAll("`","\""); String _mapperSql = stream.get(SQL); if(_mapperSql!=null){ return _mapperSql; } if (SQL.startsWith(SHOW_TABLE_STATUS_SQL_PREFIX)){ return doApaterTableStatusSql(SQL); } if(SQL.startsWith(SHOW_CREATE_TABLE_SQL_PREFIX)){ return doApaterCreateTabelSql(SQL); } if(SQL.startsWith(SHOW_COLUMNS_SQL_PREFIX)){ return doApaterColumnsSql(SQL); } if(SQL.indexOf("LIMIT")!=-1 && SQL.indexOf("OFFSET") == -1){//非pgsql 分页语句 return doApaterPagingSql(SQL,sql); } return sql; } /******* * 获取列信息SQL 语句 * @param sql * @return */ private static String doApaterColumnsSql(String sql) { return "SELECT '' as \"Field\" ,'' as \"Type\" ,''as \"Null\" ,'' as \"Key\" ,'' as \"Default\" , '' as \"Extra\" from pg_namespace where 1=2"; } private static String doApaterPagingSql(final String SQL, String sql) { int index = SQL.indexOf("LIMIT"); String pagingPart = sql.substring(index); String selectPart = sql.substring(0, index); String[] pk = pagingPart.split("(\\s+)|(,)"); List slices = new ArrayList(); for (String token : pk) { if (token.trim().length() > 0) { slices.add(token); } } if (slices.size() == 3) { return selectPart + String.format("%s %s offset %s", slices.get(0), slices.get(2), slices.get(1)); } if (slices.size() == 2) { return selectPart + String.format(" %s %s offset 0 ", slices.get(0), slices.get(1)); } return sql;// 无法处理分页sql原样返回 } private static String doApaterCreateTabelSql(String sql) { return "select '' as Table ,'' as \"Create Table\" from pg_namespace where 1=2"; } /******** * 进行表结构语句适配 * @param sql * @return */ private static String doApaterTableStatusSql(String sql) { String tableName =sql.substring(SHOW_TABLE_STATUS_SQL_PREFIX.length()); StringBuilder sb = new StringBuilder(); sb.append("SELECT").append(" "); sb.append(" attname AS NAME,").append(" "); sb.append(" 'InnoDB' AS Engine,").append(" "); sb.append(" 10 AS VERSION,").append(" "); sb.append(" 'Compact' AS Row_format,").append(" "); sb.append(" 0 AS ROWS,").append(" "); sb.append(" 10000 AS Avg_row_length,").append(" "); sb.append(" 10000 AS Data_length,").append(" "); sb.append(" 0 AS Max_data_length,").append(" "); sb.append(" 0 AS Index_length,").append(" "); sb.append(" 0 AS Data_free,").append(" "); sb.append(" NULL AS Auto_increment,").append(" "); sb.append(" NULL AS Create_time,").append(" "); sb.append(" NULL AS Update_time,").append(" "); sb.append(" NULL AS Check_time,").append(" "); sb.append(" 'utf8_general_ci' AS COLLATION,").append(" "); sb.append(" NULL AS Checksum,").append(" "); sb.append(" '' AS Create_options,").append(" "); sb.append(" '' AS COMMENT").append(" "); sb.append("FROM").append(" "); sb.append(" pg_attribute").append(" "); sb.append("INNER JOIN pg_class ON pg_attribute.attrelid = pg_class.oid").append(" "); sb.append("INNER JOIN pg_type ON pg_attribute.atttypid = pg_type.oid").append(" "); sb.append("LEFT OUTER JOIN pg_attrdef ON pg_attrdef.adrelid = pg_class.oid").append(" "); sb.append("AND pg_attrdef.adnum = pg_attribute.attnum").append(" "); sb.append("LEFT OUTER JOIN pg_description ON pg_description.objoid = pg_class.oid").append(" "); sb.append("AND pg_description.objsubid = pg_attribute.attnum").append(" "); sb.append("WHERE").append(" "); sb.append(" pg_attribute.attnum > 0").append(" "); sb.append("AND attisdropped <> 't'").append(" "); sb.append("AND pg_class.relname =").append(tableName).append(" "); sb.append("ORDER BY").append(" "); sb.append(" pg_attribute.attnum").append(" "); return sb.toString(); } public static Map stream = new HashMap<>(); static{ stream.put("SELECT @@CHARACTER_SET_DATABASE, @@COLLATION_DATABASE".toUpperCase(), "SELECT 'utf8' as \"@@character_set_database\", 'utf8_general_ci' as \"@@collation_database\""); stream.put("SHOW STATUS", "SELECT 'Aborted_clients' as \"Variable\" , 0 as \"Value\" where 1=2 "); stream.put("SHOW FULL TABLES WHERE Table_type != 'VIEW'".toUpperCase(), "select tablename as \"Tables_In_\",'BASE TABLE' as \"Table_Type\" from pg_tables where schemaname ='public'"); stream.put("SHOW ENGINES","SELECT DISTINCT 'InnoDB' as Engine ,\t'DEFAULT' as Support , \t'Supports transactions,row-level locking and foreign keys' as \"Comment\"\t,'YES' as \"Transactions\" ,\t'YES' as \"XA\",'YES' as \"Savepoints\" from pg_tablespace\n"); } } ================================================ FILE: src/main/java/io/mycat/buffer/BufferArray.java ================================================ package io.mycat.buffer; import io.mycat.util.ByteBufferUtil; import java.nio.ByteBuffer; import java.util.Collections; import java.util.LinkedList; import java.util.List; /** * used for large data write ,composed by buffer array, when a large MySQL * package write ,shoud use this object to write data * * use DirectByteBuffer for alloc buffer * @author wuzhih * @author zagnix */ public class BufferArray { private final BufferPool bufferPool; private ByteBuffer curWritingBlock; private List writedBlockLst = Collections.emptyList(); public BufferArray(BufferPool bufferPool) { super(); this.bufferPool = bufferPool; curWritingBlock = bufferPool.allocate(bufferPool.getChunkSize()); } public ByteBuffer checkWriteBuffer(int capacity) { if (capacity > curWritingBlock.remaining()) { addtoBlock(curWritingBlock); curWritingBlock = bufferPool.allocate(capacity); return curWritingBlock; } else { return curWritingBlock; } } public int getBlockCount() { return writedBlockLst.size()+1; } private void addtoBlock(ByteBuffer buffer) { if (writedBlockLst.isEmpty()) { writedBlockLst = new LinkedList(); } writedBlockLst.add(buffer); } public ByteBuffer getCurWritingBlock() { return curWritingBlock; } public List getWritedBlockLst() { return writedBlockLst; } public void clear() { curWritingBlock = null; writedBlockLst.clear(); writedBlockLst = null; } public ByteBuffer write(byte[] src) { int offset = 0; int remains = src.length; while (remains > 0) { int writeable = curWritingBlock.remaining(); if (writeable >= remains) { // can write whole srce curWritingBlock.put(src, offset, remains); break; } else { // can write partly curWritingBlock.put(src, offset, writeable); offset += writeable; remains -= writeable; addtoBlock(curWritingBlock); curWritingBlock = bufferPool.allocate(bufferPool.getChunkSize()); continue; } } return curWritingBlock; } public byte[] writeToByteArrayAndRecycle() { BufferArray bufferArray=this; try { int size=0; List blockes = bufferArray.getWritedBlockLst(); if (!bufferArray.getWritedBlockLst().isEmpty()) { for (ByteBuffer curBuf : blockes) { curBuf.flip(); size+=curBuf.remaining(); } } ByteBuffer curBuf = bufferArray.getCurWritingBlock(); curBuf.flip(); if(curBuf.hasRemaining()) { size += curBuf.remaining(); } if(size>0) { int offset=0; byte[] all=new byte[size]; if (!bufferArray.getWritedBlockLst().isEmpty()) { for (ByteBuffer tBuf : blockes) { ByteBufferUtil.arrayCopy(tBuf,0,all,offset,tBuf.remaining()); offset+=tBuf.remaining(); bufferPool.recycle(tBuf); } } ByteBuffer tBuf = bufferArray.getCurWritingBlock(); if(tBuf.hasRemaining()) { ByteBufferUtil.arrayCopy(tBuf,0,all,offset,tBuf.remaining()); bufferPool.recycle(tBuf); // offset += curBuf.remaining(); } return all; } } finally { bufferArray.clear(); } return EMPTY; } private static byte[] EMPTY=new byte[0]; } ================================================ FILE: src/main/java/io/mycat/buffer/BufferPool.java ================================================ package io.mycat.buffer; import java.nio.ByteBuffer; import java.util.concurrent.ConcurrentHashMap; /** * 缓冲池 * * @author Hash Zhang * @version 1.0 * @time 12:19 2016/5/23 */ public interface BufferPool { public ByteBuffer allocate(int size); public void recycle(ByteBuffer theBuf); public long capacity(); public long size(); public int getConReadBuferChunk(); public int getSharedOptsCount(); public int getChunkSize(); public ConcurrentHashMap getNetDirectMemoryUsage(); public BufferArray allocateArray(); } ================================================ FILE: src/main/java/io/mycat/buffer/ByteBufferArena.java ================================================ package io.mycat.buffer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.nio.ByteBuffer; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; /** * 仿照Netty的思路，针对MyCat内存缓冲策略优化 * ByteBufferArena维护着锁还有所有list * * @author Hash Zhang * @version 1.0 * @time 17:19 2016/5/17 * @see @https://github.com/netty/netty */ public class ByteBufferArena implements BufferPool { private static final Logger LOGGER = LoggerFactory.getLogger(ByteBufferChunkList.class); private final ByteBufferChunkList q[]; private final AtomicInteger chunkCount = new AtomicInteger(0); private final AtomicInteger failCount = new AtomicInteger(0); private static final int FAIL_THRESHOLD = 1000; private final int pageSize; private final int chunkSize; private final AtomicLong capacity; private final AtomicLong size; private final ConcurrentHashMap sharedOptsCount; /** * 记录对线程ID->该线程的所使用Direct Buffer的size */ private final ConcurrentHashMap memoryUsage; private final int conReadBuferChunk; public ByteBufferArena(int chunkSize, int pageSize, int chunkCount, int conReadBuferChunk) { try { this.chunkSize = chunkSize; this.pageSize = pageSize; this.chunkCount.set(chunkCount); this.conReadBuferChunk = conReadBuferChunk; q = new ByteBufferChunkList[6]; q[5] = new ByteBufferChunkList(100, Integer.MAX_VALUE, chunkSize, pageSize, 0); q[4] = new ByteBufferChunkList(75, 100, chunkSize, pageSize, 0); q[3] = new ByteBufferChunkList(50, 100, chunkSize, pageSize, 0); q[2] = new ByteBufferChunkList(25, 75, chunkSize, pageSize, 0); q[1] = new ByteBufferChunkList(1, 50, chunkSize, pageSize, 0); q[0] = new ByteBufferChunkList(Integer.MIN_VALUE, 25, chunkSize, pageSize, chunkCount); q[0].nextList = q[1]; q[1].nextList = q[2]; q[2].nextList = q[3]; q[3].nextList = q[4]; q[4].nextList = q[5]; q[5].nextList = null; q[5].prevList = q[4]; q[4].prevList = q[3]; q[3].prevList = q[2]; q[2].prevList = q[1]; q[1].prevList = q[0]; q[0].prevList = null; capacity = new AtomicLong(6 * chunkCount * chunkSize); size = new AtomicLong(6 * chunkCount * chunkSize); sharedOptsCount = new ConcurrentHashMap<>(); memoryUsage = new ConcurrentHashMap<>(); } finally { } } @Override public ByteBuffer allocate(int reqCapacity) { try { ByteBuffer byteBuffer = null; int i = 0, count = 0; while (byteBuffer == null) { if (i > 5) { i = 0; count = failCount.incrementAndGet(); if (count > FAIL_THRESHOLD) { try { expand(); } finally { } } } byteBuffer = q[i].allocate(reqCapacity); i++; } // if (count > 0) { // System.out.println("count: " + count); // System.out.println(failCount.get()); // } // printList(); capacity.addAndGet(-reqCapacity); final Thread thread = Thread.currentThread(); final long threadId = thread.getId(); if (memoryUsage.containsKey(threadId)){ memoryUsage.put(threadId,memoryUsage.get(thread.getId())+reqCapacity); }else { memoryUsage.put(threadId, (long) reqCapacity); } if (sharedOptsCount.containsKey(thread)) { int currentCount = sharedOptsCount.get(thread); currentCount++; sharedOptsCount.put(thread,currentCount); } else{ sharedOptsCount.put(thread,0); } return byteBuffer; } finally { } } private void expand() { LOGGER.warn("Current Buffer Size is not enough! Expanding Byte buffer!"); ByteBufferChunk byteBufferChunk = new ByteBufferChunk(pageSize, chunkSize); q[0].byteBufferChunks.add(byteBufferChunk); failCount.set(0); } @Override public void recycle(ByteBuffer byteBuffer) { final long size = byteBuffer != null?byteBuffer.capacity():0; try { int i; for (i = 0; i < 6; i++) { if (q[i].free(byteBuffer)) { break; } } if (i > 5) { LOGGER.warn("This ByteBuffer is not maintained in ByteBufferArena!"); return; } final Thread thread = Thread.currentThread(); final long threadId = thread.getId(); if (memoryUsage.containsKey(threadId)){ memoryUsage.put(threadId,memoryUsage.get(thread.getId())-size); } if (sharedOptsCount.containsKey(thread)) { int currentCount = sharedOptsCount.get(thread); currentCount--; sharedOptsCount.put(thread,currentCount); } else{ sharedOptsCount.put(thread,0); } capacity.addAndGet(byteBuffer.capacity()); return; } finally { } } private void printList() { for (int i = 0; i < 6; i++) { System.out.println(i + ":" + q[i].byteBufferChunks.toString()); } } @Override public long capacity() { return capacity.get(); } @Override public long size() { return size.get(); } @Override public int getConReadBuferChunk() { return conReadBuferChunk; } @Override public int getSharedOptsCount() { final Set integers = (Set) sharedOptsCount.values(); int count = 0; for(int i : integers){ count += i; } return count; } /** * 这里pageSize就是DirectByteBuffer的chunksize * @return */ @Override public int getChunkSize() { return pageSize; } @Override public ConcurrentHashMap getNetDirectMemoryUsage() { return memoryUsage; } @Override public BufferArray allocateArray() { return new BufferArray(this); } } ================================================ FILE: src/main/java/io/mycat/buffer/ByteBufferChunk.java ================================================ package io.mycat.buffer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import sun.nio.ch.DirectBuffer; import java.nio.ByteBuffer; /** * 仿照Netty的思路，针对MyCat内存缓冲策略优化 * Chunk由Page组成，是一块连续内存，由memoryMap和depthMap定义成一种平衡二叉树的管理结构 * * @author Hash Zhang * @version 1.0 * @time 17:19 2016/5/17 * @see @https://github.com/netty/netty */ public class ByteBufferChunk implements Comparable{ private static final Logger LOGGER = LoggerFactory.getLogger(ByteBufferChunk.class); private final byte[] memoryMap; private final byte[] depthMap; private final ByteBuffer buf; //in bytes private final int pageSize; //in bytes private final int chunkSize; private final int chunkPageSize; private final int maxOrder; private final byte unusable; private final int log2PageSize; final long bufAddress; private int freeBytes; ByteBufferChunk prev; ByteBufferChunk next; ByteBufferChunkList parent; public ByteBufferChunk(int pageSize, int chunkSize) { this.pageSize = pageSize; this.chunkSize = chunkSize; this.chunkPageSize = chunkSize / pageSize; this.maxOrder = log2(this.chunkPageSize) + 1; this.unusable = (byte) this.maxOrder; this.freeBytes = chunkSize; this.buf = ByteBuffer.allocateDirect(chunkSize); this.bufAddress = ((DirectBuffer) buf).address(); this.depthMap = new byte[(1 << this.maxOrder)]; this.memoryMap = new byte[this.depthMap.length]; this.log2PageSize = log2(pageSize); int memoryMapIndex = 1; for (int d = 0; d < maxOrder; ++d) { // move down the tree one level at a time int depth = 1 << d; for (int p = 0; p < depth; ++p) { // in each level traverse left to right and set value to the depth of subtree memoryMap[memoryMapIndex] = (byte) d; depthMap[memoryMapIndex] = (byte) d; memoryMapIndex++; } } } public boolean isInThisChunk(ByteBuffer byteBuffer) { long address = ((DirectBuffer) byteBuffer).address(); return (address >= bufAddress) && (address < bufAddress + chunkSize); } public int usage() { final int freeBytes = this.freeBytes; if (freeBytes == 0) { return 100; } int freePercentage = (int) (freeBytes * 100L / chunkSize); if (freePercentage == 0) { return 99; } return 100 - freePercentage; } public synchronized ByteBuffer allocateRun(int normCapacity) { if(normCapacity > chunkSize){ LOGGER.warn("try to acquire a buffer with larger size than chunkSize!"); return null; } int d = this.maxOrder - 2 - (log2(normCapacity) - this.log2PageSize); if (d > this.maxOrder - 1) { d = maxOrder - 1; } int id = allocateNode(d); if (id < 0) { return null; } freeBytes -= runLength(id); int start = calculateStart(id); int end = start + runLength(id); buf.limit(end); buf.position(start); // printMemoryMap(); return buf.slice(); } private int calculateStart(int id) { int count = 0; for (int i = 1; i < depthMap.length; i++) { if (depthMap[i] < depthMap[id]) { continue; } else if (depthMap[i] == depthMap[id]) { if (i == id) { break; } else { count += runLength(i); } } else { break; } } return count; } private int runLength(int id) { // represents the size in #bytes supported by node 'id' in the tree return 1 << log2(chunkSize) - depthMap[id]; } private int allocateNode(int d) { int id = 1; int initial = -(1 << d); // has last d bits = 0 and rest all = 1 byte val = memoryMap[id]; if (val > d) { // unusable return -1; } while (val < d || (id & initial) == 0) { // id & initial == 1 << d for all ids at depth d, for < d it is 0 id <<= 1; val = memoryMap[id]; if (val > d) { id ^= 1; val = memoryMap[id]; } } byte value = memoryMap[id]; assert value == d && (id & initial) == 1 << d : String.format("val = %d, id & initial = %d, d = %d", value, id & initial, d); memoryMap[id] = unusable; // mark as unusable updateParentsAlloc(id); return id; } private void updateParentsAlloc(int id) { while (id > 1) { int parentId = id >>> 1; byte val1 = memoryMap[id]; byte val2 = memoryMap[id ^ 1]; byte val = val1 < val2 ? val1 : val2; memoryMap[parentId] = val; id = parentId; } } public synchronized void freeByteBuffer(ByteBuffer byteBuffer) { long address = ((DirectBuffer) byteBuffer).address(); int relativeAddress = (int) (address - bufAddress); int length = byteBuffer.capacity(); int depth = maxOrder - 1 - log2(length / pageSize); int count = 0; int i; for (i = 0; i < depthMap.length; i++) { if (depthMap[i] == depth) { if (count == relativeAddress) { break; } count += length; } if (depthMap[i] > depth) { break; } } free(i); } private void free(int handle) { if (memoryMap[handle] != depthMap[handle]) { freeBytes += runLength(handle); memoryMap[handle] = depthMap[handle]; updateParentsFree(handle); } } private void updateParentsFree(int id) { int logChild = depthMap[id] + 1; while (id > 1) { int parentId = id >>> 1; byte val1 = memoryMap[id]; byte val2 = memoryMap[id ^ 1]; logChild -= 1; // in first iteration equals log, subsequently reduce 1 from logChild as we traverse up if (val1 == logChild && val2 == logChild) { memoryMap[parentId] = (byte) (logChild - 1); } else { byte val = val1 < val2 ? val1 : val2; memoryMap[parentId] = val; } id = parentId; } } private static int log2(int chunkSize) { if (chunkSize <= 0) { LOGGER.warn("invalid parameter!"); throw new IllegalArgumentException(); } return Integer.SIZE - 1 - Integer.numberOfLeadingZeros(chunkSize); } private void printMemoryMap() { int l = 1; for (int i = 0; i < this.maxOrder; i++) { int j = (int) Math.pow(2, i); for (int k = 0; k < j; k++) { System.out.print(this.memoryMap[l] + "|"); l++; } System.out.println(); } System.out.println(); } public static void main(String[] args) { int pageSize = 256; int chunkSize = 1024 * 1024 * 64; ByteBufferChunk byteBufferChunk = new ByteBufferChunk(pageSize, chunkSize); int chunkCount = 8; int allocTimes = 102400; long start = System.currentTimeMillis(); for (int i = 0; i < allocTimes; i++) { // System.out.println("allocate "+i); // long start=System.nanoTime(); int size = 256; ByteBuffer byteBufer = byteBufferChunk.allocateRun(size); // System.out.println("alloc "+size+" usage "+(System.nanoTime()-start)); // start=System.nanoTime(); // byteBufferArena.recycle(byteBufer); // System.out.println("recycle usage "+(System.nanoTime()-start)); } long used = (System.currentTimeMillis() - start); System.out.println("total used time " + used + " avg speed " + allocTimes / used); } @Override public int compareTo(Object o) { return -1; } } ================================================ FILE: src/main/java/io/mycat/buffer/ByteBufferChunkList.java ================================================ package io.mycat.buffer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentSkipListSet; /** * 仿照Netty的思路，针对MyCat内存缓冲策略优化 * ChunkList维护着一个指向一串Chunk的头结点，访问策略由minUsage，maxUsage决定 * * @author Hash Zhang * @version 1.0 * @time 17:19 2016/5/17 * @see @https://github.com/netty/netty */ public class ByteBufferChunkList { private static final Logger LOGGER = LoggerFactory.getLogger(ByteBufferChunkList.class); private final int minUsage; private final int maxUsage; Set byteBufferChunks; ByteBufferChunkList prevList; ByteBufferChunkList nextList; public ByteBufferChunkList(int minUsage, int maxUsage, int chunkSize, int pageSize, int numOfChunks) { this.minUsage = minUsage; this.maxUsage = maxUsage; byteBufferChunks = new ConcurrentSkipListSet<>(); for (int i = 0; i < numOfChunks; i++) { ByteBufferChunk chunk = new ByteBufferChunk(pageSize, chunkSize); byteBufferChunks.add(chunk); } } public ByteBufferChunk getIndex(ByteBuffer buffer) { for(ByteBufferChunk byteBufferChunk : byteBufferChunks){ if (byteBufferChunk.isInThisChunk(buffer)) { return byteBufferChunk; } } return null; } ByteBuffer allocate(int reqCapacity) { for (ByteBufferChunk cur : byteBufferChunks) { ByteBuffer buf = cur.allocateRun(reqCapacity); if (buf == null) { continue; } else { final int usage = cur.usage(); if (usage >= maxUsage) { ByteBufferChunkList next = nextList; ByteBufferChunkList current = this; while (next != null) { current.byteBufferChunks.remove(cur); next.byteBufferChunks.add(cur); if (next.maxUsage > usage) { break; } current = next; next = next.nextList; } } return buf; } } return null; } boolean free(ByteBuffer buffer) { ByteBufferChunk cur = getIndex(buffer); if (cur == null) { LOGGER.info("not in this list!"); return false; } cur.freeByteBuffer(buffer); final int usage = cur.usage(); if (usage < minUsage) { ByteBufferChunkList prev = prevList; ByteBufferChunkList current = this; while (prev != null) { current.byteBufferChunks.remove(cur); prev.byteBufferChunks.add(cur); if (prev.minUsage < usage) { break; } current = prev; prev = prev.prevList; } } return true; } } ================================================ FILE: src/main/java/io/mycat/buffer/ByteBufferPage.java ================================================ package io.mycat.buffer; import java.nio.ByteBuffer; import java.util.BitSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import sun.nio.ch.DirectBuffer; /* * 用来保存一个一个ByteBuffer为底层存储的内存页 */ @SuppressWarnings("restriction") public class ByteBufferPage { private final ByteBuffer buf; private final int chunkSize; //chunk的大小 一般为4k private final int chunkCount; //chunk的个数 private final BitSet chunkAllocateTrack; //某个chunk是否被分配 private final AtomicBoolean allocLockStatus = new AtomicBoolean(false); //锁 private final long startAddress; private final ConcurrentHashMap relationBufferThreadId; public ByteBufferPage(ByteBuffer buf, int chunkSize) { super(); this.chunkSize = chunkSize; chunkCount = buf.capacity() / chunkSize; chunkAllocateTrack = new BitSet(chunkCount); relationBufferThreadId = new ConcurrentHashMap<>(chunkCount); this.buf = buf; startAddress = ((sun.nio.ch.DirectBuffer) buf).address(); } public ByteBuffer allocatChunk(int theChunkCount) { //加锁 成功执行不成功返回 if (!allocLockStatus.compareAndSet(false, true)) { return null; } int startChunk = -1;//从startChunk开始的 int contiueCount = 0;//连续的chunk个数 try { ///枚举寻找连续的N个chunk for (int i = 0; i < chunkCount; i++) { //找到一个可用的 if (chunkAllocateTrack.get(i) == false) { //如果是第一个 则设置startChunk if (startChunk == -1) { //从头开始找 startChunk = i; contiueCount = 1; if (theChunkCount == 1) { break; } } else { //连续chunk个数加一 ,是否找到连续的chunk个数,是则返回. if (++contiueCount == theChunkCount) { break; } } } else { //不连续了 startChunk = -1; contiueCount = 0; } } //找到了 if (contiueCount == theChunkCount) { int offStart = startChunk * chunkSize; int offEnd = offStart + theChunkCount * chunkSize; buf.limit(offEnd); buf.position(offStart); ByteBuffer newBuf = buf.slice(); //分配buffer //sun.nio.ch.DirectBuffer theBuf = (DirectBuffer) newBuf; //System.out.println("offAddress " + (theBuf.address() - startAddress)); //设置chunk为已用 markChunksUsed(startChunk, theChunkCount); relationBufferThreadId.put(((DirectBuffer) newBuf).address(), Thread.currentThread().getId()); return newBuf; } else { //System.out.println("contiueCount " + contiueCount + " theChunkCount " + theChunkCount); return null; } } finally { allocLockStatus.set(false); } } //设置已用 private void markChunksUsed(int startChunk, int theChunkCount) { for (int i = 0; i < theChunkCount; i++) { chunkAllocateTrack.set(startChunk + i); } } //清空不可用 private void markChunksUnused(int startChunk, int theChunkCount) { for (int i = 0; i < theChunkCount; i++) { chunkAllocateTrack.clear(startChunk + i); } } /** * 回收buffer * @param parent 当前要释放的buf的parent * @param recycleBuf 当前要释放的recycleBuf * @param startChunk * @param chunkCount * @param relatedThreadId 用于返回当前要释放的recycleBuf关联的线程id * @return */ public boolean recycleBuffer(ByteBuffer parent, ByteBuffer recycleBuf, int startChunk, int chunkCount, StringBuilder relatedThreadId) { if (parent == this.buf) { //获取锁 必须获取成功 while (!this.allocLockStatus.compareAndSet(false, true)) { Thread.yield(); } //清空已用状态 try { markChunksUnused(startChunk,chunkCount); Long threadId = relationBufferThreadId.remove(((DirectBuffer) recycleBuf).address()); if (threadId != null) { relatedThreadId.append(threadId); } } finally { //释放锁 allocLockStatus.set(false); } return true; } return false; } } ================================================ FILE: src/main/java/io/mycat/buffer/DirectByteBufferPool.java ================================================ package io.mycat.buffer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import sun.nio.ch.DirectBuffer; import java.nio.ByteBuffer; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; /** * DirectByteBuffer池，可以分配任意指定大小的DirectByteBuffer，用完需要归还 * @author wuzhih * @author zagnix */ @SuppressWarnings("restriction") public class DirectByteBufferPool implements BufferPool{ private static final Logger LOGGER = LoggerFactory.getLogger(DirectByteBufferPool.class); public static final String LOCAL_BUF_THREAD_PREX = "$_"; private ByteBufferPage[] allPages; private final int chunkSize; // private int prevAllocatedPage = 0; //private AtomicInteger prevAllocatedPage; private AtomicLong prevAllocatedPage; private final int pageSize; private final short pageCount; private final int conReadBuferChunk ; /** * 记录对线程ID->该线程的所使用Direct Buffer的size */ private final ConcurrentHashMap memoryUsage; public DirectByteBufferPool(int pageSize, short chunkSize, short pageCount,int conReadBuferChunk) { allPages = new ByteBufferPage[pageCount]; this.chunkSize = chunkSize; this.pageSize = pageSize; this.pageCount = pageCount; this.conReadBuferChunk = conReadBuferChunk; //prevAllocatedPage = new AtomicInteger(0); prevAllocatedPage = new AtomicLong(0); for (int i = 0; i < pageCount; i++) { allPages[i] = new ByteBufferPage(ByteBuffer.allocateDirect(pageSize), chunkSize); } memoryUsage = new ConcurrentHashMap<>(); } public BufferArray allocateArray() { return new BufferArray(this); } /** * TODO 当页不够时，考虑扩展内存池的页的数量........... * @param buffer * @return */ public ByteBuffer expandBuffer(ByteBuffer buffer){ int oldCapacity = buffer.capacity(); int newCapacity = oldCapacity << 1; ByteBuffer newBuffer = allocate(newCapacity); if(newBuffer != null){ int newPosition = buffer.position(); buffer.flip(); newBuffer.put(buffer); newBuffer.position(newPosition); recycle(buffer); return newBuffer; } return null; } public ByteBuffer allocate(int size) { final int theChunkCount = size / chunkSize + (size % chunkSize == 0 ? 0 : 1); int selectedPage = (int)(prevAllocatedPage.incrementAndGet() % allPages.length); ByteBuffer byteBuf = allocateBuffer(theChunkCount, 0, selectedPage); if (byteBuf == null) { byteBuf = allocateBuffer(theChunkCount, selectedPage, allPages.length); } final long threadId = Thread.currentThread().getId(); if(byteBuf !=null){ // 这里必须加锁，因为并发情况下如果allocate和recycle函数操作同一个数据，假设它们都先get到数据，然后allocate先put操作， // recycle后进行put操作，这样allocate的put的数据就被覆盖掉 final ByteBuffer finlBuffer = byteBuf; memoryUsage.compute(threadId, (aLong, aLong2) -> { if (aLong2 != null){ return memoryUsage.get(threadId) + finlBuffer.capacity(); }else { return (long)finlBuffer.capacity(); } }); } if(byteBuf==null){ return ByteBuffer.allocate(size); } return byteBuf; } public void recycle(ByteBuffer theBuf) { //堆内buffer直接就清空就好 if(theBuf !=null && (!(theBuf instanceof DirectBuffer) )){ theBuf.clear(); return; } final long size = theBuf.capacity(); boolean recycled = false; StringBuilder relatedThreadId = new StringBuilder(); DirectBuffer thisNavBuf = (DirectBuffer) theBuf;// int chunkCount = theBuf.capacity() / chunkSize; //chunk的个数 DirectBuffer parentBuf = (DirectBuffer) thisNavBuf.attachment(); //page的DirectBuffer int startChunk = (int) ((thisNavBuf.address() - parentBuf.address()) / chunkSize); //开始chunk的序号 for (int i = 0; i < allPages.length; i++) { //在所有的页面中查找当前buffer分配的 if ((recycled = allPages[i].recycleBuffer((ByteBuffer) parentBuf, theBuf, startChunk, chunkCount, relatedThreadId) == true)) { break; } } final Long threadId = relatedThreadId.length() > 0 ? Long.parseLong(relatedThreadId.toString()) : Thread.currentThread().getId(); memoryUsage.computeIfAbsent(threadId, aLong -> (long)(memoryUsage.get(threadId) - size)); if (recycled == false) { LOGGER.warn("warning ,not recycled buffer " + theBuf); } } private ByteBuffer allocateBuffer(int theChunkCount, int startPage, int endPage) { for (int i = startPage; i < endPage; i++) { ByteBuffer buffer = allPages[i].allocatChunk(theChunkCount); if (buffer != null) { prevAllocatedPage.getAndSet(i); return buffer; } } return null; } public int getChunkSize() { return chunkSize; } @Override public ConcurrentHashMap getNetDirectMemoryUsage() { return memoryUsage; } public int getPageSize() { return pageSize; } public short getPageCount() { return pageCount; } public long capacity() { return (long) pageSize * pageCount; } public long size(){ return (long) pageSize * chunkSize * pageCount; } //TODO public int getSharedOptsCount(){ return 0; } public ByteBufferPage[] getAllPages() { return allPages; } public int getConReadBuferChunk() { return conReadBuferChunk; } } ================================================ FILE: src/main/java/io/mycat/buffer/MyCatMemoryAllocator.java ================================================ package io.mycat.buffer; import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufAllocator; import io.netty.buffer.CompositeByteBuf; import io.netty.buffer.PooledByteBufAllocator; import io.netty.util.internal.PlatformDependent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.nio.ByteBuffer; import java.util.concurrent.*; /** * Netty Direct Memory 分配器，为mycat提供内存池管理功能 * * @author zagnix * @create 2017-01-18 11:01 */ public class MyCatMemoryAllocator implements ByteBufAllocator { private static final Logger LOGGER = LoggerFactory.getLogger(MyCatMemoryAllocator.class); public final ConcurrentHashMap recycleMaps = new ConcurrentHashMap<>(); private final static MyCatMemoryAllocator INSTANCE = new MyCatMemoryAllocator(Runtime.getRuntime().availableProcessors()*2); /** netty memory pool alloctor*/ private final PooledByteBufAllocator alloc; /**arena 的数量，一般设置cpu cores*2 */ private final int numberOfArenas; /** ChunkSize 大小 = pageSize << maxOrder */ private final int chunkSize; /**页大小*/ private final int pageSize; /** * numberOfArenas 设置为处理器cores*2 * @param numberOfArenas */ public MyCatMemoryAllocator(int numberOfArenas){ this.numberOfArenas = numberOfArenas; if (!PlatformDependent.hasUnsafe()) { LOGGER.warn("Using direct memory, but sun.misc.Unsafe not available."); } boolean preferDirect = true; this.pageSize = 8192*2; int maxOrder = 11; this.chunkSize = pageSize << maxOrder; int numDirectArenas = numberOfArenas; int numHeapArenas = 0; /** for 4.1.x*/ this.alloc = new PooledByteBufAllocator( preferDirect, numHeapArenas, numDirectArenas, pageSize, maxOrder, 512, 256, 64, true); /**for 5.0.x this.alloc = new PooledByteBufAllocator(preferDirect);**/ } public static MyCatMemoryAllocator getINSTANCE() { return INSTANCE; } /** * @return alloc */ public PooledByteBufAllocator getAlloc() { return alloc; } /** * Returns the number of arenas. * * @return Number of arenas. */ public int getNumberOfArenas() { return numberOfArenas; } /** * Returns the chunk size. * * @return Chunk size. */ public int getChunkSize() { return chunkSize; } /** * page Size * @return page Size */ public int getPageSize() { return pageSize; } @Override public ByteBuf buffer() { return alloc.buffer(); } @Override public ByteBuf buffer(int initialCapacity) { return alloc.buffer(initialCapacity); } @Override public ByteBuf buffer(int initialCapacity, int maxCapacity) { return alloc.buffer(initialCapacity, maxCapacity); } @Override public ByteBuf ioBuffer() { return alloc.ioBuffer(); } @Override public ByteBuf ioBuffer(int initialCapacity) { return alloc.ioBuffer(initialCapacity); } @Override public ByteBuf ioBuffer(int initialCapacity, int maxCapacity) { return alloc.ioBuffer(initialCapacity, maxCapacity); } @Override public ByteBuf heapBuffer() { throw new UnsupportedOperationException("Heap buffer"); } @Override public ByteBuf heapBuffer(int initialCapacity) { throw new UnsupportedOperationException("Heap buffer"); } @Override public ByteBuf heapBuffer(int initialCapacity, int maxCapacity) { throw new UnsupportedOperationException("Heap buffer"); } @Override public ByteBuf directBuffer() { return alloc.directBuffer(); } @Override public ByteBuf directBuffer(int initialCapacity) { return alloc.directBuffer(initialCapacity); } @Override public ByteBuf directBuffer(int initialCapacity, int maxCapacity) { return alloc.directBuffer(initialCapacity, maxCapacity); } @Override public CompositeByteBuf compositeBuffer() { return alloc.compositeBuffer(); } @Override public CompositeByteBuf compositeBuffer(int maxNumComponents) { return alloc.compositeBuffer(maxNumComponents); } @Override public CompositeByteBuf compositeHeapBuffer() { throw new UnsupportedOperationException("Heap buffer"); } @Override public CompositeByteBuf compositeHeapBuffer(int maxNumComponents) { throw new UnsupportedOperationException("Heap buffer"); } @Override public CompositeByteBuf compositeDirectBuffer() { return alloc.compositeDirectBuffer(); } @Override public CompositeByteBuf compositeDirectBuffer(int maxNumComponents) { return alloc.compositeDirectBuffer(maxNumComponents); } @Override public boolean isDirectBufferPooled() { return alloc.isDirectBufferPooled(); } @Override public int calculateNewCapacity(int i, int i1) { return 0; } } ================================================ FILE: src/main/java/io/mycat/buffer/NettyBufferPool.java ================================================ package io.mycat.buffer; import io.netty.buffer.ByteBuf; import io.netty.buffer.PoolArenaMetric; import io.netty.buffer.PoolChunkListMetric; import io.netty.buffer.PoolChunkMetric; import io.netty.util.internal.PlatformDependent; import java.nio.ByteBuffer; import java.util.Iterator; import java.util.List; import java.util.concurrent.ConcurrentHashMap; /** * 封装netty pooled Direct Memory 接口，为mycat提供内存分配功能 * 由于Mycat目前使用ByteBuffer,而Netty分配的是ByteBuf，为了管理ByteBuf * 在MyCatMemoryAllocator中定义recycleMaps ByteBuffer(address) -->ByteBuf * 的映射关系，通过address来回收ByteBuf. * * @author zagnix * @create 2017-04-13 */ public class NettyBufferPool implements BufferPool { MyCatMemoryAllocator allocator; private int chunkSize = 0; public NettyBufferPool(int chunkSize) { allocator = MyCatMemoryAllocator.getINSTANCE(); this.chunkSize = chunkSize; } @Override public ByteBuffer allocate(int size) { ByteBuf byteBuf = allocator.directBuffer(size); ByteBuffer byteBuffer = byteBuf.nioBuffer(0, size); allocator.recycleMaps.put(PlatformDependent.directBufferAddress(byteBuffer), byteBuf); return byteBuffer; } @Override public void recycle(ByteBuffer byteBuffer) { ByteBuf byteBuf = allocator.recycleMaps.get(PlatformDependent.directBufferAddress(byteBuffer)); if (byteBuf != null) { byteBuf.release(); allocator.recycleMaps.remove(PlatformDependent.directBufferAddress(byteBuffer)); } } /** * return memory allocator * * @return */ public MyCatMemoryAllocator getAllocator() { return allocator; } /** * TODO * 下面函数需要将netty相关内存信息导出处理，然后实现 * 计算逻辑就是， * 1.先计算PoolChunk分配的页,表示已经消耗的内存， * 2.然后计算小于一页情况，记录小于一页内存使用情况， * 上面二者合起来就是整个netty 使用的内存， * 已经分配了，但是没有使用的内存的情况 */ @Override public long capacity() { return size(); } @Override public long size() { List list = allocator.getAlloc().directArenas(); long chunkSizeBytes = allocator.getChunkSize(); int chunkCount = 0; synchronized (this) { /**PoolArenas*/ for (PoolArenaMetric pool : list) { List pcks = pool.chunkLists(); /**针对PoolChunkList*/ for (PoolChunkListMetric pck : pcks) { Iterator it = pck.iterator(); while (it.hasNext()) { PoolChunkMetric p = it.next(); chunkCount++; } } } } return chunkCount * chunkSizeBytes; } @Override public int getConReadBuferChunk() { return 0; } @Override public int getSharedOptsCount() { return 0; } @Override public int getChunkSize() { return chunkSize; } @Override public ConcurrentHashMap getNetDirectMemoryUsage() { return null; } @Override public BufferArray allocateArray() { return new BufferArray(this); } } ================================================ FILE: src/main/java/io/mycat/cache/CachePool.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.cache; /** * simple cache pool for implement * * @author wuzhih * */ public interface CachePool { public void putIfAbsent(Object key, Object value); public Object get(Object key); public void clearCache(); public CacheStatic getCacheStatic(); public long getMaxSize(); public void clearCache(String cacheName); } ================================================ FILE: src/main/java/io/mycat/cache/CachePoolFactory.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.cache; /** * factory used to create cachePool * @author wuzhih * */ public abstract class CachePoolFactory { /** * create a cache pool instance * @param poolName * @param cacheSize * @param expireSeconds -1 for not expired * @return */ public abstract CachePool createCachePool(String poolName,int cacheSize,int expireSeconds); } ================================================ FILE: src/main/java/io/mycat/cache/CacheService.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.cache; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * cache service for other component default using memory cache encache * * @author wuzhih * */ public class CacheService { private static final Logger logger = LoggerFactory.getLogger(CacheService.class); private final Map poolFactorys = new HashMap(); private final Map allPools = new HashMap(); public CacheService() { // load cache pool defined try { init(); } catch (Exception e) { if (e instanceof RuntimeException) { throw (RuntimeException) e; } else { throw new RuntimeException(e); } } } public Map getAllCachePools() { return this.allPools; } private void init() throws Exception { Properties props = new Properties(); props.load(CacheService.class .getResourceAsStream("/cacheservice.properties")); final String poolFactoryPref = "factory."; final String poolKeyPref = "pool."; final String layedPoolKeyPref = "layedpool."; String[] keys = props.keySet().toArray(new String[0]); Arrays.sort(keys); for (String key : keys) { if (key.startsWith(poolFactoryPref)) { createPoolFactory(key.substring(poolFactoryPref.length()), (String) props.get(key)); } else if (key.startsWith(poolKeyPref)) { String cacheName = key.substring(poolKeyPref.length()); String value = (String) props.get(key); String[] valueItems = value.split(","); if (valueItems.length < 3) { throw new java.lang.IllegalArgumentException( "invalid cache config ,key:" + key + " value:" + value); } String type = valueItems[0]; int size = Integer.parseInt(valueItems[1]); int timeOut = Integer.parseInt(valueItems[2]); createPool(cacheName, type, size, timeOut); } else if (key.startsWith(layedPoolKeyPref)) { String cacheName = key.substring(layedPoolKeyPref.length()); String value = (String) props.get(key); String[] valueItems = value.split(","); int index = cacheName.indexOf("."); if (index < 0) {// root layer String type = valueItems[0]; int size = Integer.valueOf(valueItems[1]); int timeOut = Integer.valueOf(valueItems[2]); createLayeredPool(cacheName, type, size, timeOut); } else { // root layers' children String parent = cacheName.substring(0, index); String child = cacheName.substring(index + 1); CachePool pool = this.allPools.get(parent); if (pool == null || !(pool instanceof LayerCachePool)) { throw new java.lang.IllegalArgumentException( "parent pool not exists or not layered cache pool:" + parent + " the child cache is:" + child); } int size = Integer.valueOf(valueItems[0]); int timeOut = Integer.valueOf(valueItems[1]); ((DefaultLayedCachePool) pool).createChildCache(child, size, timeOut); } } } } private void createLayeredPool(String cacheName, String type, int size, int expireSeconds) { checkExists(cacheName); logger.info("create layer cache pool " + cacheName + " of type " + type + " ,default cache size " + size + " ,default expire seconds" + expireSeconds); DefaultLayedCachePool layerdPool = new DefaultLayedCachePool(cacheName, this.getCacheFact(type), size, expireSeconds); this.allPools.put(cacheName, layerdPool); } private void checkExists(String poolName) { if (allPools.containsKey(poolName)) { throw new java.lang.IllegalArgumentException( "duplicate cache pool name: " + poolName); } } private void createPoolFactory(String factryType, String factryClassName) throws Exception { CachePoolFactory factry = (CachePoolFactory) Class.forName( factryClassName).newInstance(); poolFactorys.put(factryType, factry); } private void createPool(String poolName, String type, int cacheSize, int expireSeconds) { checkExists(poolName); CachePoolFactory cacheFact = getCacheFact(type); CachePool cachePool = cacheFact.createCachePool(poolName, cacheSize, expireSeconds); allPools.put(poolName, cachePool); } private CachePoolFactory getCacheFact(String type) { CachePoolFactory facty = this.poolFactorys.get(type); if (facty == null) { throw new RuntimeException("CachePoolFactory not defined for type:" + type); } return facty; } /** * get cache pool by name ,caller should cache result * * @param poolName * @return CachePool */ public CachePool getCachePool(String poolName) { CachePool pool = allPools.get(poolName); if (pool == null) { throw new IllegalArgumentException("can't find cache pool:" + poolName); } else { return pool; } } public void clearCache() { logger.info("clear all cache pool "); for (CachePool pool : allPools.values()) { pool.clearCache(); } } } ================================================ FILE: src/main/java/io/mycat/cache/CacheStatic.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.cache; /** * cache static information * * @author wuzhih * */ public class CacheStatic { private long maxSize; private long memorySize; private long itemSize; private long accessTimes; private long putTimes; private long hitTimes; private long lastAccesTime; private long lastPutTime; public long getMemorySize() { return memorySize; } public void setMemorySize(long memorySize) { this.memorySize = memorySize; } public long getItemSize() { return itemSize; } public void setItemSize(long itemSize) { this.itemSize = itemSize; } public long getAccessTimes() { return accessTimes; } public void setAccessTimes(long accessTimes) { this.accessTimes = accessTimes; } public long getHitTimes() { return hitTimes; } public void setHitTimes(long hitTimes) { this.hitTimes = hitTimes; } public long getLastAccesTime() { return lastAccesTime; } public void setLastAccesTime(long lastAccesTime) { this.lastAccesTime = lastAccesTime; } public long getPutTimes() { return putTimes; } public void setPutTimes(long putTimes) { this.putTimes = putTimes; } public void incAccessTimes() { this.accessTimes++; this.lastAccesTime = System.currentTimeMillis(); } public void incHitTimes() { this.hitTimes++; this.accessTimes++; this.lastAccesTime = System.currentTimeMillis(); } public void incPutTimes() { this.putTimes++; this.lastPutTime = System.currentTimeMillis(); } public long getLastPutTime() { return lastPutTime; } public void setLastPutTime(long lastPutTime) { this.lastPutTime = lastPutTime; } public long getMaxSize() { return maxSize; } public void setMaxSize(long maxSize) { this.maxSize = maxSize; } public void reset() { this.accessTimes = 0; this.hitTimes = 0; this.itemSize = 0; this.lastAccesTime = 0; this.lastPutTime = 0; this.memorySize = 0; this.putTimes = 0; } @Override public String toString() { return "CacheStatic [memorySize=" + memorySize + ", itemSize=" + itemSize + ", accessTimes=" + accessTimes + ", putTimes=" + putTimes + ", hitTimes=" + hitTimes + ", lastAccesTime=" + lastAccesTime + ", lastPutTime=" + lastPutTime + "]"; } } ================================================ FILE: src/main/java/io/mycat/cache/DefaultLayedCachePool.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.cache; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class DefaultLayedCachePool implements LayerCachePool { private static final Logger LOGGER = LoggerFactory .getLogger(DefaultLayedCachePool.class); protected Map allCaches = new ConcurrentHashMap(); protected final ReentrantLock lock = new ReentrantLock(); protected int defaultCacheSize; protected int defaulExpiredSeconds; protected static final String defaultCache = "default"; public static final String DEFAULT_CACHE_COUNT = "DEFAULT_CACHE_COUNT"; public static final String DEFAULT_CACHE_EXPIRE_SECONDS = "DEFAULT_CACHE_EXPIRE_SECONDS"; private final CachePoolFactory poolFactory; private final String name; public DefaultLayedCachePool(String name, CachePoolFactory poolFactory, int defaultCacheSize, int defaulExpiredSeconds) { super(); this.name = name; this.poolFactory = poolFactory; this.defaultCacheSize = defaultCacheSize; this.defaulExpiredSeconds = defaulExpiredSeconds; } private CachePool getCache(String cacheName) { CachePool pool = allCaches.get(cacheName); if (pool == null) { lock.lock(); try { pool = allCaches.get(cacheName); if (pool == null) { pool = this.createChildCache(cacheName, this.defaultCacheSize, this.defaulExpiredSeconds); } } finally { lock.unlock(); } } return pool; } /** * create child cache at runtime * * @param cacheName * @return */ public CachePool createChildCache(String cacheName, int size, int expireSeconds) { LOGGER.info("create child Cache: " + cacheName+ " for layered cache "+name+ ", size "+size+", expire seconds "+expireSeconds); CachePool child = this.poolFactory.createCachePool(name + "." + cacheName, size, expireSeconds); allCaches.put(cacheName, child); return child; } @Override public void putIfAbsent(Object key, Object value) { putIfAbsent(defaultCache, key, value); } @Override public Object get(Object key) { return get(defaultCache, key); } @Override public void clearCache() { LOGGER.info("clear cache "); for (CachePool pool : allCaches.values()) { pool.clearCache(); } } @Override public void putIfAbsent(String primaryKey, Object secondKey, Object value) { CachePool pool = getCache(primaryKey); pool.putIfAbsent(secondKey, value); } @Override public Object get(String primaryKey, Object secondKey) { CachePool pool = getCache(primaryKey); return pool.get(secondKey); } @Override public CacheStatic getCacheStatic() { CacheStatic cacheStatic = new CacheStatic(); cacheStatic.setMaxSize(this.getMaxSize()); for (CacheStatic singleStatic : getAllCacheStatic().values()) { cacheStatic.setItemSize(cacheStatic.getItemSize() + singleStatic.getItemSize()); cacheStatic.setHitTimes(cacheStatic.getHitTimes() + singleStatic.getHitTimes()); cacheStatic.setAccessTimes(cacheStatic.getAccessTimes() + singleStatic.getAccessTimes()); cacheStatic.setPutTimes(cacheStatic.getPutTimes() + singleStatic.getPutTimes()); if (cacheStatic.getLastAccesTime() < singleStatic .getLastAccesTime()) { cacheStatic.setLastAccesTime(singleStatic.getLastAccesTime()); } if (cacheStatic.getLastPutTime() < singleStatic.getLastPutTime()) { cacheStatic.setLastPutTime(singleStatic.getLastPutTime()); } } return cacheStatic; } @Override public Map getAllCacheStatic() { Map results = new HashMap( this.allCaches.size()); for (Map.Entry entry : allCaches.entrySet()) { results.put(entry.getKey(), entry.getValue().getCacheStatic()); } return results; } @Override public long getMaxSize() { long maxSize=0; for(CachePool cache:this.allCaches.values()) { maxSize+=cache.getMaxSize(); } return maxSize; } //clear cache by cacheName public void clearCache(String cacheName) { LOGGER.info("clear cache :"+cacheName); CachePool pool = getCache(cacheName); pool.clearCache(); } } ================================================ FILE: src/main/java/io/mycat/cache/LayerCachePool.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.cache; import java.util.Map; /** * Layered cache pool * * @author wuzhih * */ public interface LayerCachePool extends CachePool { public void putIfAbsent(String primaryKey, Object secondKey, Object value); public Object get(String primaryKey, Object secondKey); /** * get all cache static, name is cache name * @return map of CacheStatic */ public Map getAllCacheStatic(); public void clearCache(String cacheName); } ================================================ FILE: src/main/java/io/mycat/cache/MysqlDataSetCache.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.cache; import java.io.FileOutputStream; import java.io.IOException; import java.io.Serializable; /** * cache mysql dataset ,for example "select * from A where .......",cache all * result * * @author wuzhih * */ public class MysqlDataSetCache implements Serializable { /** * */ private static final long serialVersionUID = 5426632041410472392L; // sql should not inlude page limit ,should store first record and sequnce // next private String sql; private int total; private String dataFile; private long createTime; private volatile int curCount; private volatile long lastAccesTime; private volatile boolean storing = true; public String getSql() { return sql; } public boolean isStoring() { return storing; } public void setStoring(boolean storing) { this.storing = storing; } public void setSql(String sql) { this.sql = sql; } public int getTotal() { return total; } public void setTotal(int total) { this.total = total; } public String getDataFile() { return dataFile; } public void setDataFile(String dataFile) { this.dataFile = dataFile; } public long getCreateTime() { return createTime; } public void setCreateTime(long createTime) { this.createTime = createTime; } public long getLastAccesTime() { return lastAccesTime; } public void setLastAccesTime(long lastAccesTime) { this.lastAccesTime = lastAccesTime; } public void addHeader(byte[] header) throws IOException { writeFile(header); } private void writeFile(byte[] data) throws IOException { FileOutputStream outf = null; try { outf = new FileOutputStream(dataFile, true); outf.write(data); } finally { if (outf != null) { outf.close(); } } } public void appendRecord(byte[] row) throws IOException { writeFile(row); curCount++; } } ================================================ FILE: src/main/java/io/mycat/cache/MysqlDataSetService.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.cache; import java.io.File; import java.util.HashSet; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; public class MysqlDataSetService { private volatile boolean enabled = false; // max expire time is 300 seconds private int maxExpire = 300; private final ConcurrentHashMap cachedMap = new ConcurrentHashMap(); private volatile Set needCachedSQL = new HashSet(); public boolean isEnabled() { return enabled; } public void setEnabled(boolean enabled) { this.enabled = enabled; } public int getMaxExpire() { return maxExpire; } public void setMaxExpire(int maxExpire) { this.maxExpire = maxExpire; } private static MysqlDataSetService instance = new MysqlDataSetService(); public static MysqlDataSetService getInstance() { return instance; } private MysqlDataSetService() { } /** * sql should not include LIMIT range * * @param sql * @return */ public MysqlDataSetCache findDataSetCache(String sql) { if (!enabled) { return null; } MysqlDataSetCache cache = cachedMap.get(sql); if (validCache(cache)) { return cache; } else { cachedMap.remove(sql); } return null; } public String needCache(String sql) { return needCachedSQL.contains(sql)?sql:null; } public boolean addIfNotExists(MysqlDataSetCache newCache) { return (cachedMap.putIfAbsent(newCache.getSql(), newCache) == null); } private boolean validCache(MysqlDataSetCache cache) { return (!cache.isStoring() && (cache.getCreateTime() + this.maxExpire * 1000 < System .currentTimeMillis()) && (new File(cache.getDataFile()) .exists())); } } ================================================ FILE: src/main/java/io/mycat/cache/impl/EnchachePooFactory.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.cache.impl; import io.mycat.cache.CachePool; import io.mycat.cache.CachePoolFactory; import net.sf.ehcache.Cache; import net.sf.ehcache.CacheManager; import net.sf.ehcache.config.CacheConfiguration; public class EnchachePooFactory extends CachePoolFactory { @Override public CachePool createCachePool(String poolName, int cacheSize, int expiredSeconds) { CacheManager cacheManager = CacheManager.create(); Cache enCache = cacheManager.getCache(poolName); if (enCache == null) { CacheConfiguration cacheConf = cacheManager.getConfiguration() .getDefaultCacheConfiguration().clone(); cacheConf.setName(poolName); if (cacheConf.getMaxEntriesLocalHeap() != 0) { cacheConf.setMaxEntriesLocalHeap(cacheSize); } else { cacheConf.setMaxBytesLocalHeap(String.valueOf(cacheSize)); } cacheConf.setTimeToIdleSeconds(expiredSeconds); Cache cache = new Cache(cacheConf); cacheManager.addCache(cache); return new EnchachePool(poolName,cache,cacheSize); } else { return new EnchachePool(poolName,enCache,cacheSize); } } } ================================================ FILE: src/main/java/io/mycat/cache/impl/EnchachePool.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.cache.impl; import net.sf.ehcache.Cache; import net.sf.ehcache.Element; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.cache.CachePool; import io.mycat.cache.CacheStatic; /** * ehcache based cache pool * * @author wuzhih * */ public class EnchachePool implements CachePool { private static final Logger LOGGER = LoggerFactory.getLogger(EnchachePool.class); private final Cache enCache; private final CacheStatic cacheStati = new CacheStatic(); private final String name; private final long maxSize; public EnchachePool(String name,Cache enCache,long maxSize) { this.enCache = enCache; this.name=name; this.maxSize=maxSize; cacheStati.setMaxSize(this.getMaxSize()); } @Override public void putIfAbsent(Object key, Object value) { Element el = new Element(key, value); if (enCache.putIfAbsent(el) == null) { cacheStati.incPutTimes(); if (LOGGER.isDebugEnabled()) { LOGGER.debug(name+" add cache ,key:" + key + " value:" + value); } } } @Override public Object get(Object key) { Element cacheEl = enCache.get(key); if (cacheEl != null) { if (LOGGER.isDebugEnabled()) { LOGGER.debug(name+" hit cache ,key:" + key); } cacheStati.incHitTimes(); return cacheEl.getObjectValue(); } else { if (LOGGER.isDebugEnabled()) { LOGGER.debug(name+" miss cache ,key:" + key); } cacheStati.incAccessTimes(); return null; } } @Override public void clearCache() { LOGGER.info("clear cache "+name); enCache.removeAll(); enCache.clearStatistics(); cacheStati.reset(); cacheStati.setMemorySize(enCache.getMemoryStoreSize()); } @Override public CacheStatic getCacheStatic() { cacheStati.setItemSize(enCache.getSize()); return cacheStati; } @Override public long getMaxSize() { return maxSize; } @Override public void clearCache(String cacheName) { if (cacheName != null){ enCache.remove(cacheName); } } } ================================================ FILE: src/main/java/io/mycat/cache/impl/LevelDBCachePooFactory.java ================================================ package io.mycat.cache.impl; import java.io.File; import static org.iq80.leveldb.impl.Iq80DBFactory.factory; import org.iq80.leveldb.DB; import org.iq80.leveldb.Options; import io.mycat.cache.CachePool; import io.mycat.cache.CachePoolFactory; public class LevelDBCachePooFactory extends CachePoolFactory { @Override public CachePool createCachePool(String poolName, int cacheSize, int expireSeconds) { Options options = new Options(); options.cacheSize(cacheSize * 1048576);//cacheSize M 大小 options.createIfMissing(true); DB db =null; try { db=factory.open(new File("leveldb\\"+poolName), options); // Use the db in here.... } catch (Exception e) { // Make sure you close the db to shutdown the // database and avoid resource leaks. // db.close(); } return new LevelDBPool(poolName,db,cacheSize); } } ================================================ FILE: src/main/java/io/mycat/cache/impl/LevelDBPool.java ================================================ package io.mycat.cache.impl; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.iq80.leveldb.DB; import io.mycat.cache.CachePool; import io.mycat.cache.CacheStatic; public class LevelDBPool implements CachePool { private static final Logger LOGGER = LoggerFactory.getLogger(LevelDBPool.class); private final DB cache; private final CacheStatic cacheStati = new CacheStatic(); private final String name; private final long maxSize; public LevelDBPool(String name,DB db,long maxSize) { this.cache = db; this.name=name; this.maxSize=maxSize; cacheStati.setMaxSize(maxSize); } @Override public void putIfAbsent(Object key, Object value) { cache.put(toByteArray(key),toByteArray(value)); cacheStati.incPutTimes(); if (LOGGER.isDebugEnabled()) { LOGGER.debug(name+" add leveldb cache ,key:" + key + " value:" + value); } } @Override public Object get(Object key) { Object ob= toObject(cache.get(toByteArray(key))); if (ob != null) { if (LOGGER.isDebugEnabled()) { LOGGER.debug(name+" hit cache ,key:" + key); } cacheStati.incHitTimes(); return ob; } else { if (LOGGER.isDebugEnabled()) { LOGGER.debug(name+" miss cache ,key:" + key); } cacheStati.incAccessTimes(); return null; } } @Override public void clearCache() { LOGGER.info("clear cache "+name); //cache.delete(key); cacheStati.reset(); //cacheStati.setMemorySize(cache.g); } @Override public CacheStatic getCacheStatic() { /* int i=0; try { // DBIterator iterator = cache.iterator(); for(cache.iterator().seekToFirst(); cache.iterator().hasNext(); cache.iterator().next()) { i++; } cache.iterator().close(); } catch (Exception e) { // Make sure you close the iterator to avoid resource leaks. } //long[] sizes = cache.getApproximateSizes(new Range(bytes("TESTDB"), bytes("TESTDC"))); */ //cacheStati.setItemSize(cache.getSize());//sizes[0]);//需要修改leveldb的代码 cacheStati.setItemSize(cacheStati.getPutTimes()); return cacheStati; } @Override public long getMaxSize() { return maxSize; } @Override public void clearCache(String cacheName) { if (cacheName != null){ cache.delete(toByteArray(cacheName)); } } public byte[] toByteArray (Object obj) { byte[] bytes = null; ByteArrayOutputStream bos = new ByteArrayOutputStream(); try { ObjectOutputStream oos = new ObjectOutputStream(bos); oos.writeObject(obj); oos.flush(); bytes = bos.toByteArray (); oos.close(); bos.close(); } catch (IOException ex) { LOGGER.error("toByteArrayError", ex); } return bytes; } public Object toObject (byte[] bytes) { Object obj = null; if ((bytes==null) || (bytes.length<=0)) { return obj; } try { ByteArrayInputStream bis = new ByteArrayInputStream (bytes); ObjectInputStream ois = new ObjectInputStream (bis); obj = ois.readObject(); ois.close(); bis.close(); } catch (IOException ex) { LOGGER.error("toObjectError", ex); } catch (ClassNotFoundException ex) { LOGGER.error("toObjectError", ex); } return obj; } } ================================================ FILE: src/main/java/io/mycat/cache/impl/MapDBCachePooFactory.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.cache.impl; import java.util.concurrent.TimeUnit; import org.mapdb.DB; import org.mapdb.DBMaker; import org.mapdb.HTreeMap; import io.mycat.cache.CachePool; import io.mycat.cache.CachePoolFactory; public class MapDBCachePooFactory extends CachePoolFactory { private DB db = DBMaker.newMemoryDirectDB().cacheSize(1000).cacheLRUEnable().make(); @Override public CachePool createCachePool(String poolName, int cacheSize, int expiredSeconds) { HTreeMap cache = this.db.createHashMap(poolName) .expireMaxSize(cacheSize) .expireAfterAccess(expiredSeconds, TimeUnit.SECONDS) .makeOrGet(); return new MapDBCachePool(cache, cacheSize); } } ================================================ FILE: src/main/java/io/mycat/cache/impl/MapDBCachePool.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.cache.impl; import org.mapdb.HTreeMap; import io.mycat.cache.CachePool; import io.mycat.cache.CacheStatic; public class MapDBCachePool implements CachePool { private final HTreeMap htreeMap; private final CacheStatic cacheStati = new CacheStatic(); private final long maxSize; public MapDBCachePool(HTreeMap htreeMap,long maxSize) { this.htreeMap = htreeMap; this.maxSize=maxSize; cacheStati.setMaxSize(maxSize); } @Override public void putIfAbsent(Object key, Object value) { if (htreeMap.putIfAbsent(key, value) == null) { cacheStati.incPutTimes(); } } @Override public Object get(Object key) { Object value = htreeMap.get(key); if (value != null) { cacheStati.incHitTimes(); return value; } else { cacheStati.incAccessTimes(); return null; } } @Override public void clearCache() { htreeMap.clear(); cacheStati.reset(); } @Override public CacheStatic getCacheStatic() { cacheStati.setItemSize(htreeMap.sizeLong()); return cacheStati; } @Override public long getMaxSize() { return maxSize; } @Override public void clearCache(String cacheName) { htreeMap.remove(cacheName); } } ================================================ FILE: src/main/java/io/mycat/cache/index/Shard.java ================================================ package io.mycat.cache.index; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; import java.util.List; import java.util.SortedMap; import java.util.TreeMap; /** * 分布式索引一致性 * * @author Hash Zhang * @version 1.0 * @time 00:15:03 2016/5/23 */ public class Shard { // S类封装了机器节点的信息 ，如name、password、ip、port等 private TreeMap nodes; // 虚拟节点 private List shards; // 真实机器节点 private static final int NODE_NUM = 100; // 每个机器节点关联的虚拟节点个数 public Shard(List shards) { super(); this.shards = shards; init(); } private void init() { // 初始化一致性hash环 nodes = new TreeMap(); for (int i = 0; i != shards.size(); ++i) { // 每个真实机器节点都需要关联虚拟节点 final S shardInfo = shards.get(i); for (int n = 0; n < NODE_NUM; n++) { // 一个真实机器节点关联NODE_NUM个虚拟节点 nodes.put(hash("SHARD-" + i + "-NODE-" + n), shardInfo); } } } public S getShardInfo(String key) { SortedMap tail = nodes.tailMap(hash(key)); // 沿环的顺时针找到一个虚拟节点 if (tail.size() == 0) { return nodes.get(nodes.firstKey()); } return tail.get(tail.firstKey()); // 返回该虚拟节点对应的真实机器节点的信息 } /** * MurMurHash算法，是非加密HASH算法，性能很高， * 比传统的CRC32,MD5，SHA-1（这两个算法都是加密HASH算法，复杂度本身就很高，带来的性能上的损害也不可避免） * 等HASH算法要快很多，而且据说这个算法的碰撞率很低. * http://murmurhash.googlepages.com/ */ private Long hash(String key) { ByteBuffer buf = ByteBuffer.wrap(key.getBytes()); int seed = 0x1234ABCD; ByteOrder byteOrder = buf.order(); buf.order(ByteOrder.LITTLE_ENDIAN); long m = 0xc6a4a7935bd1e995L; int r = 47; long h = seed ^ (buf.remaining() * m); long k; while (buf.remaining() >= 8) { k = buf.getLong(); k *= m; k ^= k >>> r; k *= m; h ^= k; h *= m; } if (buf.remaining() > 0) { ByteBuffer finish = ByteBuffer.allocate(8).order( ByteOrder.LITTLE_ENDIAN); // for big-endian version, do this first: // finish.position(8-buf.remaining()); finish.put(buf).rewind(); h ^= finish.getLong(); h *= m; } h ^= h >>> r; h *= m; h ^= h >>> r; buf.order(byteOrder); return h; } public static void main(String[] args) { List stringList = new ArrayList<>(); stringList.add("host1"); stringList.add("host2"); stringList.add("host3"); stringList.add("host4"); stringList.add("host5"); Shard stringShard = new Shard<>(stringList); for (int i = 0; i < 10; i++) { System.out.println(i+":"+stringShard.getShardInfo(""+i)); } stringList = new ArrayList<>(); stringList.add("host1"); stringList.add("host2"); stringList.add("host3"); stringList.add("host4"); stringShard = new Shard<>(stringList); for (int i = 0; i < 10; i++) { System.out.println(i+":"+stringShard.getShardInfo(""+i)); } } } ================================================ FILE: src/main/java/io/mycat/catlets/Catlet.java ================================================ package io.mycat.catlets; import io.mycat.cache.LayerCachePool; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.server.ServerConnection; import io.mycat.sqlengine.EngineCtx; /** * mycat catlet ,used to execute sql and return result to client,some like * database's procedure. * must implemented as a stateless class and can process many SQL concurrently * * @author wuzhih * */ public interface Catlet { /* * execute sql in EngineCtx and return result to client */ void processSQL(String sql, EngineCtx ctx); void route(SystemConfig sysConfig, SchemaConfig schema, int sqlType, String realSQL, String charset, ServerConnection sc, LayerCachePool cachePool) ; //void setRoute(RouteResultset rrs); } ================================================ FILE: src/main/java/io/mycat/catlets/JoinParser.java ================================================ package io.mycat.catlets; import java.util.LinkedHashMap; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLOrderBy; import com.alibaba.druid.sql.ast.SQLOrderingSpecification; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.ast.expr.SQLAggregateExpr; import com.alibaba.druid.sql.ast.expr.SQLAllColumnExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator; import com.alibaba.druid.sql.ast.expr.SQLBooleanExpr; import com.alibaba.druid.sql.ast.expr.SQLCharExpr; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.alibaba.druid.sql.ast.expr.SQLInListExpr; import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr; import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr; import com.alibaba.druid.sql.ast.expr.SQLNullExpr; import com.alibaba.druid.sql.ast.expr.SQLNumberExpr; import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource; import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource.JoinType; import com.alibaba.druid.sql.ast.statement.SQLSelectItem; import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem; import com.alibaba.druid.sql.ast.statement.SQLSelectQuery; import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; import com.alibaba.druid.sql.ast.statement.SQLTableSource; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; /** * 功能详细描述:分片join,解析join语句 * @author sohudo[http://blog.csdn.net/wind520] * @create 2015年01月25日 * @version 0.0.1 */ public class JoinParser { protected static final Logger LOGGER = LoggerFactory.getLogger(JoinParser.class); private MySqlSelectQueryBlock mysqlQuery; private String stmt=""; private String joinType; private String masterTable; private TableFilter tableFilter; // a table -> b table 的链表 //private LinkedHashMap fieldAliasMap = new LinkedHashMap(); public JoinParser(MySqlSelectQueryBlock selectQuery,String stmt) { this.mysqlQuery=selectQuery; this.stmt=stmt; } public void parser(){ masterTable=""; SQLTableSource table=mysqlQuery.getFrom(); //a 表 parserTable(table,tableFilter,false); // 组成链表 parserFields(mysqlQuery.getSelectList()); //查询字段放到各个查询表中。 parserMasterTable(); //查询主表 别名 parserWhere(mysqlQuery.getWhere(),""); // where 条件放到各个查询表中。 // getJoinField(); parserOrderBy(mysqlQuery.getOrderBy()); // order 条件放到各个查询表中。 parserLimit(); // limit // LOGGER.info("field "+fieldAliasMap); // LOGGER.info("master "+masterTable); // LOGGER.info("join Lkey "+getJoinLkey()); // LOGGER.info("join Rkey "+getJoinRkey()); LOGGER.info("SQL: "+this.stmt); } private void parserTable(SQLTableSource table,TableFilter tFilter,boolean isOutJoin){ if(table instanceof SQLJoinTableSource){ SQLJoinTableSource table1=(SQLJoinTableSource)table; joinType=table1.getJoinType().toString(); if ((table1.getJoinType()==JoinType.COMMA)||(table1.getJoinType()==JoinType.JOIN)||(table1.getJoinType()==JoinType.INNER_JOIN) ||(table1.getJoinType()==JoinType.LEFT_OUTER_JOIN)) { tFilter=setTableFilter(tFilter,getTableFilter(table1.getLeft(),isOutJoin)); if (tableFilter==null){ tableFilter=tFilter; } } //parserTable(table1.getLeft()); //SQLExprTableSource parserTable(table1.getRight(),tFilter,true); SQLExpr expr=table1.getCondition();//SQLBinaryOpExpr parserJoinKey(expr); } else { tFilter=setTableFilter(tFilter,getTableFilter(table,isOutJoin)); LOGGER.info("table "+table.toString() +" Alias:"+table.getAlias()+" Hints:"+table.getHints()); } } private TableFilter setTableFilter(TableFilter tFilter,TableFilter newFilter){ if (tFilter==null) { tFilter=newFilter; return tFilter; } else { tFilter.setTableJoin(newFilter); return tFilter.getTableJoin(); } } private TableFilter getTableFilter(SQLTableSource table,boolean isOutJoin){ String key ; String value = table.toString().trim(); if (table.getAlias()==null) { key=value; } else { key = table.getAlias().trim(); } return new TableFilter(value,key,isOutJoin); } private void parserJoinKey(SQLExpr expr){ if (expr==null) { return; } parserWhere(expr,""); } private String getExprFieldName(SQLAggregateExpr expr){ StringBuilder field = new StringBuilder(); for (SQLExpr item :expr.getArguments()){ field.append(item.toString()); } return expr.getMethodName()+"("+field.toString()+")"; } private String getFieldName(SQLSelectItem item){ if (item.getExpr() instanceof SQLPropertyExpr) { return item.getExpr().toString();//字段别名 } else { return item.toString(); } } private String getMethodInvokeFieldName(SQLSelectItem item){ SQLMethodInvokeExpr invoke = (SQLMethodInvokeExpr)item.getExpr(); List itemExprs = invoke.getParameters(); for(SQLExpr itemExpr:itemExprs){ if (itemExpr instanceof SQLPropertyExpr) { return itemExpr.toString();//字段别名 } } return item.toString(); } private void parserFields(List mysqlSelectList){ //显示的字段 String key=""; String value =""; String exprfield = ""; for(SQLSelectItem item : mysqlSelectList) { if (item.getExpr() instanceof SQLAllColumnExpr) { //*解析 setField(item.toString(), item.toString()); } else { if (item.getExpr() instanceof SQLAggregateExpr) { SQLAggregateExpr expr =(SQLAggregateExpr)item.getExpr(); key = getExprFieldName(expr); setField(key, value); }else if(item.getExpr() instanceof SQLMethodInvokeExpr){ key = getMethodInvokeFieldName(item); exprfield=getFieldName(item); // value=item.getAlias(); setField(key, value,exprfield); }else { key=getFieldName(item); value=item.getAlias(); setField(key, value); } } } } private void setField(String key,String value){ //fieldAliasMap.put(key, value); if (tableFilter!=null){ tableFilter.addField(key, value); } } private void setField(String key,String value,String expr){ //fieldAliasMap.put(key, value); if (tableFilter!=null){ tableFilter.addField(key, value,expr); } } //判断并获得主表 private void parserMasterTable(){ if (tableFilter!=null){ masterTable=tableFilter.getTableAlia(); } } private boolean checkJoinField(String value){ if (value==null){ return false; } else { int i=value.indexOf('.'); return i>0; } } //解析 a.field = b.field private void parserWhere(SQLExpr aexpr,String Operator){ if (aexpr==null) { return; } if (aexpr instanceof SQLBinaryOpExpr){ SQLBinaryOpExpr expr=(SQLBinaryOpExpr)aexpr; SQLExpr exprL=expr.getLeft(); if (!(exprL instanceof SQLBinaryOpExpr)) { opSQLExpr((SQLBinaryOpExpr)aexpr,Operator); } else { // if (expr.getOperator().getName().equals("AND")) { if (expr.getOperator()==SQLBinaryOperator.BooleanAnd) { //parserWhere(exprL); //parserWhere(expr.getRight()); andorWhere(exprL,expr.getOperator().getName(),expr.getRight()); } else if (expr.getOperator()==SQLBinaryOperator.BooleanOr){//.getName().equals("OR")) { andorWhere(exprL,expr.getOperator().getName(),expr.getRight()); } else { throw new RuntimeException("Can't identify the operation of of where"); } } }else if(aexpr instanceof SQLInListExpr){ SQLInListExpr expr = (SQLInListExpr)aexpr; SQLExpr exprL = expr.getExpr(); String field=exprL.toString(); tableFilter.addWhere(field, SQLUtils.toMySqlString(expr), Operator); } } private void andorWhere(SQLExpr exprL,String Operator,SQLExpr exprR ){ parserWhere(exprL,""); parserWhere(exprR,Operator); } private void opSQLExpr(SQLBinaryOpExpr expr,String Operator) { if (expr==null) { return; } SQLExpr exprL=expr.getLeft(); if (!(exprL instanceof SQLBinaryOpExpr)) { String field=exprL.toString(); //获取表达式 左边的值 String value=getExpValue(expr.getRight()).toString(); //获取表达式右边的值 if (expr.getOperator()==SQLBinaryOperator.Equality) { if (checkJoinField(value)) {//设置joinKey //joinLkey=field; //joinRkey=value; tableFilter.setJoinKey(field,value); } else { tableFilter.addWhere(field, value, expr.getOperator().getName(), Operator); } } else { tableFilter.addWhere(field, value, expr.getOperator().getName(), Operator); } } } private Object getExpValue(SQLExpr expr){ if (expr instanceof SQLIntegerExpr){ return ((SQLIntegerExpr)expr).getNumber().longValue(); } if (expr instanceof SQLNumberExpr){ return ((SQLNumberExpr)expr).getNumber().doubleValue(); } if (expr instanceof SQLCharExpr){ String va=((SQLCharExpr)expr).toString(); return va;//remove(va,'\''); } if (expr instanceof SQLBooleanExpr){ return ((SQLBooleanExpr)expr).getValue(); } if (expr instanceof SQLNullExpr){ return null; } return expr; } private void parserOrderBy(SQLOrderBy orderby) { if (orderby != null ){ for (int i = 0; i < orderby.getItems().size(); i++) { SQLSelectOrderByItem orderitem = orderby.getItems().get(i); tableFilter.addOrders(i, orderitem.getExpr().toString(), getSQLExprToAsc(orderitem.getType())); } } } private void parserLimit(){ int limitoff=0; int limitnum=0; if (this.mysqlQuery.getLimit()!=null) { limitoff=getSQLExprToInt(this.mysqlQuery.getLimit().getOffset()); limitnum=getSQLExprToInt(this.mysqlQuery.getLimit().getRowCount()); tableFilter.addLimit(limitoff,limitnum); } } private int getSQLExprToInt(SQLExpr expr){ if (expr instanceof SQLIntegerExpr){ return ((SQLIntegerExpr)expr).getNumber().intValue(); } return 0; } private String getSQLExprToAsc(SQLOrderingSpecification ASC){ if (ASC==null ) { return " ASC "; } if (ASC==SQLOrderingSpecification.DESC){ return " DESC "; } else { return " ASC "; } } public String getChildSQL(){ //String sql="select "+joinRkey+","+sql+" from "+mtable+" where "+joinRkey+" in "; String sql=tableFilter.getTableJoin().getSQL(); return sql; } public String getSql(){ stmt=tableFilter.getSQL(); return stmt; } public String getJoinType(){ return joinType; } public String getJoinLkey(){ return tableFilter.getJoinKey(true); } public String getJoinRkey(){ return tableFilter.getJoinKey(false); } //返回a表排序的字段 public LinkedHashMap getOrderByCols(){ return tableFilter.getOrderByCols(); } //返回b表排序的字段 public LinkedHashMap getChildByCols(){ return tableFilter.getTableJoin().getOrderByCols(); } //是否有order 排序 public boolean hasOrder() { return tableFilter.getOrderByCols().size() > 0 || tableFilter.getTableJoin().getOrderByCols().size() > 0; } /* * limit 的 start*/ public int getOffset() { return tableFilter.getOffset(); } /* * limit 的 rowCount*/ public int getRowCount() { return tableFilter.getRowCount(); } /* * 是否有limit 输出。 */ public boolean hasLimit() { return tableFilter.getOffset() > 0 || tableFilter.getRowCount() > 0 ; } } ================================================ FILE: src/main/java/io/mycat/catlets/ShareJoin.java ================================================ package io.mycat.catlets; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.ast.statement.SQLSelectQuery; import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; import io.mycat.backend.mysql.nio.handler.MiddlerQueryResultHandler; import io.mycat.backend.mysql.nio.handler.MiddlerResultHandler; import io.mycat.cache.LayerCachePool; import io.mycat.config.ErrorCode; import io.mycat.config.Fields; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.route.RouteResultset; import io.mycat.route.RouteResultsetNode; import io.mycat.route.factory.RouteStrategyFactory; import io.mycat.server.NonBlockingSession; import io.mycat.server.ServerConnection; import io.mycat.server.parser.ServerParse; import io.mycat.sqlengine.AllJobFinishedListener; import io.mycat.sqlengine.EngineCtx; import io.mycat.sqlengine.SQLJobHandler; import io.mycat.sqlengine.mpp.ColMeta; import io.mycat.sqlengine.mpp.OrderCol; import io.mycat.sqlengine.mpp.tmp.RowDataSorter; import io.mycat.util.ByteUtil; import io.mycat.util.ResultSetUtil; /** * 功能详细描述:分片join * @author sohudo[http://blog.csdn.net/wind520] * @create 2015年01月22日 下午6:50:23 * @version 0.0.1 */ public class ShareJoin implements Catlet { private EngineCtx ctx; private RouteResultset rrs ; private JoinParser joinParser; private Map rows = new ConcurrentHashMap(); private Map ids = new ConcurrentHashMap(); //private ConcurrentLinkedQueue ids = new ConcurrentLinkedQueue(); private List fields; //主表的字段 private ArrayList allfields;//所有的字段 private boolean isMfield=false; private int mjob=0; private int maxjob=0; private int joinindex=0;//关联join表字段的位置 private int sendField=0; private boolean childRoute=false; private boolean jointTableIsData=false; // join 字段的类型，一般情况都是int, long; 增加该字段为了支持非int,long类型的(一般为varchar)joinkey的sharejoin // 参见：io.mycat.server.packet.FieldPacket 属性： public int type; // 参见：http://dev.mysql.com/doc/internals/en/com-query-response.html#packet-Protocol::ColumnDefinition private int joinKeyType = Fields.FIELD_TYPE_LONG; // 默认 join 字段为int型 //重新路由使用 private SystemConfig sysConfig; private SchemaConfig schema; private int sqltype; private String charset; private ServerConnection sc; private LayerCachePool cachePool; private RowDataSorter sorter; //排序器。 private volatile boolean isInit = false; public void setRoute(RouteResultset rrs){ this.rrs =rrs; } public void route(SystemConfig sysConfig, SchemaConfig schema,int sqlType, String realSQL, String charset, ServerConnection sc, LayerCachePool cachePool) { int rs = ServerParse.parse(realSQL); this.sqltype = rs & 0xff; this.sysConfig=sysConfig; this.schema=schema; this.charset=charset; this.sc=sc; this.cachePool=cachePool; try { // RouteStrategy routes=RouteStrategyFactory.getRouteStrategy(); // rrs =RouteStrategyFactory.getRouteStrategy().route(sysConfig, schema, sqlType2, realSQL,charset, sc, cachePool); MySqlStatementParser parser = new MySqlStatementParser(realSQL); SQLStatement statement = parser.parseStatement(); if(statement instanceof SQLSelectStatement) { SQLSelectStatement st=(SQLSelectStatement)statement; SQLSelectQuery sqlSelectQuery =st.getSelect().getQuery(); if(sqlSelectQuery instanceof MySqlSelectQueryBlock) { MySqlSelectQueryBlock mysqlSelectQuery = (MySqlSelectQueryBlock)sqlSelectQuery; joinParser=new JoinParser(mysqlSelectQuery,realSQL); joinParser.parser(); } } /* if (routes instanceof DruidMysqlRouteStrategy) { SQLSelectStatement st=((DruidMysqlRouteStrategy) routes).getSQLStatement(); SQLSelectQuery sqlSelectQuery =st.getSelect().getQuery(); if(sqlSelectQuery instanceof MySqlSelectQueryBlock) { MySqlSelectQueryBlock mysqlSelectQuery = (MySqlSelectQueryBlock)st.getSelect().getQuery(); joinParser=new JoinParser(mysqlSelectQuery,realSQL); joinParser.parser(); } } */ } catch (Exception e) { } } private void getRoute(String sql){ try { if (joinParser!=null){ rrs =RouteStrategyFactory.getRouteStrategy().route(sysConfig, schema, sqltype,sql,charset, sc, cachePool); } } catch (Exception e) { } } private String[] getDataNodes(){ String[] dataNodes =new String[rrs.getNodes().length] ; for (int i=0;i 一行数据 ids.put(id, nid); // 主键 -> joinkey value joinindex=findex; //ids.offer(nid); int batchSize = 999; // 满1000条，发送一个查询请求 if (ids.size() > batchSize) { createQryJob(batchSize); } } public void putDBFields(List mFields){ if (!isMfield){ fields=mFields; } } //发送最后的主表查询语句 public void endJobInput(String dataNode, boolean failed){ mjob++; //结束任务+1 if (mjob>=maxjob){ //发送最后一个右边表的查询语句。 createQryJob(Integer.MAX_VALUE); ctx.endJobInput(); } // EngineCtx.LOGGER.info("完成"+mjob+":" + dataNode+" failed:"+failed); } //private void createQryJob(String dataNode,int batchSize) { private void createQryJob(int batchSize) { int count = 0; Map batchRows = new ConcurrentHashMap(); String theId = null; StringBuilder sb = new StringBuilder().append('('); String svalue=""; for(Map.Entry e: ids.entrySet() ){ theId=e.getKey(); byte[] rowbyte = rows.remove(theId); if(rowbyte!=null){ batchRows.put(theId, rowbyte); } if (!svalue.equals(e.getValue())){ if(joinKeyType == Fields.FIELD_TYPE_VAR_STRING || joinKeyType == Fields.FIELD_TYPE_STRING){ // joinkey 为varchar sb.append("'").append(e.getValue()).append("'").append(','); // ('digdeep','yuanfang') }else{ // 默认joinkey为int/long sb.append(e.getValue()).append(','); // (1,2,3) } } svalue=e.getValue(); if (count++ > batchSize) { break; } } /* while ((theId = ids.poll()) != null) { batchRows.put(theId, rows.remove(theId)); sb.append(theId).append(','); if (count++ > batchSize) { break; } } */ if (count == 0) { return; } jointTableIsData=true; sb.deleteCharAt(sb.length() - 1).append(')'); //select * from tableB where joinKey in (id1,id2); String sql = String.format(joinParser.getChildSQL(), sb); //if (!childRoute){ getRoute(sql); //childRoute=true; //} // ctx.executeNativeSQLParallJob(getDataNodes(),sql, new ShareRowOutPutDataHandler(this,fields,joinindex,joinParser.getJoinRkey(), batchRows,ctx.getSession())); EngineCtx.LOGGER.info("SQLParallJob:"+getDataNode(getDataNodes())+" sql:" + sql); } public void writeHeader(String dataNode,List afields, List bfields) { sendField++; if (sendField==1){ //huangyiming add 只是中间过程数据不能发送给客户端 MiddlerResultHandler middlerResultHandler = sc.getSession2().getMiddlerResultHandler(); if(middlerResultHandler ==null ){ ctx.writeHeader(afields, bfields); } setAllFields(afields, bfields); // EngineCtx.LOGGER.info("发送字段2:" + dataNode); } setRowDataSorterHeader(afields, bfields); } //设置排序结果收集齐 private void setRowDataSorterHeader(List afields, List bfields) { if(!ctx.getIsStreamOutputResult() && !isInit) { synchronized (joinParser) { if(!isInit){ LinkedHashMap orderByCols = joinParser.getOrderByCols(); LinkedHashMap childOrderByCols = joinParser.getChildByCols(); OrderCol[] orderCols = new OrderCol[orderByCols.size() + childOrderByCols.size()]; //a 表 排序字段 for(String fileldName : orderByCols.keySet()) { ColMeta colMeta = getCommonFieldIndex(afields, fileldName); //colMeta 放置类型，字段的位置 int val = orderByCols.get(fileldName); int orignIndex = TableFilter.decodeOrignOrder(val); //字段在排序时候的位子 int orderType = TableFilter.decodeOrderType(val); //0:asc or 1:desc orderCols[orignIndex] = new OrderCol(colMeta, orderType); } //b 表 for(String fileldName : childOrderByCols.keySet()) { ColMeta colMeta = getCommonFieldIndex(bfields, fileldName); colMeta.setColIndex(colMeta.getColIndex() + afields.size() -1); // b的字段的位子 在a表字段之后 而且 少了一个joinKey 所以-1 int val = childOrderByCols.get(fileldName); int orignIndex = TableFilter.decodeOrignOrder(val); int orderType = TableFilter.decodeOrderType(val); orderCols[orignIndex] = new OrderCol(colMeta, orderType); } RowDataSorter tmp = new RowDataSorter(orderCols); tmp.setLimit(joinParser.getOffset(), joinParser.getRowCount()); sorter = tmp; isInit = true; } } } } private void setAllFields(List afields, List bfields){ allfields=new ArrayList(); for (byte[] field : afields) { allfields.add(field); } //EngineCtx.LOGGER.info("所有字段2:" +allfields.size()); for (int i=1;i getAllFields(){ return allfields; } public void writeRow(RowDataPacket rowDataPkg){ if(ctx.getIsStreamOutputResult()){ ctx.writeRow(rowDataPkg); } else { if(isInit ){ //保证可见性 sorter.addRow(rowDataPkg); } else { EngineCtx.LOGGER.error("===怎么会还没初始化==="); } } } protected void writeEof() { boolean t = jointTableIsData; if(ctx.getIsStreamOutputResult() || (!jointTableIsData)){ ctx.writeEof(); } else { //limit 输出。 int start = joinParser.getOffset(); int end = start + joinParser.getRowCount(); List results = sorter.getSortedResult(); if (start < 0) { start = 0; } if (joinParser.getRowCount() <= 0) { end = results.size(); } if (end > results.size()) { end = results.size(); } for (int i = start; i < end; i++) { ctx.writeRow(results.get(i)); } ctx.writeEof(); } } //获取fKey在字段 的索引位置。 public int getFieldIndex(List fields,String fkey){ int i=0; for (byte[] field :fields) { FieldPacket fieldPacket = new FieldPacket(); fieldPacket.read(field); if (ByteUtil.getString(fieldPacket.orgName).equals(fkey)){ joinKeyType = fieldPacket.type; return i; } i++; } return i; } //获取fKey在字段 的索引位置。 public ColMeta getCommonFieldIndex(List fields,String fkey){ int i=0; for (byte[] field :fields) { FieldPacket fieldPacket = new FieldPacket(); fieldPacket.read(field); if (ByteUtil.getString(fieldPacket.orgName).equals(fkey)){ return new ColMeta(i, fieldPacket.type); } i++; } return null; } } class ShareDBJoinHandler implements SQLJobHandler { private List fields; private final ShareJoin ctx; private String joinkey; private NonBlockingSession session; public ShareDBJoinHandler(ShareJoin ctx,String joinField,NonBlockingSession session) { super(); this.ctx = ctx; this.joinkey=joinField; this.session = session; //EngineCtx.LOGGER.info("二次查询:" +" sql:" + querySQL+"/"+joinkey); } //private Map rows = new ConcurrentHashMap(); //private ConcurrentLinkedQueue ids = new ConcurrentLinkedQueue(); @Override public void onHeader(String dataNode, byte[] header, List fields) { this.fields = fields; ctx.putDBFields(fields); } /* public static String getFieldNames(List fields){ String str=""; for (byte[] field :fields) { FieldPacket fieldPacket = new FieldPacket(); fieldPacket.read(field); str+=ByteUtil.getString(fieldPacket.name)+","; } return str; } public static String getFieldName(byte[] field){ FieldPacket fieldPacket = new FieldPacket(); fieldPacket.read(field); return ByteUtil.getString(fieldPacket.name); } */ @Override public boolean onRowData(String dataNode, byte[] rowData) { int fid=this.ctx.getFieldIndex(fields,joinkey); String id = ResultSetUtil.getColumnValAsString(rowData, fields, 0);//主键，默认id String nid = ResultSetUtil.getColumnValAsString(rowData, fields, fid); //joinKey 的value // 放入结果集 //rows.put(id, rowData); ctx.putDBRow(id,nid, rowData,fid); return false; } // 收到结束包调用 或者发生错误时候调用。 @Override public void finished(String dataNode, boolean failed, String errorMsg) { if(failed){ session.getSource().writeErrMessage(ErrorCode.ER_UNKNOWN_ERROR, errorMsg); }else{ ctx.endJobInput(dataNode,failed); } } } class ShareRowOutPutDataHandler implements SQLJobHandler { private final List afields; // a表的字段信息 private List bfields; //B表(右边) 字段信息 private final ShareJoin ctx; // sharejoin 的context private final Map arows; // map a表的记录值 id -> 行记录 private int joinL;//A表(左边)关联字段的位置 private int joinR;//B表(右边)关联字段的位置 private String joinRkey;//B表(右边)关联字段 public NonBlockingSession session; public ShareRowOutPutDataHandler(ShareJoin ctx,List afields,int joini,String joinField,Map arows,NonBlockingSession session) { super(); this.afields = afields; // a表的字段信息 this.ctx = ctx; // sharejoin 的context this.arows = arows; // map a表的记录值 id -> 行记录 this.joinL =joini; //a 表 joinKey的索引位置。 this.joinRkey= joinField; // b 表 joinKey的字段名称。 this.session = session; // mycatSession //EngineCtx.LOGGER.info("二次查询:" +arows.size()+ " afields："+FenDBJoinHandler.getFieldNames(afields)); } @Override public void onHeader(String dataNode, byte[] header, List bfields) { this.bfields=bfields; joinR=this.ctx.getFieldIndex(bfields,joinRkey); MiddlerResultHandler middlerResultHandler = session.getMiddlerResultHandler(); if( middlerResultHandler ==null ){ ctx.writeHeader(dataNode,afields, bfields); } } //不是主键，获取join左边的的记录 private byte[] getRow(Map batchRowsCopy,String value,int index){ for(Map.Entry e: batchRowsCopy.entrySet() ){ String key=e.getKey(); RowDataPacket rowDataPkg = ResultSetUtil.parseRowData(e.getValue(), afields); byte[] columnValue = rowDataPkg.fieldValues.get(index); //a 表记录 joinKey的value if(columnValue == null ) continue; String id = ByteUtil.getString(columnValue); if (id.equals(value)){ return batchRowsCopy.remove(key); } } return null; } @Override public boolean onRowData(String dataNode, byte[] rowData) { RowDataPacket rowDataPkgold = ResultSetUtil.parseRowData(rowData, bfields); //拷贝一份batchRows Map batchRowsCopy = new ConcurrentHashMap(); batchRowsCopy.putAll(arows); // 获取Id字段， String id = ByteUtil.getString(rowDataPkgold.fieldValues.get(joinR)); // 查找ID对应的A表的记录 byte[] arow = getRow(batchRowsCopy,id,joinL);//arows.remove(id); // byte[] arow = getRow(id,joinL);//arows.remove(id); while (arow!=null) { RowDataPacket rowDataPkg = ResultSetUtil.parseRowData(arow,afields );//ctx.getAllFields()); //将 b记录的值 复制到 a记录中 成为新的记录。 for (int i=1;i0){ String rowValue = new String(columnData); middlerResultHandler.add(rowValue); } //} } } arow = getRow(batchRowsCopy,id,joinL); // arow = getRow(id,joinL); } return false; } @Override public void finished(String dataNode, boolean failed, String errorMsg) { if(failed){ session.getSource().writeErrMessage(ErrorCode.ER_UNKNOWN_ERROR, errorMsg); } } } ================================================ FILE: src/main/java/io/mycat/catlets/TableFilter.java ================================================ package io.mycat.catlets; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; import java.util.Map.Entry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.sqlengine.mpp.OrderCol; import io.mycat.util.StringUtil; /** * 功能详细描述:分片join,单独的语句 * @author sohudo[http://blog.csdn.net/wind520] * @create 2015年02月01日 * @version 0.0.1 */ public class TableFilter { protected static final Logger LOGGER = LoggerFactory.getLogger(TableFilter.class); private LinkedHashMap fieldAliasMap = new LinkedHashMap(); private String tName; //table 名称 private String tAlia; //table别名 private String where=""; //where 条件 private String order=""; private String parenTable="";//左连接的join的表 private String joinParentkey="";//左连接的join字段 private String joinKey=""; //join字段 private TableFilter join; //右连接的join表 private TableFilter parent; //左连接的join的表 private int offset=0; private int rowCount=0; private boolean outJoin; private boolean allField; public TableFilter(String taName,String taAlia,boolean outJoin) { this.tName=taName; this.tAlia=taAlia; this.outJoin=outJoin; this.where=""; } /** * a.fieldName 获取表名称 * 返回 table 名称。 */ private String getTablefrom(String key){ if (key==null){ return ""; } else { int i=key.indexOf('.'); if (i==-1){ return key; } else { return key.substring(0, i); } } } /** * a.fieldName 获取字段 * 返回 fieldName */ private String getFieldfrom(String key){ if (key==null){ return ""; } else { int i=key.indexOf('.'); if (i==-1){ return key; } else { return key.substring(i + 1); } } } public void addField(String fieldName,String fieldAlia){ String atable=getTablefrom(fieldName); //表名称 String afield=getFieldfrom(fieldName); //字段 boolean allfield=afield.equals("*")?true:false; //是否有* if (atable.equals("*")) { fieldAliasMap.put(afield, null); // 放入字段 * 到 fieldMap中。 setAllField(allfield); if (join!=null) { join.addField(fieldName,null); //递归设置子表 join.setAllField(allfield); } } else { if (atable.equals(tAlia)) { //将字段放入对应的表中。 fieldAliasMap.put(afield, fieldAlia); setAllField(allfield); } else { if (join!=null) {//B表中查询 是否可以放入字段。 join.addField(fieldName,fieldAlia); join.setAllField(allfield); } } } } //解析字段 public void addField(String fieldName,String fieldAlia,String expr){ String atable=getTablefrom(fieldName); String afield=getFieldfrom(fieldName); boolean allfield=afield.equals("*")?true:false; if (atable.equals("*")) { fieldAliasMap.put(afield, null); setAllField(allfield); if (join!=null) { join.addField(fieldName,null); join.setAllField(allfield); } } else { if (atable.equals(tAlia)) { expr = expr.replace(fieldName, afield); fieldAliasMap.put(expr, fieldAlia); setAllField(allfield); } else { if (join!=null) { join.addField(fieldName,fieldAlia,expr); join.setAllField(allfield); } } } } public void addWhere(String fieldName,String value,String Operator,String and){ String atable=getTablefrom(fieldName); String afield=getFieldfrom(fieldName); if (atable.equals(tAlia)) { // 修复在拼接sql的时候少空格的bug where=unionsql(where,afield + " " + Operator + " " + value,and); } else { if (join!=null) { join.addWhere(fieldName,value,Operator,and); } } } public void addWhere(String fieldName,String condition,String and){ String atable=getTablefrom(fieldName); String afield=getFieldfrom(fieldName); condition = condition.replace(fieldName, afield); if (atable.equals(tAlia)) { where=unionsql(where,condition,and); } else { if (join!=null) { join.addWhere(fieldName,condition,and); } } } private String unionsql(String key,String value,String Operator){ if (key.trim().equals("")){ key=value; } else { key+=" "+Operator+" "+value; } return key; } LinkedHashMap orderByCols = new LinkedHashMap(); public void addOrders(int index, String fieldName,String des){ // String atable=getTablefrom(fieldName); String afield=getFieldfrom(fieldName); if (fieldInTable(fieldName)) { order=unionsql(order,afield+" "+des,","); //将order 类型加入到链表中. 還得看下是否使用別名 int orderType = StringUtil.isEmpty(des)||"asc".equals(des.toLowerCase().trim())?OrderCol.COL_ORDER_TYPE_ASC:OrderCol.COL_ORDER_TYPE_DESC; orderByCols.put(afield, encodeOrignOrderType(index, orderType)); } else { if (join!=null) { join.addOrders(index, fieldName,des); } } } /* * 判断字段是否在表中。 * */ private boolean fieldInTable(String fieldName) { String atable=getTablefrom(fieldName); if(atable.equals(tAlia) ){ return true; } if(StringUtil.isEmpty(atable)){ String afield=getFieldfrom(fieldName); for(String tableField : fieldAliasMap.keySet()) { if(afield.equals(fieldAliasMap.get(tableField))) { return true; } } } return false; } public LinkedHashMap getOrderByCols(){ return orderByCols; } public void addLimit(int offset,int rowCount){ this.offset=offset; this.rowCount=rowCount; } public void setJoinKey(String fieldName,String value){ if (parent==null){ if (join!=null) { join.setJoinKey(fieldName,value); } } else { //1 ，fileName 为当前的joinKey //2 value 为当前的joinKey int i=joinLkey(fieldName,value); if (i==1){ joinParentkey=getFieldfrom(value); parenTable =getTablefrom(value); joinKey=getFieldfrom(fieldName); } else { if (i==2){ joinParentkey=getFieldfrom(fieldName); parenTable =getTablefrom(fieldName); joinKey=getFieldfrom(value); } else { if (join!=null) { join.setJoinKey(fieldName,value); } } } } } private String getChildJoinKey(boolean left){ if (join!=null){ if (left) { return join.joinParentkey; } else { return join.joinKey; } } else { return ""; } } public String getJoinKey(boolean left){ return getChildJoinKey(left); } private int joinLkey(String fieldName,String value){ String key1=getTablefrom(fieldName); String key2=getTablefrom(value); if (key1.equals(tAlia) ) { return 1; } if (key2.equals(tAlia) ) { return 2; } /* String bAlia=""; if (join!=null){ bAlia=join.getTableAlia(); } if (key1.equals(tAlia)&& key2.equals(bAlia) ) { return 1; } if (key2.equals(tAlia)&& key1.equals(bAlia) ) { return 2; } */ return 0; } public String getTableName(){ return tName; } public void setTableName(String value){ tName=value; } public String getTableAlia(){ return tAlia; } public void setTableAlia(String value){ tAlia=value; } public boolean getOutJoin(){ return outJoin; } public void setOutJoin(boolean value){ outJoin=value; } public boolean getAllField(){ return allField; } public void setAllField(boolean value){ allField=value; } public TableFilter getTableJoin(){ return join; } public void setTableJoin(TableFilter value){ join=value; join.setParent(this); } public TableFilter getParent() { return parent; } public void setParent(TableFilter parent) { this.parent = parent; } private String unionField(String field,String key,String Operator){ if (key.trim().equals("")){ key=field; } else { key=field+Operator+" "+key; } return key; } public String getSQL(){ String sql=""; Iterator> iter = fieldAliasMap.entrySet().iterator(); while (iter.hasNext()) { Map.Entry entry = (Map.Entry) iter.next(); String key = entry.getKey(); String val = entry.getValue(); if (val==null) { sql=unionsql(sql,getFieldfrom(key),","); } else { sql = unionsql(sql, getFieldfrom(key) + " as " + val, ","); } } if (parent==null){ // on/where 等于号左边的表 String parentJoinKey = getJoinKey(true); // fix sharejoin bug： // (AbstractConnection.java:458) -close connection,reason:program err:java.lang.IndexOutOfBoundsException: // 原因是左表的select列没有包含 join 列，在获取结果时报上面的错误 if(sql != null && parentJoinKey != null && sql.toUpperCase().indexOf(parentJoinKey.trim().toUpperCase()) == -1){ sql += ", " + parentJoinKey; } sql="select "+sql+" from "+tName; if (!(where.trim().equals(""))){ sql+=" where "+where.trim(); } } else { // on/where 等于号右边边的表 if (allField) { sql="select "+sql+" from "+tName; } else { sql=unionField("select "+joinKey,sql,","); sql=sql+" from "+tName; //sql="select "+joinKey+","+sql+" from "+tName; } if (!(where.trim().equals(""))){ sql+=" where "+where.trim()+" and ("+joinKey+" in %s )"; } else { sql+=" where "+joinKey+" in %s "; } } /* if (!(order.trim().equals(""))){ sql+=" order by "+order.trim(); } if (parent==null){ if ((rowCount>0)&& (offset>0)){ sql+=" limit "+offset+","+rowCount; } else { if (rowCount>0){ sql+=" limit "+rowCount; } } } */ return sql; } public int getOffset() { return offset; } public int getRowCount() { return rowCount; } public static int encodeOrignOrderType(int index , int orderType) { return index << 1 | orderType; } public static int decodeOrderType(int orignOrderType){ return orignOrderType & 1; } public static int decodeOrignOrder(int orignOrderType){ return orignOrderType >> 1; } public static void main(String[] args) { int val = encodeOrignOrderType(45, 1); System.out.println(decodeOrderType(val)); System.out.println(decodeOrignOrder(val)); } @Override public String toString() { return "TableFilter [fieldAliasMap=" + fieldAliasMap + ", tName=" + tName + ", tAlia=" + tAlia + ", where=" + where + ", order=" + order + ", parenTable=" + parenTable + ", joinParentkey=" + joinParentkey + ", joinKey=" + joinKey + ", join=" + join + ", offset=" + offset + ", rowCount=" + rowCount + ", outJoin=" + outJoin + ", allField=" + allField + "]"; } } ================================================ FILE: src/main/java/io/mycat/config/Alarms.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config; /** * Mycat报警关键词定义 * * @author mycat */ public interface Alarms { /** 默认报警关键词 **/ public static final String DEFAULT = "#!MyCat#"; /** 集群无有效的节点可提供服务 **/ public static final String CLUSTER_EMPTY = "#!CLUSTER_EMPTY#"; /** 数据节点的数据源发生切换 **/ public static final String DATANODE_SWITCH = "#!DN_SWITCH#"; /** 防火墙非法用户访问 **/ public static final String FIREWALL_ATTACK = "#!QT_ATTACK#"; /** 非法DML **/ public static final String DML_ATTACK = "#!DML_ATTACK#"; } ================================================ FILE: src/main/java/io/mycat/config/Capabilities.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config; /** * 处理能力标识定义 * * @author mycat */ public interface Capabilities { /** * server capabilities * * 
     * server:        11110111 11111111
     * client_cmd: 11 10100110 10000101
     * client_jdbc:10 10100010 10001111
     *  
     * @see http://dev.mysql.com/doc/refman/5.1/en/mysql-real-connect.html
     *
*/ // new more secure passwords public static final int CLIENT_LONG_PASSWORD = 1; // Found instead of affected rows // 返回找到（匹配）的行数，而不是改变了的行数。 public static final int CLIENT_FOUND_ROWS = 2; // Get all column flags public static final int CLIENT_LONG_FLAG = 4; // One can specify db on connect public static final int CLIENT_CONNECT_WITH_DB = 8; // Don't allow database.table.column // 不允许“数据库名.表名.列名”这样的语法。这是对于ODBC的设置。 // 当使用这样的语法时解析器会产生一个错误，这对于一些ODBC的程序限制bug来说是有用的。 public static final int CLIENT_NO_SCHEMA = 16; // Can use compression protocol // 使用压缩协议 public static final int CLIENT_COMPRESS = 32; // Odbc client public static final int CLIENT_ODBC = 64; // Can use LOAD DATA LOCAL public static final int CLIENT_LOCAL_FILES = 128; // Ignore spaces before '(' // 允许在函数名后使用空格。所有函数名可以预留字。 public static final int CLIENT_IGNORE_SPACE = 256; // New 4.1 protocol This is an interactive client public static final int CLIENT_PROTOCOL_41 = 512; // This is an interactive client // 允许使用关闭连接之前的不活动交互超时的描述，而不是等待超时秒数。 // 客户端的会话等待超时变量变为交互超时变量。 public static final int CLIENT_INTERACTIVE = 1024; // Switch to SSL after handshake // 使用SSL。这个设置不应该被应用程序设置，他应该是在客户端库内部是设置的。 // 可以在调用mysql_real_connect()之前调用mysql_ssl_set()来代替设置。 public static final int CLIENT_SSL = 2048; // IGNORE sigpipes // 阻止客户端库安装一个SIGPIPE信号处理器。 // 这个可以用于当应用程序已经安装该处理器的时候避免与其发生冲突。 public static final int CLIENT_IGNORE_SIGPIPE = 4096; // Client knows about transactions public static final int CLIENT_TRANSACTIONS = 8192; // Old flag for 4.1 protocol public static final int CLIENT_RESERVED = 16384; // New 4.1 authentication public static final int CLIENT_SECURE_CONNECTION = 32768; // Enable/disable multi-stmt support // 通知服务器客户端可以发送多条语句（由分号分隔）。如果该标志为没有被设置，多条语句执行。 public static final int CLIENT_MULTI_STATEMENTS = 65536; // Enable/disable multi-results // 通知服务器客户端可以处理由多语句或者存储过程执行生成的多结果集。 // 当打开CLIENT_MULTI_STATEMENTS时，这个标志自动的被打开。 public static final int CLIENT_MULTI_RESULTS = 131072; public static final int CLIENT_PLUGIN_AUTH = 0x00080000; // 524288 } ================================================ FILE: src/main/java/io/mycat/config/ConfigInitializer.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config; import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Set; import io.mycat.config.loader.zkprocess.comm.ZkConfig; import io.mycat.config.loader.zkprocess.comm.ZkParamCfg; import org.apache.log4j.Logger; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.jdbc.JDBCDatasource; import io.mycat.backend.mysql.nio.MySQLDataSource; import io.mycat.backend.postgresql.PostgreSQLDataSource; import io.mycat.config.loader.ConfigLoader; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.loader.xml.XMLConfigLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.DBHostConfig; import io.mycat.config.model.DataHostConfig; import io.mycat.config.model.DataNodeConfig; import io.mycat.config.model.FirewallConfig; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.config.model.UserConfig; import io.mycat.config.util.ConfigException; import io.mycat.route.sequence.handler.DistributedSequenceHandler; import io.mycat.route.sequence.handler.IncrSequenceMySQLHandler; import io.mycat.route.sequence.handler.IncrSequenceTimeHandler; import io.mycat.route.sequence.handler.IncrSequenceZKHandler; /** * @author mycat */ public class ConfigInitializer { private static final Logger LOGGER = Logger.getLogger( ConfigInitializer.class ); private volatile SystemConfig system; private volatile MycatCluster cluster; private volatile FirewallConfig firewall; private volatile Map users; private volatile Map schemas; private volatile Map dataNodes; private volatile Map dataHosts; public ConfigInitializer(boolean loadDataHost) { //读取rule.xml和schema.xml SchemaLoader schemaLoader = new XMLSchemaLoader(); //读取server.xml XMLConfigLoader configLoader = new XMLConfigLoader(schemaLoader); schemaLoader = null; //加载配置 this.system = configLoader.getSystemConfig(); this.users = configLoader.getUserConfigs(); this.schemas = configLoader.getSchemaConfigs(); //是否重新加载DataHost和对应的DataNode if (loadDataHost) { this.dataHosts = initDataHosts(configLoader); this.dataNodes = initDataNodes(configLoader); } //权限管理 this.firewall = configLoader.getFirewallConfig(); this.cluster = initCobarCluster(configLoader); //不同类型的全局序列处理器的配置加载 if (system.getSequenceHandlerType() == SystemConfig.SEQUENCEHANDLER_MYSQLDB) { IncrSequenceMySQLHandler.getInstance().load(); } if (system.getSequenceHandlerType() == SystemConfig.SEQUENCEHANDLER_LOCAL_TIME) { IncrSequenceTimeHandler.getInstance().load(); } if (system.getSequenceHandlerType() == SystemConfig.SEQUENCEHANDLER_ZK_DISTRIBUTED) { DistributedSequenceHandler.getInstance(system).load(); } if (system.getSequenceHandlerType() == SystemConfig.SEQUENCEHANDLER_ZK_GLOBAL_INCREMENT) { IncrSequenceZKHandler.getInstance().load(); } /** * 配置文件初始化， 自检 */ this.selfChecking0(); } private void selfChecking0() throws ConfigException { // 检查user与schema配置对应以及schema配置不为空 if (users == null || users.isEmpty()) { throw new ConfigException("SelfCheck### user all node is empty!"); } else { for (UserConfig uc : users.values()) { if (uc == null) { throw new ConfigException("SelfCheck### users node within the item is empty!"); } Set authSchemas = uc.getSchemas(); if (authSchemas == null) { throw new ConfigException("SelfCheck### user " + uc.getName() + "refered schemas is empty!"); } for (String schema : authSchemas) { if ( !schemas.containsKey(schema) ) { String errMsg = "SelfCheck### schema " + schema + " refered by user " + uc.getName() + " is not exist!"; throw new ConfigException(errMsg); } } } } // schema 配置检测 for (SchemaConfig sc : schemas.values()) { if (null == sc) { throw new ConfigException("SelfCheck### schema all node is empty!"); } else { // check dataNode / dataHost 节点 if ( this.dataNodes != null && this.dataHosts != null ) { Set dataNodeNames = sc.getAllDataNodes(); for(String dataNodeName: dataNodeNames) { PhysicalDBNode node = this.dataNodes.get(dataNodeName); if ( node == null ) { throw new ConfigException("SelfCheck### schema dbnode is empty!"); } } } } } } public void testConnection() { // 实际链路的连接测试 if ( this.dataNodes != null && this.dataHosts != null ) { Map map = new HashMap(); for(PhysicalDBNode dataNode: dataNodes.values() ) { String database = dataNode.getDatabase(); PhysicalDBPool pool = dataNode.getDbPool(); for (PhysicalDatasource ds : pool.getAllDataSources()) { String key = ds.getName() + "_" + database; if ( map.get( key ) == null ) { map.put( key, false ); boolean isConnected = false; try { isConnected = ds.testConnection( database ); map.put( key, isConnected ); } catch (IOException e) { LOGGER.warn("test conn error:", e); } } } } // boolean isConnectivity = true; for (Map.Entry entry : map.entrySet()) { String key = entry.getKey(); Boolean value = entry.getValue(); if ( !value && isConnectivity ) { LOGGER.warn("SelfCheck### test " + key + " database connection failed "); isConnectivity = false; } else { LOGGER.info("SelfCheck### test " + key + " database connection success "); } } if ( !isConnectivity ) { throw new ConfigException("SelfCheck### there are some datasource connection failed, pls check!"); } } } public SystemConfig getSystem() { return system; } public MycatCluster getCluster() { return cluster; } public FirewallConfig getFirewall() { return firewall; } public Map getUsers() { return users; } public Map getSchemas() { return schemas; } public Map getDataNodes() { return dataNodes; } public Map getDataHosts() { return this.dataHosts; } private MycatCluster initCobarCluster(ConfigLoader configLoader) { return new MycatCluster(configLoader.getClusterConfig()); } private Map initDataHosts(ConfigLoader configLoader) { Map nodeConfs = configLoader.getDataHosts(); boolean isBooster="booster".equalsIgnoreCase(ZkConfig.getInstance().getValue(ZkParamCfg.MYCAT_SERVER_TYPE) ) ; //根据DataHost建立PhysicalDBPool，其实就是实际数据库连接池，每个DataHost对应一个PhysicalDBPool Map nodes = new HashMap( nodeConfs.size()); for (DataHostConfig conf : nodeConfs.values()) { if(isBooster){ conf.setMinCon(2); } //建立PhysicalDBPool PhysicalDBPool pool = getPhysicalDBPool(conf, configLoader); nodes.put(pool.getHostName(), pool); } return nodes; } private PhysicalDatasource[] createDataSource(DataHostConfig conf, String hostName, String dbType, String dbDriver, DBHostConfig[] nodes, boolean isRead) { PhysicalDatasource[] dataSources = new PhysicalDatasource[nodes.length]; if (dbType.equals("mysql") && dbDriver.equals("native")) { for (int i = 0; i < nodes.length; i++) { //设置最大idle时间，默认为30分钟 nodes[i].setIdleTimeout(system.getIdleTimeout()); MySQLDataSource ds = new MySQLDataSource(nodes[i], conf, isRead); dataSources[i] = ds; } } else if (dbDriver.equals("jdbc")) { for (int i = 0; i < nodes.length; i++) { nodes[i].setIdleTimeout(system.getIdleTimeout()); JDBCDatasource ds = new JDBCDatasource(nodes[i], conf, isRead); dataSources[i] = ds; } } else if ("postgresql".equalsIgnoreCase(dbType) && dbDriver.equalsIgnoreCase("native")){ for (int i = 0; i < nodes.length; i++) { nodes[i].setIdleTimeout(system.getIdleTimeout()); PostgreSQLDataSource ds = new PostgreSQLDataSource(nodes[i], conf, isRead); dataSources[i] = ds; } } else{ throw new ConfigException("not supported yet !" + hostName); } return dataSources; } private PhysicalDBPool getPhysicalDBPool(DataHostConfig conf, ConfigLoader configLoader) { String name = conf.getName(); //数据库类型，我们这里只讨论MySQL String dbType = conf.getDbType(); //连接数据库驱动，我们这里只讨论MyCat自己实现的native String dbDriver = conf.getDbDriver(); //针对所有写节点创建PhysicalDatasource PhysicalDatasource[] writeSources = createDataSource(conf, name, dbType, dbDriver, conf.getWriteHosts(), false); Map readHostsMap = conf.getReadHosts(); Map readSourcesMap = new HashMap( readHostsMap.size()); //对于每个读节点建立key为writeHost下标value为readHost的PhysicalDatasource[]的哈希表 for (Map.Entry entry : readHostsMap.entrySet()) { PhysicalDatasource[] readSources = createDataSource(conf, name, dbType, dbDriver, entry.getValue(), true); readSourcesMap.put(entry.getKey(), readSources); } PhysicalDBPool pool = new PhysicalDBPool(conf.getName(), conf, writeSources, readSourcesMap, conf.getBalance(), conf.getWriteType()); pool.setSlaveIDs(conf.getSlaveIDs()); return pool; } private Map initDataNodes(ConfigLoader configLoader) { Map nodeConfs = configLoader.getDataNodes(); Map nodes = new HashMap( nodeConfs.size()); for (DataNodeConfig conf : nodeConfs.values()) { PhysicalDBPool pool = this.dataHosts.get(conf.getDataHost()); if (pool == null) { throw new ConfigException("dataHost not exists " + conf.getDataHost()); } PhysicalDBNode dataNode = new PhysicalDBNode(conf.getName(), conf.getDatabase(), pool); nodes.put(dataNode.getName(), dataNode); } return nodes; } } ================================================ FILE: src/main/java/io/mycat/config/ErrorCode.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config; /** * @author mycat */ public interface ErrorCode { // mycat error code public static final int ERR_BAD_LOGICDB = 3000; public static final int ERR_OPEN_SOCKET = 3001; public static final int ERR_CONNECT_SOCKET = 3002; public static final int ERR_REGISTER = 3004; public static final int ERR_READ = 3005; public static final int ERR_PUT_WRITE_QUEUE = 3006; public static final int ERR_WRITE_BY_EVENT = 3007; public static final int ERR_WRITE_BY_QUEUE = 3008; public static final int ERR_HANDLE_DATA = 3009; public static final int ERR_NOT_SUPPORTED = 3010; public static final int ERR_MULTI_NODE_FAILED = 3011; public static final int ERR_WRONG_USED = 3012; public static final int ERR_FOUND_EXCEPTION = 3344; // mysql error code public static final int ER_HASHCHK = 1000; public static final int ER_NISAMCHK = 1001; public static final int ER_NO = 1002; public static final int ER_YES = 1003; public static final int ER_CANT_CREATE_FILE = 1004; public static final int ER_CANT_CREATE_TABLE = 1005; public static final int ER_CANT_CREATE_DB = 1006; public static final int ER_DB_CREATE_EXISTS = 1007; public static final int ER_DB_DROP_EXISTS = 1008; public static final int ER_DB_DROP_DELETE = 1009; public static final int ER_DB_DROP_RMDIR = 1010; public static final int ER_CANT_DELETE_FILE = 1011; public static final int ER_CANT_FIND_SYSTEM_REC = 1012; public static final int ER_CANT_GET_STAT = 1013; public static final int ER_CANT_GET_WD = 1014; public static final int ER_CANT_LOCK = 1015; public static final int ER_CANT_OPEN_FILE = 1016; public static final int ER_FILE_NOT_FOUND = 1017; public static final int ER_CANT_READ_DIR = 1018; public static final int ER_CANT_SET_WD = 1019; public static final int ER_CHECKREAD = 1020; public static final int ER_DISK_FULL = 1021; public static final int ER_DUP_KEY = 1022; public static final int ER_ERROR_ON_CLOSE = 1023; public static final int ER_ERROR_ON_READ = 1024; public static final int ER_ERROR_ON_RENAME = 1025; public static final int ER_ERROR_ON_WRITE = 1026; public static final int ER_FILE_USED = 1027; public static final int ER_FILSORT_ABORT = 1028; public static final int ER_FORM_NOT_FOUND = 1029; public static final int ER_GET_ERRNO = 1030; public static final int ER_ILLEGAL_HA = 1031; public static final int ER_KEY_NOT_FOUND = 1032; public static final int ER_NOT_FORM_FILE = 1033; public static final int ER_NOT_KEYFILE = 1034; public static final int ER_OLD_KEYFILE = 1035; public static final int ER_OPEN_AS_READONLY = 1036; public static final int ER_OUTOFMEMORY = 1037; public static final int ER_OUT_OF_SORTMEMORY = 1038; public static final int ER_UNEXPECTED_EOF = 1039; public static final int ER_CON_COUNT_ERROR = 1040; public static final int ER_OUT_OF_RESOURCES = 1041; public static final int ER_BAD_HOST_ERROR = 1042; public static final int ER_HANDSHAKE_ERROR = 1043; public static final int ER_DBACCESS_DENIED_ERROR = 1044; public static final int ER_ACCESS_DENIED_ERROR = 1045; public static final int ER_NO_DB_ERROR = 1046; public static final int ER_UNKNOWN_COM_ERROR = 1047; public static final int ER_BAD_NULL_ERROR = 1048; public static final int ER_BAD_DB_ERROR = 1049; public static final int ER_TABLE_EXISTS_ERROR = 1050; public static final int ER_BAD_TABLE_ERROR = 1051; public static final int ER_NON_UNIQ_ERROR = 1052; public static final int ER_SERVER_SHUTDOWN = 1053; public static final int ER_BAD_FIELD_ERROR = 1054; public static final int ER_WRONG_FIELD_WITH_GROUP = 1055; public static final int ER_WRONG_GROUP_FIELD = 1056; public static final int ER_WRONG_SUM_SELECT = 1057; public static final int ER_WRONG_VALUE_COUNT = 1058; public static final int ER_TOO_LONG_IDENT = 1059; public static final int ER_DUP_FIELDNAME = 1060; public static final int ER_DUP_KEYNAME = 1061; public static final int ER_DUP_ENTRY = 1062; public static final int ER_WRONG_FIELD_SPEC = 1063; public static final int ER_PARSE_ERROR = 1064; public static final int ER_EMPTY_QUERY = 1065; public static final int ER_NONUNIQ_TABLE = 1066; public static final int ER_INVALID_DEFAULT = 1067; public static final int ER_MULTIPLE_PRI_KEY = 1068; public static final int ER_TOO_MANY_KEYS = 1069; public static final int ER_TOO_MANY_KEY_PARTS = 1070; public static final int ER_TOO_LONG_KEY = 1071; public static final int ER_KEY_COLUMN_DOES_NOT_EXITS = 1072; public static final int ER_BLOB_USED_AS_KEY = 1073; public static final int ER_TOO_BIG_FIELDLENGTH = 1074; public static final int ER_WRONG_AUTO_KEY = 1075; public static final int ER_READY = 1076; public static final int ER_NORMAL_SHUTDOWN = 1077; public static final int ER_GOT_SIGNAL = 1078; public static final int ER_SHUTDOWN_COMPLETE = 1079; public static final int ER_FORCING_CLOSE = 1080; public static final int ER_IPSOCK_ERROR = 1081; public static final int ER_NO_SUCH_INDEX = 1082; public static final int ER_WRONG_FIELD_TERMINATORS = 1083; public static final int ER_BLOBS_AND_NO_TERMINATED = 1084; public static final int ER_TEXTFILE_NOT_READABLE = 1085; public static final int ER_FILE_EXISTS_ERROR = 1086; public static final int ER_LOAD_INFO = 1087; public static final int ER_ALTER_INFO = 1088; public static final int ER_WRONG_SUB_KEY = 1089; public static final int ER_CANT_REMOVE_ALL_FIELDS = 1090; public static final int ER_CANT_DROP_FIELD_OR_KEY = 1091; public static final int ER_INSERT_INFO = 1092; public static final int ER_UPDATE_TABLE_USED = 1093; public static final int ER_NO_SUCH_THREAD = 1094; public static final int ER_KILL_DENIED_ERROR = 1095; public static final int ER_NO_TABLES_USED = 1096; public static final int ER_TOO_BIG_SET = 1097; public static final int ER_NO_UNIQUE_LOGFILE = 1098; public static final int ER_TABLE_NOT_LOCKED_FOR_WRITE = 1099; public static final int ER_TABLE_NOT_LOCKED = 1100; public static final int ER_BLOB_CANT_HAVE_DEFAULT = 1101; public static final int ER_WRONG_DB_NAME = 1102; public static final int ER_WRONG_TABLE_NAME = 1103; public static final int ER_TOO_BIG_SELECT = 1104; public static final int ER_UNKNOWN_ERROR = 1105; public static final int ER_UNKNOWN_PROCEDURE = 1106; public static final int ER_WRONG_PARAMCOUNT_TO_PROCEDURE = 1107; public static final int ER_WRONG_PARAMETERS_TO_PROCEDURE = 1108; public static final int ER_UNKNOWN_TABLE = 1109; public static final int ER_FIELD_SPECIFIED_TWICE = 1110; public static final int ER_INVALID_GROUP_FUNC_USE = 1111; public static final int ER_UNSUPPORTED_EXTENSION = 1112; public static final int ER_TABLE_MUST_HAVE_COLUMNS = 1113; public static final int ER_RECORD_FILE_FULL = 1114; public static final int ER_UNKNOWN_CHARACTER_SET = 1115; public static final int ER_TOO_MANY_TABLES = 1116; public static final int ER_TOO_MANY_FIELDS = 1117; public static final int ER_TOO_BIG_ROWSIZE = 1118; public static final int ER_STACK_OVERRUN = 1119; public static final int ER_WRONG_OUTER_JOIN = 1120; public static final int ER_NULL_COLUMN_IN_INDEX = 1121; public static final int ER_CANT_FIND_UDF = 1122; public static final int ER_CANT_INITIALIZE_UDF = 1123; public static final int ER_UDF_NO_PATHS = 1124; public static final int ER_UDF_EXISTS = 1125; public static final int ER_CANT_OPEN_LIBRARY = 1126; public static final int ER_CANT_FIND_DL_ENTRY = 1127; public static final int ER_FUNCTION_NOT_DEFINED = 1128; public static final int ER_HOST_IS_BLOCKED = 1129; public static final int ER_HOST_NOT_PRIVILEGED = 1130; public static final int ER_PASSWORD_ANONYMOUS_USER = 1131; public static final int ER_PASSWORD_NOT_ALLOWED = 1132; public static final int ER_PASSWORD_NO_MATCH = 1133; public static final int ER_UPDATE_INFO = 1134; public static final int ER_CANT_CREATE_THREAD = 1135; public static final int ER_WRONG_VALUE_COUNT_ON_ROW = 1136; public static final int ER_CANT_REOPEN_TABLE = 1137; public static final int ER_INVALID_USE_OF_NULL = 1138; public static final int ER_REGEXP_ERROR = 1139; public static final int ER_MIX_OF_GROUP_FUNC_AND_FIELDS = 1140; public static final int ER_NONEXISTING_GRANT = 1141; public static final int ER_TABLEACCESS_DENIED_ERROR = 1142; public static final int ER_COLUMNACCESS_DENIED_ERROR = 1143; public static final int ER_ILLEGAL_GRANT_FOR_TABLE = 1144; public static final int ER_GRANT_WRONG_HOST_OR_USER = 1145; public static final int ER_NO_SUCH_TABLE = 1146; public static final int ER_NONEXISTING_TABLE_GRANT = 1147; public static final int ER_NOT_ALLOWED_COMMAND = 1148; public static final int ER_SYNTAX_ERROR = 1149; public static final int ER_DELAYED_CANT_CHANGE_LOCK = 1150; public static final int ER_TOO_MANY_DELAYED_THREADS = 1151; public static final int ER_ABORTING_CONNECTION = 1152; public static final int ER_NET_PACKET_TOO_LARGE = 1153; public static final int ER_NET_READ_ERROR_FROM_PIPE = 1154; public static final int ER_NET_FCNTL_ERROR = 1155; public static final int ER_NET_PACKETS_OUT_OF_ORDER = 1156; public static final int ER_NET_UNCOMPRESS_ERROR = 1157; public static final int ER_NET_READ_ERROR = 1158; public static final int ER_NET_READ_INTERRUPTED = 1159; public static final int ER_NET_ERROR_ON_WRITE = 1160; public static final int ER_NET_WRITE_INTERRUPTED = 1161; public static final int ER_TOO_LONG_STRING = 1162; public static final int ER_TABLE_CANT_HANDLE_BLOB = 1163; public static final int ER_TABLE_CANT_HANDLE_AUTO_INCREMENT = 1164; public static final int ER_DELAYED_INSERT_TABLE_LOCKED = 1165; public static final int ER_WRONG_COLUMN_NAME = 1166; public static final int ER_WRONG_KEY_COLUMN = 1167; public static final int ER_WRONG_MRG_TABLE = 1168; public static final int ER_DUP_UNIQUE = 1169; public static final int ER_BLOB_KEY_WITHOUT_LENGTH = 1170; public static final int ER_PRIMARY_CANT_HAVE_NULL = 1171; public static final int ER_TOO_MANY_ROWS = 1172; public static final int ER_REQUIRES_PRIMARY_KEY = 1173; public static final int ER_NO_RAID_COMPILED = 1174; public static final int ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE = 1175; public static final int ER_KEY_DOES_NOT_EXITS = 1176; public static final int ER_CHECK_NO_SUCH_TABLE = 1177; public static final int ER_CHECK_NOT_IMPLEMENTED = 1178; public static final int ER_CANT_DO_THIS_DURING_AN_TRANSACTION = 1179; public static final int ER_ERROR_DURING_COMMIT = 1180; public static final int ER_ERROR_DURING_ROLLBACK = 1181; public static final int ER_ERROR_DURING_FLUSH_LOGS = 1182; public static final int ER_ERROR_DURING_CHECKPOINT = 1183; public static final int ER_NEW_ABORTING_CONNECTION = 1184; public static final int ER_DUMP_NOT_IMPLEMENTED = 1185; public static final int ER_FLUSH_MASTER_BINLOG_CLOSED = 1186; public static final int ER_INDEX_REBUILD = 1187; public static final int ER_MASTER = 1188; public static final int ER_MASTER_NET_READ = 1189; public static final int ER_MASTER_NET_WRITE = 1190; public static final int ER_FT_MATCHING_KEY_NOT_FOUND = 1191; public static final int ER_LOCK_OR_ACTIVE_TRANSACTION = 1192; public static final int ER_UNKNOWN_SYSTEM_VARIABLE = 1193; public static final int ER_CRASHED_ON_USAGE = 1194; public static final int ER_CRASHED_ON_REPAIR = 1195; public static final int ER_WARNING_NOT_COMPLETE_ROLLBACK = 1196; public static final int ER_TRANS_CACHE_FULL = 1197; public static final int ER_SLAVE_MUST_STOP = 1198; public static final int ER_SLAVE_NOT_RUNNING = 1199; public static final int ER_BAD_SLAVE = 1200; public static final int ER_MASTER_INFO = 1201; public static final int ER_SLAVE_THREAD = 1202; public static final int ER_TOO_MANY_USER_CONNECTIONS = 1203; public static final int ER_SET_CONSTANTS_ONLY = 1204; public static final int ER_LOCK_WAIT_TIMEOUT = 1205; public static final int ER_LOCK_TABLE_FULL = 1206; public static final int ER_READ_ONLY_TRANSACTION = 1207; public static final int ER_DROP_DB_WITH_READ_LOCK = 1208; public static final int ER_CREATE_DB_WITH_READ_LOCK = 1209; public static final int ER_WRONG_ARGUMENTS = 1210; public static final int ER_NO_PERMISSION_TO_CREATE_USER = 1211; public static final int ER_UNION_TABLES_IN_DIFFERENT_DIR = 1212; public static final int ER_LOCK_DEADLOCK = 1213; public static final int ER_TABLE_CANT_HANDLE_FT = 1214; public static final int ER_CANNOT_ADD_FOREIGN = 1215; public static final int ER_NO_REFERENCED_ROW = 1216; public static final int ER_ROW_IS_REFERENCED = 1217; public static final int ER_CONNECT_TO_MASTER = 1218; public static final int ER_QUERY_ON_MASTER = 1219; public static final int ER_ERROR_WHEN_EXECUTING_COMMAND = 1220; public static final int ER_WRONG_USAGE = 1221; public static final int ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT = 1222; public static final int ER_CANT_UPDATE_WITH_READLOCK = 1223; public static final int ER_MIXING_NOT_ALLOWED = 1224; public static final int ER_DUP_ARGUMENT = 1225; public static final int ER_USER_LIMIT_REACHED = 1226; public static final int ER_SPECIFIC_ACCESS_DENIED_ERROR = 1227; public static final int ER_LOCAL_VARIABLE = 1228; public static final int ER_GLOBAL_VARIABLE = 1229; public static final int ER_NO_DEFAULT = 1230; public static final int ER_WRONG_VALUE_FOR_VAR = 1231; public static final int ER_WRONG_TYPE_FOR_VAR = 1232; public static final int ER_VAR_CANT_BE_READ = 1233; public static final int ER_CANT_USE_OPTION_HERE = 1234; public static final int ER_NOT_SUPPORTED_YET = 1235; public static final int ER_MASTER_FATAL_ERROR_READING_BINLOG = 1236; public static final int ER_SLAVE_IGNORED_TABLE = 1237; public static final int ER_INCORRECT_GLOBAL_LOCAL_VAR = 1238; public static final int ER_WRONG_FK_DEF = 1239; public static final int ER_KEY_REF_DO_NOT_MATCH_TABLE_REF = 1240; public static final int ER_OPERAND_COLUMNS = 1241; public static final int ER_SUBQUERY_NO_1_ROW = 1242; public static final int ER_UNKNOWN_STMT_HANDLER = 1243; public static final int ER_CORRUPT_HELP_DB = 1244; public static final int ER_CYCLIC_REFERENCE = 1245; public static final int ER_AUTO_CONVERT = 1246; public static final int ER_ILLEGAL_REFERENCE = 1247; public static final int ER_DERIVED_MUST_HAVE_ALIAS = 1248; public static final int ER_SELECT_REDUCED = 1249; public static final int ER_TABLENAME_NOT_ALLOWED_HERE = 1250; public static final int ER_NOT_SUPPORTED_AUTH_MODE = 1251; public static final int ER_SPATIAL_CANT_HAVE_NULL = 1252; public static final int ER_COLLATION_CHARSET_MISMATCH = 1253; public static final int ER_SLAVE_WAS_RUNNING = 1254; public static final int ER_SLAVE_WAS_NOT_RUNNING = 1255; public static final int ER_TOO_BIG_FOR_UNCOMPRESS = 1256; public static final int ER_ZLIB_Z_MEM_ERROR = 1257; public static final int ER_ZLIB_Z_BUF_ERROR = 1258; public static final int ER_ZLIB_Z_DATA_ERROR = 1259; public static final int ER_CUT_VALUE_GROUP_CONCAT = 1260; public static final int ER_WARN_TOO_FEW_RECORDS = 1261; public static final int ER_WARN_TOO_MANY_RECORDS = 1262; public static final int ER_WARN_NULL_TO_NOTNULL = 1263; public static final int ER_WARN_DATA_OUT_OF_RANGE = 1264; public static final int WARN_DATA_TRUNCATED = 1265; public static final int ER_WARN_USING_OTHER_HANDLER = 1266; public static final int ER_CANT_AGGREGATE_2COLLATIONS = 1267; public static final int ER_DROP_USER = 1268; public static final int ER_REVOKE_GRANTS = 1269; public static final int ER_CANT_AGGREGATE_3COLLATIONS = 1270; public static final int ER_CANT_AGGREGATE_NCOLLATIONS = 1271; public static final int ER_VARIABLE_IS_NOT_STRUCT = 1272; public static final int ER_UNKNOWN_COLLATION = 1273; public static final int ER_SLAVE_IGNORED_SSL_PARAMS = 1274; public static final int ER_SERVER_IS_IN_SECURE_AUTH_MODE = 1275; public static final int ER_WARN_FIELD_RESOLVED = 1276; public static final int ER_BAD_SLAVE_UNTIL_COND = 1277; public static final int ER_MISSING_SKIP_SLAVE = 1278; public static final int ER_UNTIL_COND_IGNORED = 1279; public static final int ER_WRONG_NAME_FOR_INDEX = 1280; public static final int ER_WRONG_NAME_FOR_CATALOG = 1281; public static final int ER_WARN_QC_RESIZE = 1282; public static final int ER_BAD_FT_COLUMN = 1283; public static final int ER_UNKNOWN_KEY_CACHE = 1284; public static final int ER_WARN_HOSTNAME_WONT_WORK = 1285; public static final int ER_UNKNOWN_STORAGE_ENGINE = 1286; public static final int ER_WARN_DEPRECATED_SYNTAX = 1287; public static final int ER_NON_UPDATABLE_TABLE = 1288; public static final int ER_FEATURE_DISABLED = 1289; public static final int ER_OPTION_PREVENTS_STATEMENT = 1290; public static final int ER_DUPLICATED_VALUE_IN_TYPE = 1291; public static final int ER_TRUNCATED_WRONG_VALUE = 1292; public static final int ER_TOO_MUCH_AUTO_TIMESTAMP_COLS = 1293; public static final int ER_INVALID_ON_UPDATE = 1294; public static final int ER_UNSUPPORTED_PS = 1295; public static final int ER_GET_ERRMSG = 1296; public static final int ER_GET_TEMPORARY_ERRMSG = 1297; public static final int ER_UNKNOWN_TIME_ZONE = 1298; public static final int ER_WARN_INVALID_TIMESTAMP = 1299; public static final int ER_INVALID_CHARACTER_STRING = 1300; public static final int ER_WARN_ALLOWED_PACKET_OVERFLOWED = 1301; public static final int ER_CONFLICTING_DECLARATIONS = 1302; public static final int ER_SP_NO_RECURSIVE_CREATE = 1303; public static final int ER_SP_ALREADY_EXISTS = 1304; public static final int ER_SP_DOES_NOT_EXIST = 1305; public static final int ER_SP_DROP_FAILED = 1306; public static final int ER_SP_STORE_FAILED = 1307; public static final int ER_SP_LILABEL_MISMATCH = 1308; public static final int ER_SP_LABEL_REDEFINE = 1309; public static final int ER_SP_LABEL_MISMATCH = 1310; public static final int ER_SP_UNINIT_VAR = 1311; public static final int ER_SP_BADSELECT = 1312; public static final int ER_SP_BADRETURN = 1313; public static final int ER_SP_BADSTATEMENT = 1314; public static final int ER_UPDATE_LOG_DEPRECATED_IGNORED = 1315; public static final int ER_UPDATE_LOG_DEPRECATED_TRANSLATED = 1316; public static final int ER_QUERY_INTERRUPTED = 1317; public static final int ER_SP_WRONG_NO_OF_ARGS = 1318; public static final int ER_SP_COND_MISMATCH = 1319; public static final int ER_SP_NORETURN = 1320; public static final int ER_SP_NORETURNEND = 1321; public static final int ER_SP_BAD_CURSOR_QUERY = 1322; public static final int ER_SP_BAD_CURSOR_SELECT = 1323; public static final int ER_SP_CURSOR_MISMATCH = 1324; public static final int ER_SP_CURSOR_ALREADY_OPEN = 1325; public static final int ER_SP_CURSOR_NOT_OPEN = 1326; public static final int ER_SP_UNDECLARED_VAR = 1327; public static final int ER_SP_WRONG_NO_OF_FETCH_ARGS = 1328; public static final int ER_SP_FETCH_NO_DATA = 1329; public static final int ER_SP_DUP_PARAM = 1330; public static final int ER_SP_DUP_VAR = 1331; public static final int ER_SP_DUP_COND = 1332; public static final int ER_SP_DUP_CURS = 1333; public static final int ER_SP_CANT_ALTER = 1334; public static final int ER_SP_SUBSELECT_NYI = 1335; public static final int ER_STMT_NOT_ALLOWED_IN_SF_OR_TRG = 1336; public static final int ER_SP_VARCOND_AFTER_CURSHNDLR = 1337; public static final int ER_SP_CURSOR_AFTER_HANDLER = 1338; public static final int ER_SP_CASE_NOT_FOUND = 1339; public static final int ER_FPARSER_TOO_BIG_FILE = 1340; public static final int ER_FPARSER_BAD_HEADER = 1341; public static final int ER_FPARSER_EOF_IN_COMMENT = 1342; public static final int ER_FPARSER_ERROR_IN_PARAMETER = 1343; public static final int ER_FPARSER_EOF_IN_UNKNOWN_PARAMETER = 1344; public static final int ER_VIEW_NO_EXPLAIN = 1345; public static final int ER_FRM_UNKNOWN_TYPE = 1346; public static final int ER_WRONG_OBJECT = 1347; public static final int ER_NONUPDATEABLE_COLUMN = 1348; public static final int ER_VIEW_SELECT_DERIVED = 1349; public static final int ER_VIEW_SELECT_CLAUSE = 1350; public static final int ER_VIEW_SELECT_VARIABLE = 1351; public static final int ER_VIEW_SELECT_TMPTABLE = 1352; public static final int ER_VIEW_WRONG_LIST = 1353; public static final int ER_WARN_VIEW_MERGE = 1354; public static final int ER_WARN_VIEW_WITHOUT_KEY = 1355; public static final int ER_VIEW_INVALID = 1356; public static final int ER_SP_NO_DROP_SP = 1357; public static final int ER_SP_GOTO_IN_HNDLR = 1358; public static final int ER_TRG_ALREADY_EXISTS = 1359; public static final int ER_TRG_DOES_NOT_EXIST = 1360; public static final int ER_TRG_ON_VIEW_OR_TEMP_TABLE = 1361; public static final int ER_TRG_CANT_CHANGE_ROW = 1362; public static final int ER_TRG_NO_SUCH_ROW_IN_TRG = 1363; public static final int ER_NO_DEFAULT_FOR_FIELD = 1364; public static final int ER_DIVISION_BY_ZERO = 1365; public static final int ER_TRUNCATED_WRONG_VALUE_FOR_FIELD = 1366; public static final int ER_ILLEGAL_VALUE_FOR_TYPE = 1367; public static final int ER_VIEW_NONUPD_CHECK = 1368; public static final int ER_VIEW_CHECK_FAILED = 1369; public static final int ER_PROCACCESS_DENIED_ERROR = 1370; public static final int ER_RELAY_LOG_FAIL = 1371; public static final int ER_PASSWD_LENGTH = 1372; public static final int ER_UNKNOWN_TARGET_BINLOG = 1373; public static final int ER_IO_ERR_LOG_INDEX_READ = 1374; public static final int ER_BINLOG_PURGE_PROHIBITED = 1375; public static final int ER_FSEEK_FAIL = 1376; public static final int ER_BINLOG_PURGE_FATAL_ERR = 1377; public static final int ER_LOG_IN_USE = 1378; public static final int ER_LOG_PURGE_UNKNOWN_ERR = 1379; public static final int ER_RELAY_LOG_INIT = 1380; public static final int ER_NO_BINARY_LOGGING = 1381; public static final int ER_RESERVED_SYNTAX = 1382; public static final int ER_WSAS_FAILED = 1383; public static final int ER_DIFF_GROUPS_PROC = 1384; public static final int ER_NO_GROUP_FOR_PROC = 1385; public static final int ER_ORDER_WITH_PROC = 1386; public static final int ER_LOGGING_PROHIBIT_CHANGING_OF = 1387; public static final int ER_NO_FILE_MAPPING = 1388; public static final int ER_WRONG_MAGIC = 1389; public static final int ER_PS_MANY_PARAM = 1390; public static final int ER_KEY_PART_0 = 1391; public static final int ER_VIEW_CHECKSUM = 1392; public static final int ER_VIEW_MULTIUPDATE = 1393; public static final int ER_VIEW_NO_INSERT_FIELD_LIST = 1394; public static final int ER_VIEW_DELETE_MERGE_VIEW = 1395; public static final int ER_CANNOT_USER = 1396; public static final int ER_XAER_NOTA = 1397; public static final int ER_XAER_INVAL = 1398; public static final int ER_XAER_RMFAIL = 1399; public static final int ER_XAER_OUTSIDE = 1400; public static final int ER_XAER_RMERR = 1401; public static final int ER_XA_RBROLLBACK = 1402; public static final int ER_NONEXISTING_PROC_GRANT = 1403; public static final int ER_PROC_AUTO_GRANT_FAIL = 1404; public static final int ER_PROC_AUTO_REVOKE_FAIL = 1405; public static final int ER_DATA_TOO_LONG = 1406; public static final int ER_SP_BAD_SQLSTATE = 1407; public static final int ER_STARTUP = 1408; public static final int ER_LOAD_FROM_FIXED_SIZE_ROWS_TO_VAR = 1409; public static final int ER_CANT_CREATE_USER_WITH_GRANT = 1410; public static final int ER_WRONG_VALUE_FOR_TYPE = 1411; public static final int ER_TABLE_DEF_CHANGED = 1412; public static final int ER_SP_DUP_HANDLER = 1413; public static final int ER_SP_NOT_VAR_ARG = 1414; public static final int ER_SP_NO_RETSET = 1415; public static final int ER_CANT_CREATE_GEOMETRY_OBJECT = 1416; public static final int ER_FAILED_ROUTINE_BREAK_BINLOG = 1417; public static final int ER_BINLOG_UNSAFE_ROUTINE = 1418; public static final int ER_BINLOG_CREATE_ROUTINE_NEED_SUPER = 1419; public static final int ER_EXEC_STMT_WITH_OPEN_CURSOR = 1420; public static final int ER_STMT_HAS_NO_OPEN_CURSOR = 1421; public static final int ER_COMMIT_NOT_ALLOWED_IN_SF_OR_TRG = 1422; public static final int ER_NO_DEFAULT_FOR_VIEW_FIELD = 1423; public static final int ER_SP_NO_RECURSION = 1424; public static final int ER_TOO_BIG_SCALE = 1425; public static final int ER_TOO_BIG_PRECISION = 1426; public static final int ER_M_BIGGER_THAN_D = 1427; public static final int ER_WRONG_LOCK_OF_SYSTEM_TABLE = 1428; public static final int ER_CONNECT_TO_FOREIGN_DATA_SOURCE = 1429; public static final int ER_QUERY_ON_FOREIGN_DATA_SOURCE = 1430; public static final int ER_FOREIGN_DATA_SOURCE_DOESNT_EXIST = 1431; public static final int ER_FOREIGN_DATA_STRING_INVALID_CANT_CREATE = 1432; public static final int ER_FOREIGN_DATA_STRING_INVALID = 1433; public static final int ER_CANT_CREATE_FEDERATED_TABLE = 1434; public static final int ER_TRG_IN_WRONG_SCHEMA = 1435; public static final int ER_STACK_OVERRUN_NEED_MORE = 1436; public static final int ER_TOO_LONG_BODY = 1437; public static final int ER_WARN_CANT_DROP_DEFAULT_KEYCACHE = 1438; public static final int ER_TOO_BIG_DISPLAYWIDTH = 1439; public static final int ER_XAER_DUPID = 1440; public static final int ER_DATETIME_FUNCTION_OVERFLOW = 1441; public static final int ER_CANT_UPDATE_USED_TABLE_IN_SF_OR_TRG = 1442; public static final int ER_VIEW_PREVENT_UPDATE = 1443; public static final int ER_PS_NO_RECURSION = 1444; public static final int ER_SP_CANT_SET_AUTOCOMMIT = 1445; public static final int ER_NO_VIEW_USER = 1446; public static final int ER_VIEW_FRM_NO_USER = 1447; public static final int ER_VIEW_OTHER_USER = 1448; public static final int ER_NO_SUCH_USER = 1449; public static final int ER_FORBID_SCHEMA_CHANGE = 1450; public static final int ER_ROW_IS_REFERENCED_2 = 1451; public static final int ER_NO_REFERENCED_ROW_2 = 1452; public static final int ER_SP_BAD_VAR_SHADOW = 1453; public static final int ER_PARTITION_REQUIRES_VALUES_ERROR = 1454; public static final int ER_PARTITION_WRONG_VALUES_ERROR = 1455; public static final int ER_PARTITION_MAXVALUE_ERROR = 1456; public static final int ER_PARTITION_SUBPARTITION_ERROR = 1457; public static final int ER_PARTITION_WRONG_NO_PART_ERROR = 1458; public static final int ER_PARTITION_WRONG_NO_SUBPART_ERROR = 1459; public static final int ER_CONST_EXPR_IN_PARTITION_FUNC_ERROR = 1460; public static final int ER_NO_CONST_EXPR_IN_RANGE_OR_LIST_ERROR = 1461; public static final int ER_FIELD_NOT_FOUND_PART_ERROR = 1462; public static final int ER_LIST_OF_FIELDS_ONLY_IN_HASH_ERROR = 1463; public static final int ER_INCONSISTENT_PARTITION_INFO_ERROR = 1464; public static final int ER_PARTITION_FUNC_NOT_ALLOWED_ERROR = 1465; public static final int ER_PARTITIONS_MUST_BE_DEFINED_ERROR = 1466; public static final int ER_RANGE_NOT_INCREASING_ERROR = 1467; public static final int ER_INCONSISTENT_TYPE_OF_FUNCTIONS_ERROR = 1468; public static final int ER_MULTIPLE_DEF_CONST_IN_LIST_PART_ERROR = 1469; public static final int ER_PARTITION_ENTRY_ERROR = 1470; public static final int ER_MIX_HANDLER_ERROR = 1471; public static final int ER_PARTITION_NOT_DEFINED_ERROR = 1472; public static final int ER_TOO_MANY_PARTITIONS_ERROR = 1473; public static final int ER_SUBPARTITION_ERROR = 1474; public static final int ER_CANT_CREATE_HANDLER_FILE = 1475; public static final int ER_BLOB_FIELD_IN_PART_FUNC_ERROR = 1476; public static final int ER_CHAR_SET_IN_PART_FIELD_ERROR = 1477; public static final int ER_UNIQUE_KEY_NEED_ALL_FIELDS_IN_PF = 1478; public static final int ER_NO_PARTS_ERROR = 1479; public static final int ER_PARTITION_MGMT_ON_NONPARTITIONED = 1480; public static final int ER_DROP_PARTITION_NON_EXISTENT = 1481; public static final int ER_DROP_LAST_PARTITION = 1482; public static final int ER_COALESCE_ONLY_ON_HASH_PARTITION = 1483; public static final int ER_ONLY_ON_RANGE_LIST_PARTITION = 1484; public static final int ER_ADD_PARTITION_SUBPART_ERROR = 1485; public static final int ER_ADD_PARTITION_NO_NEW_PARTITION = 1486; public static final int ER_COALESCE_PARTITION_NO_PARTITION = 1487; public static final int ER_REORG_PARTITION_NOT_EXIST = 1488; public static final int ER_SAME_NAME_PARTITION = 1489; public static final int ER_CONSECUTIVE_REORG_PARTITIONS = 1490; public static final int ER_REORG_OUTSIDE_RANGE = 1491; public static final int ER_DROP_PARTITION_FAILURE = 1492; public static final int ER_DROP_PARTITION_WHEN_FK_DEFINED = 1493; public static final int ER_PLUGIN_IS_NOT_LOADED = 1494; public static final int ER_USER_READ_ONLY = 1495; } ================================================ FILE: src/main/java/io/mycat/config/Fields.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config; /** * 字段类型及标识定义 * * @author mycat */ public interface Fields { /** field data type */ public static final int FIELD_TYPE_DECIMAL = 0; public static final int FIELD_TYPE_TINY = 1; public static final int FIELD_TYPE_SHORT = 2; public static final int FIELD_TYPE_LONG = 3; public static final int FIELD_TYPE_FLOAT = 4; public static final int FIELD_TYPE_DOUBLE = 5; public static final int FIELD_TYPE_NULL = 6; public static final int FIELD_TYPE_TIMESTAMP = 7; public static final int FIELD_TYPE_LONGLONG = 8; public static final int FIELD_TYPE_INT24 = 9; public static final int FIELD_TYPE_DATE = 10; public static final int FIELD_TYPE_TIME = 11; public static final int FIELD_TYPE_DATETIME = 12; public static final int FIELD_TYPE_YEAR = 13; public static final int FIELD_TYPE_NEWDATE = 14; public static final int FIELD_TYPE_VARCHAR = 15; public static final int FIELD_TYPE_BIT = 16; public static final int FIELD_TYPE_NEW_DECIMAL = 246; public static final int FIELD_TYPE_ENUM = 247; public static final int FIELD_TYPE_SET = 248; public static final int FIELD_TYPE_TINY_BLOB = 249; public static final int FIELD_TYPE_MEDIUM_BLOB = 250; public static final int FIELD_TYPE_LONG_BLOB = 251; public static final int FIELD_TYPE_BLOB = 252; public static final int FIELD_TYPE_VAR_STRING = 253; public static final int FIELD_TYPE_STRING = 254; public static final int FIELD_TYPE_GEOMETRY = 255; /** field flag */ public static final int NOT_NULL_FLAG = 0x0001; public static final int PRI_KEY_FLAG = 0x0002; public static final int UNIQUE_KEY_FLAG = 0x0004; public static final int MULTIPLE_KEY_FLAG = 0x0008; public static final int BLOB_FLAG = 0x0010; public static final int UNSIGNED_FLAG = 0x0020; public static final int ZEROFILL_FLAG = 0x0040; public static final int BINARY_FLAG = 0x0080; public static final int ENUM_FLAG = 0x0100; public static final int AUTO_INCREMENT_FLAG = 0x0200; public static final int TIMESTAMP_FLAG = 0x0400; public static final int SET_FLAG = 0x0800; } ================================================ FILE: src/main/java/io/mycat/config/Isolations.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config; /** * 事务隔离级别定义 * * @author mycat */ public interface Isolations { public static final int READ_UNCOMMITTED = 1; public static final int READ_COMMITTED = 2; public static final int REPEATED_READ = 3; public static final int SERIALIZABLE = 4; } ================================================ FILE: src/main/java/io/mycat/config/MycatCluster.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config; import java.util.HashMap; import java.util.List; import java.util.Map; import io.mycat.config.model.ClusterConfig; import io.mycat.config.model.MycatNodeConfig; /** * @author mycat */ public final class MycatCluster { private final Map nodes; private final Map> groups; public MycatCluster(ClusterConfig clusterConf) { this.nodes = new HashMap(clusterConf.getNodes().size()); this.groups = clusterConf.getGroups(); for (MycatNodeConfig conf : clusterConf.getNodes().values()) { String name = conf.getName(); MycatNode node = new MycatNode(conf); this.nodes.put(name, node); } } public Map getNodes() { return nodes; } public Map> getGroups() { return groups; } } ================================================ FILE: src/main/java/io/mycat/config/MycatConfig.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config; import java.io.IOException; import java.net.StandardSocketOptions; import java.nio.channels.NetworkChannel; import java.util.ArrayList; import java.util.Map; import java.util.concurrent.locks.ReentrantLock; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.config.model.FirewallConfig; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.config.model.UserConfig; import io.mycat.net.AbstractConnection; import io.mycat.util.TimeUtil; /** * @author mycat */ public class MycatConfig { private static final int RELOAD = 1; private static final int ROLLBACK = 2; private static final int RELOAD_ALL = 3; private volatile SystemConfig system; private volatile MycatCluster cluster; private volatile MycatCluster _cluster; private volatile FirewallConfig firewall; private volatile FirewallConfig _firewall; private volatile Map users; private volatile Map _users; private volatile Map schemas; private volatile Map _schemas; private volatile Map dataNodes; private volatile Map _dataNodes; private volatile Map dataHosts; private volatile Map _dataHosts; private long reloadTime; private long rollbackTime; private int status; private final ReentrantLock lock; public MycatConfig() { //读取schema.xml，rule.xml和server.xml ConfigInitializer confInit = new ConfigInitializer(true); this.system = confInit.getSystem(); this.users = confInit.getUsers(); this.schemas = confInit.getSchemas(); this.dataHosts = confInit.getDataHosts(); this.dataNodes = confInit.getDataNodes(); for (PhysicalDBPool dbPool : dataHosts.values()) { dbPool.setSchemas(getDataNodeSchemasOfDataHost(dbPool.getHostName())); } this.firewall = confInit.getFirewall(); this.cluster = confInit.getCluster(); //初始化重加载配置时间 this.reloadTime = TimeUtil.currentTimeMillis(); this.rollbackTime = -1L; this.status = RELOAD; //配置加载锁 this.lock = new ReentrantLock(); } public SystemConfig getSystem() { return system; } public void setSocketParams(AbstractConnection con, boolean isFrontChannel) throws IOException { int sorcvbuf = 0; int sosndbuf = 0; int soNoDelay = 0; if ( isFrontChannel ) { sorcvbuf = system.getFrontsocketsorcvbuf(); sosndbuf = system.getFrontsocketsosndbuf(); soNoDelay = system.getFrontSocketNoDelay(); } else { sorcvbuf = system.getBacksocketsorcvbuf(); sosndbuf = system.getBacksocketsosndbuf(); soNoDelay = system.getBackSocketNoDelay(); } NetworkChannel channel = con.getChannel(); channel.setOption(StandardSocketOptions.SO_RCVBUF, sorcvbuf); channel.setOption(StandardSocketOptions.SO_SNDBUF, sosndbuf); channel.setOption(StandardSocketOptions.TCP_NODELAY, soNoDelay == 1); channel.setOption(StandardSocketOptions.SO_REUSEADDR, true); channel.setOption(StandardSocketOptions.SO_KEEPALIVE, true); con.setMaxPacketSize(system.getMaxPacketSize()); con.setPacketHeaderSize(system.getPacketHeaderSize()); con.setIdleTimeout(system.getIdleTimeout()); con.setCharset(system.getCharset()); } public Map getUsers() { return users; } public Map getBackupUsers() { return _users; } public Map getSchemas() { return schemas; } public Map getBackupSchemas() { return _schemas; } public Map getDataNodes() { return dataNodes; } public void setDataNodes( Map map) { this.dataNodes = map; } public String[] getDataNodeSchemasOfDataHost(String dataHost) { ArrayList schemas = new ArrayList(30); for (PhysicalDBNode dn: dataNodes.values()) { if (dn.getDbPool().getHostName().equals(dataHost)) { schemas.add(dn.getDatabase()); } } return schemas.toArray(new String[schemas.size()]); } public Map getBackupDataNodes() { return _dataNodes; } public Map getDataHosts() { return dataHosts; } public Map getBackupDataHosts() { return _dataHosts; } public MycatCluster getCluster() { return cluster; } public MycatCluster getBackupCluster() { return _cluster; } public FirewallConfig getFirewall() { return firewall; } public FirewallConfig getBackupFirewall() { return _firewall; } public ReentrantLock getLock() { return lock; } public long getReloadTime() { return reloadTime; } public long getRollbackTime() { return rollbackTime; } public void reload( Map newUsers, Map newSchemas, Map newDataNodes, Map newDataHosts, MycatCluster newCluster, FirewallConfig newFirewall, boolean reloadAll) { apply(newUsers, newSchemas, newDataNodes, newDataHosts, newCluster, newFirewall, reloadAll); this.reloadTime = TimeUtil.currentTimeMillis(); this.status = reloadAll?RELOAD_ALL:RELOAD; } public boolean canRollback() { if (_users == null || _schemas == null || _dataNodes == null || _dataHosts == null || _cluster == null || _firewall == null || status == ROLLBACK) { return false; } else { return true; } } public void rollback( Map users, Map schemas, Map dataNodes, Map dataHosts, MycatCluster cluster, FirewallConfig firewall) { apply(users, schemas, dataNodes, dataHosts, cluster, firewall, status==RELOAD_ALL); this.rollbackTime = TimeUtil.currentTimeMillis(); this.status = ROLLBACK; } private void apply(Map newUsers, Map newSchemas, Map newDataNodes, Map newDataHosts, MycatCluster newCluster, FirewallConfig newFirewall, boolean isLoadAll) { final ReentrantLock lock = this.lock; lock.lock(); try { // old 处理 // 1、停止老的数据源心跳 // 2、备份老的数据源配置 //-------------------------------------------- if (isLoadAll) { Map oldDataHosts = this.dataHosts; if (oldDataHosts != null) { for (PhysicalDBPool oldDbPool : oldDataHosts.values()) { if (oldDbPool != null) { oldDbPool.stopHeartbeat(); } } } this._dataNodes = this.dataNodes; this._dataHosts = this.dataHosts; } this._users = this.users; this._schemas = this.schemas; this._cluster = this.cluster; this._firewall = this.firewall; // new 处理 // 1、启动新的数据源心跳 // 2、执行新的配置 //--------------------------------------------------- if (isLoadAll) { if (newDataHosts != null) { for (PhysicalDBPool newDbPool : newDataHosts.values()) { if ( newDbPool != null) { newDbPool.startHeartbeat(); } } } this.dataNodes = newDataNodes; this.dataHosts = newDataHosts; } this.users = newUsers; this.schemas = newSchemas; this.cluster = newCluster; this.firewall = newFirewall; } finally { lock.unlock(); } } } ================================================ FILE: src/main/java/io/mycat/config/MycatNode.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.config.model.MycatNodeConfig; /** * @author mycat */ public class MycatNode { private static final Logger LOGGER = LoggerFactory.getLogger(MycatNode.class); private final String name; private final MycatNodeConfig config; public MycatNode(MycatNodeConfig config) { this.name = config.getName(); this.config = config; } public String getName() { return name; } public MycatNodeConfig getConfig() { return config; } public boolean isOnline() { return (true); } } ================================================ FILE: src/main/java/io/mycat/config/MycatPrivileges.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config; import java.util.List; import java.util.Map; import java.util.Set; import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.ast.statement.SQLDeleteStatement; import com.alibaba.druid.sql.ast.statement.SQLInsertStatement; import com.alibaba.druid.sql.ast.statement.SQLReplaceStatement; import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; import com.alibaba.druid.sql.ast.statement.SQLUpdateStatement; import com.alibaba.druid.sql.parser.SQLStatementParser; import com.alibaba.druid.wall.WallCheckResult; import com.alibaba.druid.wall.WallProvider; import io.mycat.MycatServer; import io.mycat.config.loader.xml.XMLServerLoader; import io.mycat.config.model.FirewallConfig; import io.mycat.config.model.UserConfig; import io.mycat.config.model.UserPrivilegesConfig; import io.mycat.net.handler.FrontendPrivileges; import io.mycat.route.parser.druid.MycatSchemaStatVisitor; import io.mycat.route.parser.druid.MycatStatementParser; /** * @author mycat */ public class MycatPrivileges implements FrontendPrivileges { /** * 无需每次建立连接都new实例。 */ private static MycatPrivileges instance = new MycatPrivileges(); private static final Logger ALARM = LoggerFactory.getLogger("alarm"); private static boolean check = false; private final static ThreadLocal contextLocal = new ThreadLocal(); public static MycatPrivileges instance() { return instance; } private MycatPrivileges() { super(); } @Override public boolean schemaExists(String schema) { MycatConfig conf = MycatServer.getInstance().getConfig(); return conf.getSchemas().containsKey(schema); } @Override public boolean userExists(String user, String host) { //检查用户及白名单 return checkFirewallWhiteHostPolicy(user, host); } @Override public String getPassword(String user) { MycatConfig conf = MycatServer.getInstance().getConfig(); if (user != null && user.equals(conf.getSystem().getClusterHeartbeatUser())) { return conf.getSystem().getClusterHeartbeatPass(); } else { UserConfig uc = conf.getUsers().get(user); if (uc != null) { return uc.getPassword(); } else { return null; } } } @Override public Set getUserSchemas(String user) { MycatConfig conf = MycatServer.getInstance().getConfig(); UserConfig uc = conf.getUsers().get(user); if (uc != null) { return uc.getSchemas(); } else { return null; } } @Override public Boolean isReadOnly(String user) { MycatConfig conf = MycatServer.getInstance().getConfig(); UserConfig uc = conf.getUsers().get(user); if (uc != null) { return uc.isReadOnly(); } else { return null; } } @Override public int getBenchmark(String user) { MycatConfig conf = MycatServer.getInstance().getConfig(); UserConfig uc = conf.getUsers().get(user); if (uc != null) { return uc.getBenchmark(); } else { return 0; } } /** * 防火墙白名单处理，根据防火墙配置，判断目前主机是否可以通过某用户登陆 * 白名单配置请参考： * @see XMLServerLoader * @see FirewallConfig * * @modification 修改增加网段白名单识别配置 * @date 2016/12/8 * @modifiedBy Hash Zhang */ @Override public boolean checkFirewallWhiteHostPolicy(String user, String host) { MycatConfig mycatConfig = MycatServer.getInstance().getConfig(); FirewallConfig firewallConfig = mycatConfig.getFirewall(); //防火墙 白名单处理 boolean isPassed = false; Map> whitehost = firewallConfig.getWhitehost(); Map> whitehostMask = firewallConfig.getWhitehostMask(); if ((whitehost == null || whitehost.size() == 0)&&(whitehostMask == null || whitehostMask.size() == 0)) { Map users = mycatConfig.getUsers(); isPassed = users.containsKey(user); } else { List list = whitehost.get(host); Set patterns = whitehostMask.keySet(); if(patterns != null && patterns.size() > 0){ for(Pattern pattern : patterns) { if(pattern.matcher(host).find()){ isPassed = true; break; } } } if (list != null) { for (UserConfig userConfig : list) { if (userConfig.getName().equals(user)) { isPassed = true; break; } } } } if ( !isPassed ) { ALARM.error(new StringBuilder().append(Alarms.FIREWALL_ATTACK).append("[host=").append(host) .append(",user=").append(user).append(']').toString()); return false; } return true; } /** * @see https://github.com/alibaba/druid/wiki/%E9%85%8D%E7%BD%AE-wallfilter */ @Override public boolean checkFirewallSQLPolicy(String user, String sql) { boolean isPassed = true; if( contextLocal.get() == null ){ FirewallConfig firewallConfig = MycatServer.getInstance().getConfig().getFirewall(); if ( firewallConfig != null) { if ( firewallConfig.isCheck() ) { contextLocal.set(firewallConfig.getProvider()); check = true; } } } if( check ){ WallCheckResult result = contextLocal.get().check(sql); // 修复 druid 防火墙在处理SHOW FULL TABLES WHERE Table_type != 'VIEW' 的时候存在的 BUG // 此代码有问题，由于Druid WallCheck 对同一条SQL语句只做一次解析，下面代码会导致第二次拦截失效 // 并且 目前已经提供 ShowFullTables 来处理show full tables 命令，故对代码进行修改 // List stmts = result.getStatementList(); // if ( !stmts.isEmpty() && !( stmts.get(0) instanceof SQLShowTablesStatement) ) { // if ( !result.getViolations().isEmpty()) { // isPassed = false; // ALARM.warn("Firewall to intercept the '" + user + "' unsafe SQL , errMsg:" // + result.getViolations().get(0).getMessage() + // " \r\n " + sql); // } // } if ( !result.getViolations().isEmpty()) { isPassed = false; ALARM.warn("Firewall to intercept the '" + user + "' unsafe SQL , errMsg:" + result.getViolations().get(0).getMessage() + " \r\n " + sql); } } return isPassed; } // 审计SQL权限 @Override public boolean checkDmlPrivilege(String user, String schema, String sql) { if ( schema == null ) { return true; } boolean isPassed = false; MycatConfig conf = MycatServer.getInstance().getConfig(); UserConfig userConfig = conf.getUsers().get(user); if (userConfig != null) { UserPrivilegesConfig userPrivilege = userConfig.getPrivilegesConfig(); if ( userPrivilege != null && userPrivilege.isCheck() ) { UserPrivilegesConfig.SchemaPrivilege schemaPrivilege = userPrivilege.getSchemaPrivilege( schema ); if ( schemaPrivilege != null ) { String tableName = null; int index = -1; //TODO 此处待优化，寻找更优SQL 解析器 //修复bug // https://github.com/alibaba/druid/issues/1309 //com.alibaba.druid.sql.parser.ParserException: syntax error, error in :'begin',expect END, actual EOF begin if ( sql != null && sql.length() == 5 && sql.equalsIgnoreCase("begin") ) { return true; } SQLStatementParser parser = new MycatStatementParser(sql); SQLStatement stmt = parser.parseStatement(); if (stmt instanceof SQLReplaceStatement || stmt instanceof SQLInsertStatement) { index = 0; } else if (stmt instanceof SQLUpdateStatement ) { index = 1; } else if (stmt instanceof SQLSelectStatement ) { index = 2; } else if (stmt instanceof SQLDeleteStatement ) { index = 3; } if ( index > -1) { MycatSchemaStatVisitor schemaStatVisitor = new MycatSchemaStatVisitor(); stmt.accept(schemaStatVisitor); String key = schemaStatVisitor.getCurrentTable(); if ( key != null ) { if (key.contains("`")) { key = key.replaceAll("`", ""); } int dotIndex = key.indexOf("."); if (dotIndex > 0) { tableName = key.substring(dotIndex + 1); } else { tableName = key; } //获取table 权限, 此处不需要检测空值, 无设置则自动继承父级权限 UserPrivilegesConfig.TablePrivilege tablePrivilege = schemaPrivilege.getTablePrivilege( tableName ); if ( tablePrivilege.getDml()[index] > 0 ) { isPassed = true; } } else { //skip isPassed = true; } } else { //skip isPassed = true; } } else { //skip isPassed = true; } } else { //skip isPassed = true; } } else { //skip isPassed = true; } if( !isPassed ) { ALARM.error(new StringBuilder().append(Alarms.DML_ATTACK ).append("[sql=").append( sql ) .append(",user=").append(user).append(']').toString()); } return isPassed; } @Override public boolean checkDataNodeDmlPrivilege(String user, String dataNode, String sql) { if (dataNode == null) { return true; } boolean isPassed = false; MycatConfig conf = MycatServer.getInstance().getConfig(); UserConfig userConfig = conf.getUsers().get(user); if (userConfig != null) { UserPrivilegesConfig userPrivilege = userConfig.getPrivilegesConfig(); if (userPrivilege != null && userPrivilege.isCheck()) { UserPrivilegesConfig.DataNodePrivilege dataNodePrivilege = userPrivilege.getDataNodePrivilege(dataNode); if (dataNodePrivilege != null) { if (sql != null && sql.length() == 5 && sql.equalsIgnoreCase("begin")) { return true; } //获取 dataNode 的 select 权限, 此处不需要检测空值, 无设置则自动继承父级权限 if (dataNodePrivilege.getDml()[2] > 0) { isPassed = true; } } } else { return true; } } return isPassed; } } ================================================ FILE: src/main/java/io/mycat/config/Versions.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config; /** * @author mycat */ public abstract class Versions { /**协议版本**/ public static final byte PROTOCOL_VERSION = 10; /**服务器版本**/ public static byte[] SERVER_VERSION = "5.6.29-mycat-1.6.7.5-release-20200428154739".getBytes(); public static void setServerVersion(String version) { byte[] mysqlVersionPart = version.getBytes(); int startIndex; for (startIndex = 0; startIndex < SERVER_VERSION.length; startIndex++) { if (SERVER_VERSION[startIndex] == '-') break; } // 重新拼接mycat version字节数组 byte[] newMycatVersion = new byte[mysqlVersionPart.length + SERVER_VERSION.length - startIndex]; System.arraycopy(mysqlVersionPart, 0, newMycatVersion, 0, mysqlVersionPart.length); System.arraycopy(SERVER_VERSION, startIndex, newMycatVersion, mysqlVersionPart.length, SERVER_VERSION.length - startIndex); SERVER_VERSION = newMycatVersion; } } ================================================ FILE: src/main/java/io/mycat/config/Versions.template ================================================ /* * Copyright (c) 2013, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config; /** * @author mycat */ public abstract class Versions { /**协议版本**/ public static final byte PROTOCOL_VERSION = 10; /**服务器版本**/ public static byte[] SERVER_VERSION = "@server-version@".getBytes(); public static void setServerVersion(String version) { byte[] mysqlVersionPart = version.getBytes(); int startIndex; for (startIndex = 0; startIndex < SERVER_VERSION.length; startIndex++) { if (SERVER_VERSION[startIndex] == '-') break; } // 重新拼接mycat version字节数组 byte[] newMycatVersion = new byte[mysqlVersionPart.length + SERVER_VERSION.length - startIndex]; System.arraycopy(mysqlVersionPart, 0, newMycatVersion, 0, mysqlVersionPart.length); System.arraycopy(SERVER_VERSION, startIndex, newMycatVersion, mysqlVersionPart.length, SERVER_VERSION.length - startIndex); SERVER_VERSION = newMycatVersion; } } ================================================ FILE: src/main/java/io/mycat/config/classloader/DynaClassLoader.java ================================================ package io.mycat.config.classloader; import java.io.BufferedInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.util.Iterator; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * used for mycat's catlet class loader ,catlet's class file is stored in * Mycat_home/catlet dir * * @author wuzhih * */ public class DynaClassLoader { private static final Logger LOGGER = LoggerFactory.getLogger("DynaClassLoader"); /** key- class full name */ private static Map loadedDynaClassMap = new ConcurrentHashMap(); private final String extClassHome; private final MyDynaClassLoader myClassLoader; private final long classCheckMilis; public DynaClassLoader(String extClassHome, int classCheckSeconds) { super(); this.extClassHome = extClassHome; classCheckMilis = classCheckSeconds * 1000L; myClassLoader = new MyDynaClassLoader(); LOGGER.info("dyna class load from " + extClassHome + ",and auto check for class file modified every " + classCheckSeconds + " seconds"); } public Object getInstanceofClass(String className) throws Exception { DynaClass dynaClass = loadedDynaClassMap.get(className); boolean needReload = (dynaClass == null || (dynaClass .needReloadClass(classCheckMilis) && checkChanged(dynaClass))); Class newClass = null; if (needReload) { newClass = myClassLoader.loadClass(className); dynaClass = loadedDynaClassMap.get(className); } else { newClass = dynaClass.realClass; } if (dynaClass != null) { Object val = dynaClass.classObj; if (val == null) { val = dynaClass.realClass.newInstance(); dynaClass.classObj = val; } return val; } else { return newClass.newInstance(); } } /** * 加载某个类的字节码 * * @param c * @return * @throws IOException */ private static byte[] loadFile(String path) throws IOException { BufferedInputStream in = null; try { in = new BufferedInputStream(new FileInputStream(path)); byte[] readed = new byte[1024 * 4]; ByteArrayOutputStream out = new ByteArrayOutputStream(); int count = 0; while ((count = in.read(readed)) != -1) { out.write(readed, 0, count); } return out.toByteArray(); } finally { if (in != null) { in.close(); } } } private boolean checkChanged(DynaClass dynaClass) throws IOException { boolean isChanged = false; File f = new File(dynaClass.filePath); if (f.exists()) { long newTime = f.lastModified(); long oldTime = dynaClass.lastModified; if (oldTime != newTime) { // need reload dynaClass.lastModified = newTime; dynaClass.classObj = null; dynaClass.realClass = null; isChanged = true; } } return isChanged; } class MyDynaClassLoader extends ClassLoader { public MyDynaClassLoader() { } public MyDynaClassLoader(ClassLoader parentLoader) { super(parentLoader); } /** * 加载某个类 * * @param c * @return * @throws ClassNotFoundException * @throws IOException */ public Class loadClass(String name) throws ClassNotFoundException { if (name.startsWith("java") || name.startsWith("sun") || name.startsWith("io.mycat")) { return super.loadClass(name); } DynaClass dynaClass = loadedDynaClassMap.get(name); if (dynaClass != null) { if (dynaClass.realClass != null) { return dynaClass.realClass; } } else { try { dynaClass = searchFile(extClassHome, name); } catch (Exception e) { LOGGER.error("SearchFileError", e); } } if (dynaClass == null) { return super.loadClass(name); } else { LOGGER.info("load class from file "+dynaClass.filePath); Class cNew = null; if (dynaClass.isJar) { cNew =dynaClass.realClass; } else { byte[] content; try { content = loadFile(dynaClass.filePath); } catch (IOException e) { throw new ClassNotFoundException(e.toString()); } cNew = super.defineClass(name, content, 0,content.length); dynaClass.realClass = cNew; } dynaClass.classObj = null; loadedDynaClassMap.put(name, dynaClass); return cNew; } } private DynaClass searchFile(String classpath, String fileName) throws Exception { DynaClass dynCls = null; String path = fileName.replace('.', File.separatorChar) + ".class"; System.out.println("class " + classpath + " file " + path); File f = new File(classpath, path); if (f.isFile()) { String theName = f.getPath(); System.out.println("found " + theName); dynCls = new DynaClass(f.getPath()); dynCls.lastModified = f.lastModified(); return dynCls; } else { path = fileName.replace('.', File.separatorChar) + ".jar"; //classpath="D:\\code\\mycat\\Mycat-Server\\catlet\\"; System.out.println("jar " + classpath + " file " + path); f = new File(classpath, path); if (f.isFile()) { try { dynCls = new DynaClass(f.getPath()); dynCls.lastModified = f.lastModified(); dynCls.realClass=JarLoader.loadJar(classpath+"/"+path,fileName); dynCls.isJar=true; return dynCls; } catch(Exception err) { return null; } } return null; } } } public void clearUnUsedClass() { long deadTime = System.currentTimeMillis() - 30 * 60 * 1000L; Iterator> itor = loadedDynaClassMap .entrySet().iterator(); while (itor.hasNext()) { Map.Entry entry = itor.next(); DynaClass dyCls = entry.getValue(); if (dyCls.lastModified < deadTime) { LOGGER.info("clear unused catlet " + entry.getKey()); dyCls.clear(); itor.remove(); } } } } class DynaClass { public final String filePath; public volatile long lastModified; public Class realClass; public Object classObj; public boolean isJar=false; public boolean needReloadClass(long classCheckMilis) { if (lastModified + classCheckMilis < System.currentTimeMillis()) { return true; } else { return false; } } public void clear() { this.realClass = null; this.classObj = null; } public DynaClass(String filePath) { super(); this.filePath = filePath; } } ================================================ FILE: src/main/java/io/mycat/config/classloader/JarLoader.java ================================================ package io.mycat.config.classloader; import java.util.jar.*; import java.lang.reflect.*; import java.net.URL; import java.net.URLClassLoader; import java.io.*; import java.util.*; public class JarLoader { /** Unpack a jar file into a directory. */ public static void unJar(File jarFile, File toDir) throws IOException { JarFile jar = new JarFile(jarFile); try { Enumeration entries = jar.entries(); while (entries.hasMoreElements()) { JarEntry entry = (JarEntry)entries.nextElement(); if (!entry.isDirectory()) { InputStream in = jar.getInputStream(entry); try { File file = new File(toDir, entry.getName()); if (!file.getParentFile().mkdirs() && !file.getParentFile().isDirectory()) { throw new IOException("Mkdirs failed to create " + file.getParentFile().toString()); } OutputStream out = new FileOutputStream(file); try { byte[] buffer = new byte[8192]; int i; while ((i = in.read(buffer)) != -1) { out.write(buffer, 0, i); } } finally { out.close(); } } finally { in.close(); } } } } finally { jar.close(); } } public static Class loadJar(String fileName,String mainJavaclass) throws Exception { File file = new File(fileName); String mainClassName = null; JarFile jarFile; try { jarFile = new JarFile(fileName); } catch(IOException io) { throw new IOException("Error opening jar: " + fileName); } Manifest manifest = jarFile.getManifest(); if (manifest != null) { mainClassName = manifest.getMainAttributes().getValue("Main-Class"); } jarFile.close(); if (mainClassName == null) { mainClassName = mainJavaclass; } mainClassName = mainClassName.replaceAll("/", "."); File tmpDir = new File(System.getProperty("java.io.tmpdir")); tmpDir.mkdirs(); if (!tmpDir.isDirectory()) { System.out.println("Mkdirs failed to create " + tmpDir); } final File workDir = File.createTempFile("unjar", "", tmpDir); workDir.delete(); workDir.mkdirs(); if (!workDir.isDirectory()) { System.out.println("Mkdirs failed to create " + workDir); } Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { try { fullyDelete(workDir); } catch (IOException e) { } } }); unJar(file, workDir); ArrayList classPath = new ArrayList(); classPath.add(new File(workDir+"/").toURL()); classPath.add(file.toURL()); classPath.add(new File(workDir, "classes/").toURL()); File[] libs = new File(workDir, "lib").listFiles(); if (libs != null) { for (int i = 0; i < libs.length; i++) { classPath.add(libs[i].toURL()); } } ClassLoader loader = new URLClassLoader(classPath.toArray(new URL[0])); Thread.currentThread().setContextClassLoader(loader); Class mainClass = Class.forName(mainClassName, true, loader); return mainClass; } public static boolean fullyDelete(File dir) throws IOException { if (!fullyDeleteContents(dir)) { return false; } return dir.delete(); } /** * Delete the contents of a directory, not the directory itself. If * we return false, the directory may be partially-deleted. */ public static boolean fullyDeleteContents(File dir) throws IOException { boolean deletionSucceeded = true; File contents[] = dir.listFiles(); if (contents != null) { for (int i = 0; i < contents.length; i++) { if (contents[i].isFile()) { if (!contents[i].delete()) { deletionSucceeded = false; continue; // continue deletion of other files/dirs under dir } } else { //try deleting the directory // this might be a symlink boolean b = false; b = contents[i].delete(); if (b){ //this was indeed a symlink or an empty directory continue; } // if not an empty directory or symlink let // fullydelete handle it. if (!fullyDelete(contents[i])) { deletionSucceeded = false; continue; // continue deletion of other files/dirs under dir } } } } return deletionSucceeded; } } ================================================ FILE: src/main/java/io/mycat/config/loader/ConfigLoader.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.loader; import java.util.Map; import io.mycat.config.model.ClusterConfig; import io.mycat.config.model.DataHostConfig; import io.mycat.config.model.DataNodeConfig; import io.mycat.config.model.FirewallConfig; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.config.model.UserConfig; /** * @author mycat */ public interface ConfigLoader { SchemaConfig getSchemaConfig(String schema); Map getSchemaConfigs(); Map getDataNodes(); Map getDataHosts(); SystemConfig getSystemConfig(); UserConfig getUserConfig(String user); Map getUserConfigs(); FirewallConfig getFirewallConfig(); ClusterConfig getClusterConfig(); } ================================================ FILE: src/main/java/io/mycat/config/loader/SchemaLoader.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.loader; import java.util.Map; import io.mycat.config.model.DataHostConfig; import io.mycat.config.model.DataNodeConfig; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.rule.TableRuleConfig; /** * @author mycat */ public interface SchemaLoader { Map getTableRules(); Map getDataHosts(); Map getDataNodes(); Map getSchemas(); } ================================================ FILE: src/main/java/io/mycat/config/loader/console/ZookeeperPath.java ================================================ package io.mycat.config.loader.console; /** * 专门用来操作zookeeper路径的文件信息 * 源文件名：ZkPath.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public enum ZookeeperPath { /** * zk的路径分隔符 * @字段说明 ZK_SEPARATOR */ ZK_SEPARATOR("/"), /** * 最基础的mycat节点 * @字段说明 FLOW_ZK_PATH_LINE */ FLOW_ZK_PATH_BASE("mycat"), /** * 在当前在线的节点 * @字段说明 FLOW_ZK_PATH_LINE */ FLOW_ZK_PATH_LINE("line"), /** * schema父路径 * @字段说明 FOW_ZK_PATH_SCHEMA */ FOW_ZK_PATH_SCHEMA("schema"), /** * 配制schema信息 * @字段说明 FLOW_ZK_PATH_SCHEMA */ FLOW_ZK_PATH_SCHEMA_SCHEMA("schema"), /** * 对应数据库信息 * @字段说明 FLOW_ZK_PATH_SCHEMA_DATANODE */ FLOW_ZK_PATH_SCHEMA_DATANODE("dataNode"), /** * 数据库信息dataHost * @字段说明 FLOW_ZK_PATH_SCHEMA_DATANODE */ FLOW_ZK_PATH_SCHEMA_DATAHOST("dataHost"), /** * 路由信息 * @字段说明 FLOW_ZK_PATH_SCHEMA_DATANODE */ FLOW_ZK_PATH_RULE("rules"), /** * 路由信息 * @字段说明 FLOW_ZK_PATH_SCHEMA_DATANODE */ FLOW_ZK_PATH_RULE_TABLERULE("tableRule"), /** * 路由信息 * @字段说明 FLOW_ZK_PATH_SCHEMA_DATANODE */ FLOW_ZK_PATH_RULE_FUNCTION("function"), /** * 服务端配制路径 * @字段说明 FLOW_ZK_PATH_SERVER */ FLOW_ZK_PATH_SERVER("server"), /** * 默认配制信息 * @字段说明 FLOW_ZK_PATH_SERVER_DEFAULT */ FLOW_ZK_PATH_SERVER_DEFAULT("default"), /** * 针对集群的配制信息 * @字段说明 FLOW_ZK_PATH_SERVER_DEFAULT */ FLOW_ZK_PATH_SERVER_CLUSTER("cluster"), /** * 配制的用户信息 * @字段说明 FLOW_ZK_PATH_SERVER_DEFAULT */ FLOW_ZK_PATH_SERVER_USER("user"), /** * 配制的防火墙信息,如黑白名单信息 * @字段说明 FLOW_ZK_PATH_SERVER_DEFAULT */ FLOW_ZK_PATH_SERVER_FIREWALL("firewall"), /** * 表的权限信息 * @字段说明 FLOW_ZK_PATH_SERVER_AUTH */ FLOW_ZK_PATH_SERVER_AUTH("auth"), /** * 序列信息 * @字段说明 FLOW_ZK_PATH_SERVER_AUTH */ FLOW_ZK_PATH_SEQUENCE("sequences"), /** * 序列信息中公共配制信息 * @字段说明 FLOW_ZK_PATH_SERVER_AUTH */ FLOW_ZK_PATH_SEQUENCE_COMMON("common"), /** * 用来存放序列值的信息 * @字段说明 FLOW_ZK_PATH_SERVER_AUTH */ FLOW_ZK_PATH_SEQUENCE_INSTANCE("instance"), /** * 用来存放序列值的 * @字段说明 FLOW_ZK_PATH_SERVER_AUTH */ FLOW_ZK_PATH_SEQUENCE_LEADER("leader"), /** * 递增序列号 * @字段说明 FLOW_ZK_PATH_SERVER_AUTH */ FLOW_ZK_PATH_SEQUENCE_INCREMENT_SEQ("incr_sequence"), /** * 序列信息中需要单独节点配制的信息 * @字段说明 FLOW_ZK_PATH_SERVER_AUTH */ FLOW_ZK_PATH_SEQUENCE_CLUSTER("cluster"), /** * 缓存信息 * @字段说明 FLOW_ZK_PATH_CACHE */ FLOW_ZK_PATH_CACHE("cache"), /** * 配制切换及状态目录信息 * @字段说明 FLOW_ZK_PATH_BINDATA */ FLOW_ZK_PATH_BINDATA("bindata"), /** * 配制切换及状态目录信息 * @字段说明 FLOW_ZK_PATH_RULEDATA */ FLOW_ZK_PATH_RULEATA("ruledata"), /** * dnindex切换信息 * @字段说明 FLOW_ZK_PATH_CACHE */ FLOW_ZK_PATH_BINDATA_DNINDEX("dnindex"), /** * 迁移的信息 * @字段说明 FLOW_ZK_PATH_CACHE */ FLOW_ZK_PATH_BINDATA_MOVE("move"), /** * 节点单独的配制信息 * @字段说明 FLOW_ZK_PATH_NODE */ FLOW_ZK_PATH_NODE("node"), /**zk写入本地的路径信息 * @字段说明 ZK_LOCAL_WRITE_PATH */ ZK_LOCAL_WRITE_PATH("./"), // /**zk写入本地的路径信息 // * @字段说明 ZK_LOCAL_WRITE_PATH // */ // ZK_LOCAL_WRITE_PATH("zkdownload/"), /** * zk本地配制目录信息 * @字段说明 ZK_LOCAL_WRITE_PATH */ ZK_LOCAL_CFG_PATH("/zkconf/"), ; /** * 配制的key的信息 * @字段说明 key */ private String key; private ZookeeperPath(String key) { this.key = key; } public String getKey() { return key; } public void setKey(String key) { this.key = key; } } ================================================ FILE: src/main/java/io/mycat/config/loader/xml/XMLConfigLoader.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.loader.xml; import java.util.Map; import io.mycat.config.loader.ConfigLoader; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.model.ClusterConfig; import io.mycat.config.model.DataHostConfig; import io.mycat.config.model.DataNodeConfig; import io.mycat.config.model.FirewallConfig; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.config.model.UserConfig; /** * @author mycat */ public class XMLConfigLoader implements ConfigLoader { /** unmodifiable */ private final Map dataHosts; /** unmodifiable */ private final Map dataNodes; /** unmodifiable */ private final Map schemas; private final SystemConfig system; /** unmodifiable */ private final Map users; private final FirewallConfig firewall; private final ClusterConfig cluster; public XMLConfigLoader(SchemaLoader schemaLoader) { XMLServerLoader serverLoader = new XMLServerLoader(); this.system = serverLoader.getSystem(); this.users = serverLoader.getUsers(); this.firewall = serverLoader.getFirewall(); this.cluster = serverLoader.getCluster(); this.dataHosts = schemaLoader.getDataHosts(); this.dataNodes = schemaLoader.getDataNodes(); this.schemas = schemaLoader.getSchemas(); schemaLoader = null; } @Override public ClusterConfig getClusterConfig() { return cluster; } @Override public FirewallConfig getFirewallConfig() { return firewall; } @Override public UserConfig getUserConfig(String user) { return users.get(user); } @Override public Map getUserConfigs() { return users; } @Override public SystemConfig getSystemConfig() { return system; } @Override public Map getSchemaConfigs() { return schemas; } @Override public Map getDataNodes() { return dataNodes; } @Override public Map getDataHosts() { return dataHosts; } @Override public SchemaConfig getSchemaConfig(String schema) { return schemas.get(schema); } } ================================================ FILE: src/main/java/io/mycat/config/loader/xml/XMLRuleLoader.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.loader.xml; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.InvocationTargetException; import java.sql.SQLSyntaxErrorException; import java.util.Collections; import java.util.HashMap; import java.util.Map; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import io.mycat.config.model.rule.RuleConfig; import io.mycat.config.model.rule.TableRuleConfig; import io.mycat.config.util.ConfigException; import io.mycat.config.util.ConfigUtil; import io.mycat.config.util.ParameterMapping; import io.mycat.route.function.AbstractPartitionAlgorithm; import io.mycat.util.SplitUtil; /** * @author mycat */ @SuppressWarnings("unchecked") public class XMLRuleLoader { private final static String DEFAULT_DTD = "/rule.dtd"; private final static String DEFAULT_XML = "/rule.xml"; private final Map tableRules; // private final Set rules; private final Map functions; public XMLRuleLoader(String ruleFile) { // this.rules = new HashSet(); //rule名 -> rule this.tableRules = new HashMap(); //function名 -> 具体分片算法 this.functions = new HashMap(); load(DEFAULT_DTD, ruleFile == null ? DEFAULT_XML : ruleFile); } public XMLRuleLoader() { this(null); } public Map getTableRules() { return (Map) (tableRules.isEmpty() ? Collections .emptyMap() : tableRules); } private void load(String dtdFile, String xmlFile) { InputStream dtd = null; InputStream xml = null; try { dtd = XMLRuleLoader.class.getResourceAsStream(dtdFile); xml = XMLRuleLoader.class.getResourceAsStream(xmlFile); //读取出语意树 Element root = ConfigUtil.getDocument(dtd, xml) .getDocumentElement(); //加载Function loadFunctions(root); //加载TableRule loadTableRules(root); } catch (ConfigException e) { throw e; } catch (Exception e) { throw new ConfigException(e); } finally { if (dtd != null) { try { dtd.close(); } catch (IOException e) { } } if (xml != null) { try { xml.close(); } catch (IOException e) { } } } } /** * tableRule标签结构： * * * create_date * partbymonth * * * @param root * @throws SQLSyntaxErrorException */ private void loadTableRules(Element root) throws SQLSyntaxErrorException { //获取每个tableRule标签 NodeList list = root.getElementsByTagName("tableRule"); for (int i = 0, n = list.getLength(); i < n; ++i) { Node node = list.item(i); if (node instanceof Element) { Element e = (Element) node; //先判断是否重复 String name = e.getAttribute("name"); if (tableRules.containsKey(name)) { throw new ConfigException("table rule " + name + " duplicated!"); } //获取rule标签 NodeList ruleNodes = e.getElementsByTagName("rule"); int length = ruleNodes.getLength(); if (length > 1) { throw new ConfigException("only one rule can defined :" + name); } //目前只处理第一个，未来可能有多列复合逻辑需求 //RuleConfig是保存着rule与function对应关系的对象 RuleConfig rule = loadRule((Element) ruleNodes.item(0)); String funName = rule.getFunctionName(); //判断function是否存在，获取function AbstractPartitionAlgorithm func = functions.get(funName); if (func == null) { throw new ConfigException("can't find function of name :" + funName); } rule.setRuleAlgorithm(func); //保存到tableRules tableRules.put(name, new TableRuleConfig(name, rule)); } } } private RuleConfig loadRule(Element element) throws SQLSyntaxErrorException { //读取columns Element columnsEle = ConfigUtil.loadElement(element, "columns"); String column = columnsEle.getTextContent(); String[] columns = SplitUtil.split(column, ',', true); if (columns.length > 1) { throw new ConfigException("table rule coulmns has multi values:" + columnsEle.getTextContent()); } //读取algorithm Element algorithmEle = ConfigUtil.loadElement(element, "algorithm"); String algorithm = algorithmEle.getTextContent(); return new RuleConfig(column.toUpperCase(), algorithm); } /** * function标签结构： * * yyyy-MM-dd * 2015-01-01 * * @param root * @throws ClassNotFoundException * @throws InstantiationException * @throws IllegalAccessException * @throws InvocationTargetException */ private void loadFunctions(Element root) throws ClassNotFoundException, InstantiationException, IllegalAccessException, InvocationTargetException { NodeList list = root.getElementsByTagName("function"); for (int i = 0, n = list.getLength(); i < n; ++i) { Node node = list.item(i); if (node instanceof Element) { Element e = (Element) node; //获取name标签 String name = e.getAttribute("name"); //如果Map已有，则function重复 if (functions.containsKey(name)) { throw new ConfigException("rule function " + name + " duplicated!"); } //获取class标签 String clazz = e.getAttribute("class"); //根据class利用反射新建分片算法 AbstractPartitionAlgorithm function = createFunction(name, clazz); //根据读取参数配置分片算法 ParameterMapping.mapping(function, ConfigUtil.loadElements(e)); //每个AbstractPartitionAlgorithm可能会实现init来初始化 function.init(); //放入functions map functions.put(name, function); } } } private AbstractPartitionAlgorithm createFunction(String name, String clazz) throws ClassNotFoundException, InstantiationException, IllegalAccessException, InvocationTargetException { Class clz = Class.forName(clazz); //判断是否继承AbstractPartitionAlgorithm if (!AbstractPartitionAlgorithm.class.isAssignableFrom(clz)) { throw new IllegalArgumentException("rule function must implements " + AbstractPartitionAlgorithm.class.getName() + ", name=" + name); } return (AbstractPartitionAlgorithm) clz.newInstance(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/xml/XMLSchemaLoader.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.loader.xml; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.model.*; import io.mycat.config.model.rule.RuleConfig; import io.mycat.config.model.rule.TableRuleConfig; import io.mycat.config.util.ConfigException; import io.mycat.config.util.ConfigUtil; import io.mycat.route.function.AbstractPartitionAlgorithm; import io.mycat.route.function.TableRuleAware; import io.mycat.util.DecryptUtil; import io.mycat.util.ObjectUtil; import io.mycat.util.SplitUtil; import io.mycat.util.StringUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.text.SimpleDateFormat; import java.util.*; /** * @author mycat */ @SuppressWarnings("unchecked") public class XMLSchemaLoader implements SchemaLoader { private static final Logger LOGGER = LoggerFactory.getLogger(XMLSchemaLoader.class); private final static String DEFAULT_DTD = "/schema.dtd"; private final static String DEFAULT_XML = "/schema.xml"; private final Map tableRules; private final Map dataHosts; private final Map dataNodes; private final Map schemas; public XMLSchemaLoader(String schemaFile, String ruleFile) { //先读取rule.xml XMLRuleLoader ruleLoader = new XMLRuleLoader(ruleFile); //将tableRules拿出，用于这里加载Schema做rule有效判断，以及之后的分片路由计算 this.tableRules = ruleLoader.getTableRules(); //释放ruleLoader ruleLoader = null; this.dataHosts = new HashMap(); this.dataNodes = new HashMap(); this.schemas = new HashMap(); //读取加载schema配置 this.load(DEFAULT_DTD, schemaFile == null ? DEFAULT_XML : schemaFile); } public XMLSchemaLoader() { this(null, null); } @Override public Map getTableRules() { return tableRules; } @Override public Map getDataHosts() { return (Map) (dataHosts.isEmpty() ? Collections.emptyMap() : dataHosts); } @Override public Map getDataNodes() { return (Map) (dataNodes.isEmpty() ? Collections.emptyMap() : dataNodes); } @Override public Map getSchemas() { return (Map) (schemas.isEmpty() ? Collections.emptyMap() : schemas); } private void load(String dtdFile, String xmlFile) { InputStream dtd = null; InputStream xml = null; try { dtd = XMLSchemaLoader.class.getResourceAsStream(dtdFile); xml = XMLSchemaLoader.class.getResourceAsStream(xmlFile); Element root = ConfigUtil.getDocument(dtd, xml).getDocumentElement(); //先加载所有的DataHost loadDataHosts(root); //再加载所有的DataNode loadDataNodes(root); //最后加载所有的Schema loadSchemas(root); } catch (ConfigException e) { throw e; } catch (Exception e) { throw new ConfigException(e); } finally { if (dtd != null) { try { dtd.close(); } catch (IOException e) { } } if (xml != null) { try { xml.close(); } catch (IOException e) { } } } } private void loadSchemas(Element root) { NodeList list = root.getElementsByTagName("schema"); for (int i = 0, n = list.getLength(); i < n; i++) { Element schemaElement = (Element) list.item(i); //读取各个属性 String name = schemaElement.getAttribute("name"); String dataNode = schemaElement.getAttribute("dataNode"); String randomDataNode = schemaElement.getAttribute("randomDataNode"); String checkSQLSchemaStr = schemaElement.getAttribute("checkSQLschema"); String sqlMaxLimitStr = schemaElement.getAttribute("sqlMaxLimit"); int sqlMaxLimit = -1; //读取sql返回结果集限制 if (sqlMaxLimitStr != null && !sqlMaxLimitStr.isEmpty()) { sqlMaxLimit = Integer.parseInt(sqlMaxLimitStr); } // check dataNode already exists or not,看schema标签中是否有datanode String defaultDbType = null; //校验检查并添加dataNode if (dataNode != null && !dataNode.isEmpty()) { List dataNodeLst = new ArrayList(1); dataNodeLst.add(dataNode); checkDataNodeExists(dataNodeLst); String dataHost = dataNodes.get(dataNode).getDataHost(); defaultDbType = dataHosts.get(dataHost).getDbType(); } else { dataNode = null; } //加载schema下所有tables Map tables = loadTables(schemaElement); //判断schema是否重复 if (schemas.containsKey(name)) { throw new ConfigException("schema " + name + " duplicated!"); } // 设置了table的不需要设置dataNode属性，没有设置table的必须设置dataNode属性 if (dataNode == null && tables.size() == 0) { throw new ConfigException( "schema " + name + " didn't config tables,so you must set dataNode property!"); } SchemaConfig schemaConfig = new SchemaConfig(name, dataNode, tables, sqlMaxLimit, "true".equalsIgnoreCase(checkSQLSchemaStr),randomDataNode); //设定DB类型，这对之后的sql语句路由解析有帮助 if (defaultDbType != null) { schemaConfig.setDefaultDataNodeDbType(defaultDbType); if (!"mysql".equalsIgnoreCase(defaultDbType)) { schemaConfig.setNeedSupportMultiDBType(true); } } // 判断是否有不是mysql的数据库类型，方便解析判断是否启用多数据库分页语法解析 for (TableConfig tableConfig : tables.values()) { if (isHasMultiDbType(tableConfig)) { schemaConfig.setNeedSupportMultiDBType(true); break; } } //记录每种dataNode的DB类型 Map dataNodeDbTypeMap = new HashMap<>(); for (String dataNodeName : dataNodes.keySet()) { DataNodeConfig dataNodeConfig = dataNodes.get(dataNodeName); String dataHost = dataNodeConfig.getDataHost(); DataHostConfig dataHostConfig = dataHosts.get(dataHost); if (dataHostConfig != null) { String dbType = dataHostConfig.getDbType(); dataNodeDbTypeMap.put(dataNodeName, dbType); } } schemaConfig.setDataNodeDbTypeMap(dataNodeDbTypeMap); schemas.put(name, schemaConfig); } } /** * 处理动态日期表, 支持 YYYYMM、YYYYMMDD 两种格式 *

* YYYYMM格式： yyyymm,2015,01,60 * YYYYMMDD格式: yyyymmdd,2015,01,10,50 * * @param tableNameElement * @param tableNameSuffixElement * @return */ private String doTableNameSuffix(String tableNameElement, String tableNameSuffixElement) { String newTableName = tableNameElement; String[] params = tableNameSuffixElement.split(","); String suffixFormat = params[0].toUpperCase(); if (suffixFormat.equals("YYYYMM")) { //读取参数 int yyyy = Integer.parseInt(params[1]); int mm = Integer.parseInt(params[2]); int mmEndIdx = Integer.parseInt(params[3]); //日期处理 SimpleDateFormat yyyyMMSDF = new SimpleDateFormat("yyyyMM"); Calendar cal = Calendar.getInstance(); cal.set(Calendar.YEAR, yyyy); cal.set(Calendar.MONTH, mm - 1); cal.set(Calendar.DATE, 0); //表名改写 StringBuffer tableNameBuffer = new StringBuffer(); for (int mmIdx = 0; mmIdx <= mmEndIdx; mmIdx++) { tableNameBuffer.append(tableNameElement); tableNameBuffer.append(yyyyMMSDF.format(cal.getTime())); cal.add(Calendar.MONTH, 1); if (mmIdx != mmEndIdx) { tableNameBuffer.append(","); } } newTableName = tableNameBuffer.toString(); } else if (suffixFormat.equals("YYYYMMDD")) { //读取参数 int yyyy = Integer.parseInt(params[1]); int mm = Integer.parseInt(params[2]); int dd = Integer.parseInt(params[3]); int ddEndIdx = Integer.parseInt(params[4]); //日期处理 SimpleDateFormat yyyyMMddSDF = new SimpleDateFormat("yyyyMMdd"); Calendar cal = Calendar.getInstance(); cal.set(Calendar.YEAR, yyyy); cal.set(Calendar.MONTH, mm - 1); cal.set(Calendar.DATE, dd); //表名改写 StringBuffer tableNameBuffer = new StringBuffer(); for (int ddIdx = 0; ddIdx <= ddEndIdx; ddIdx++) { tableNameBuffer.append(tableNameElement); tableNameBuffer.append(yyyyMMddSDF.format(cal.getTime())); cal.add(Calendar.DATE, 1); if (ddIdx != ddEndIdx) { tableNameBuffer.append(","); } } newTableName = tableNameBuffer.toString(); } return newTableName; } private Map loadTables(Element node) { // Map tables = new HashMap(); // 支持表名中包含引号[`] BEN GONG final String schemaName = node.getAttribute("name"); Map tables = new TableConfigMap(); NodeList nodeList = node.getElementsByTagName("table"); List list = new ArrayList<>(); for (int i = 0; i < nodeList.getLength(); i++) { Element tableElement = (Element) nodeList.item(i); String tableNameElement = tableElement.getAttribute("name").toUpperCase(); if("true".equalsIgnoreCase(tableElement.getAttribute("splitTableNames"))){ String[] split = tableNameElement.split(","); for (String name : split) { Element node1 = (Element)tableElement.cloneNode(true); node1.setAttribute("name",name); list.add(node1); } }else { list.add(tableElement); } } loadTable(schemaName, tables, list); return tables; } private void loadTable(String schemaName, Map tables, List nodeList) { for (int i = 0; i < nodeList.size(); i++) { Element tableElement = (Element) nodeList.get(i); String tableNameElement = tableElement.getAttribute("name").toUpperCase(); //TODO:路由, 增加对动态日期表的支持 String tableNameSuffixElement = tableElement.getAttribute("nameSuffix").toUpperCase(); if (!"".equals(tableNameSuffixElement)) { if (tableNameElement.split(",").length > 1) { throw new ConfigException("nameSuffix " + tableNameSuffixElement + ", require name parameter cannot multiple breaks!"); } //前缀用来标明日期格式 tableNameElement = doTableNameSuffix(tableNameElement, tableNameSuffixElement); } //记录主键，用于之后路由分析，以及启用自增长主键 String[] tableNames = tableNameElement.split(","); String primaryKey = tableElement.hasAttribute("primaryKey") ? tableElement.getAttribute("primaryKey").toUpperCase() : null; //记录是否主键自增，默认不是，（启用全局sequence handler） boolean autoIncrement = false; if (tableElement.hasAttribute("autoIncrement")) { autoIncrement = Boolean.parseBoolean(tableElement.getAttribute("autoIncrement")); } boolean fetchStoreNodeByJdbc = false; if (tableElement.hasAttribute("fetchStoreNodeByJdbc")) { fetchStoreNodeByJdbc = Boolean.parseBoolean(tableElement.getAttribute("fetchStoreNodeByJdbc")); } //记录是否需要加返回结果集限制，默认需要加 boolean needAddLimit = true; if (tableElement.hasAttribute("needAddLimit")) { needAddLimit = Boolean.parseBoolean(tableElement.getAttribute("needAddLimit")); } //记录type，是否为global String tableTypeStr = tableElement.hasAttribute("type") ? tableElement.getAttribute("type") : null; int tableType = TableConfig.TYPE_GLOBAL_DEFAULT; if ("global".equalsIgnoreCase(tableTypeStr)) { tableType = TableConfig.TYPE_GLOBAL_TABLE; } //记录dataNode，就是分布在哪些dataNode上 String dataNode = tableElement.getAttribute("dataNode"); TableRuleConfig tableRule = null; if (tableElement.hasAttribute("rule")) { String ruleName = tableElement.getAttribute("rule"); tableRule = tableRules.get(ruleName); if (tableRule == null) { throw new ConfigException("rule " + ruleName + " is not found!"); } } boolean ruleRequired = false; //记录是否绑定有分片规则 if (tableElement.hasAttribute("ruleRequired")) { ruleRequired = Boolean.parseBoolean(tableElement.getAttribute("ruleRequired")); } if (tableNames == null) { throw new ConfigException("table name is not found!"); } //distribute函数，重新编排dataNode String distPrex = "distribute("; boolean distTableDns = dataNode.startsWith(distPrex); if (distTableDns) { dataNode = dataNode.substring(distPrex.length(), dataNode.length() - 1); } //分表功能 String subTables = tableElement.getAttribute("subTables"); for (int j = 0; j < tableNames.length; j++) { String tableName = tableNames[j]; TableRuleConfig tableRuleConfig = tableRule; if (tableRuleConfig != null) { //对于实现TableRuleAware的function进行特殊处理 根据每个表新建个实例 RuleConfig rule = tableRuleConfig.getRule(); if (rule.getRuleAlgorithm() instanceof TableRuleAware) { //因为ObjectUtil.copyObject是深拷贝,所以会把crc32的算法也拷贝一份状态,而不是公用一个分片数 tableRuleConfig = (TableRuleConfig) ObjectUtil.copyObject(tableRuleConfig); String name = tableRuleConfig.getName(); String newRuleName = getNewRuleName(schemaName, tableName, name); tableRuleConfig.setName(newRuleName); TableRuleAware tableRuleAware = (TableRuleAware) tableRuleConfig.getRule().getRuleAlgorithm(); tableRuleAware.setRuleName(newRuleName); tableRules.put(newRuleName, tableRuleConfig); } } TableConfig table = new TableConfig(tableName, primaryKey, autoIncrement, needAddLimit, tableType, dataNode, getDbType(dataNode), (tableRuleConfig != null) ? tableRuleConfig.getRule() : null, ruleRequired, null, false, null, null, subTables, fetchStoreNodeByJdbc); //因为需要等待TableConfig构造完毕才可以拿到dataNode节点数量,所以Rule构造延后到此处 @cjw if ((tableRuleConfig != null) && (tableRuleConfig.getRule().getRuleAlgorithm() instanceof TableRuleAware)) { AbstractPartitionAlgorithm newRuleAlgorithm = tableRuleConfig.getRule().getRuleAlgorithm(); ((TableRuleAware)newRuleAlgorithm).setTableConfig(table); newRuleAlgorithm.init(); } checkDataNodeExists(table.getDataNodes()); // 检查分片表分片规则配置是否合法 if (table.getRule() != null) { checkRuleSuitTable(table); } if (distTableDns) { distributeDataNodes(table.getDataNodes()); } //检查去重 if (tables.containsKey(table.getName())) { throw new ConfigException("table " + tableName + " duplicated!"); } //放入map tables.put(table.getName(), table); } //只有tableName配置的是单个表（没有逗号）的时候才能有子表 if (tableNames.length == 1) { TableConfig table = tables.get(tableNames[0]); // process child tables processChildTables(tables, table, dataNode, tableElement); } } } private String getNewRuleName(String schemaName, String tableName, String name) { return name + "_" + schemaName + "_" + tableName; } /** * distribute datanodes in multi hosts,means ,dn1 (host1),dn100 * (host2),dn300(host3),dn2(host1),dn101(host2),dn301(host3)...etc * 将每个host上的datanode按照host重新排列。比如上面的例子host1拥有dn1,dn2，host2拥有dn100，dn101，host3拥有dn300，dn301, * 按照host重新排列： 0->dn1 (host1),1->dn100(host2),2->dn300(host3),3->dn2(host1),4->dn101(host2),5->dn301(host3) * * @param theDataNodes */ private void distributeDataNodes(ArrayList theDataNodes) { Map> newDataNodeMap = new HashMap>(dataHosts.size()); for (String dn : theDataNodes) { DataNodeConfig dnConf = dataNodes.get(dn); String host = dnConf.getDataHost(); ArrayList hostDns = newDataNodeMap.get(host); hostDns = (hostDns == null) ? new ArrayList() : hostDns; hostDns.add(dn); newDataNodeMap.put(host, hostDns); } ArrayList result = new ArrayList(theDataNodes.size()); boolean hasData = true; while (hasData) { hasData = false; for (ArrayList dns : newDataNodeMap.values()) { if (!dns.isEmpty()) { result.add(dns.remove(0)); hasData = true; } } } theDataNodes.clear(); theDataNodes.addAll(result); } private Set getDbType(String dataNode) { Set dbTypes = new HashSet<>(); String[] dataNodeArr = SplitUtil.split(dataNode, ',', '$', '-'); for (String node : dataNodeArr) { DataNodeConfig datanode = dataNodes.get(node); DataHostConfig datahost = dataHosts.get(datanode.getDataHost()); dbTypes.add(datahost.getDbType()); } return dbTypes; } private Set getDataNodeDbTypeMap(String dataNode) { Set dbTypes = new HashSet<>(); String[] dataNodeArr = SplitUtil.split(dataNode, ',', '$', '-'); for (String node : dataNodeArr) { DataNodeConfig datanode = dataNodes.get(node); DataHostConfig datahost = dataHosts.get(datanode.getDataHost()); dbTypes.add(datahost.getDbType()); } return dbTypes; } private boolean isHasMultiDbType(TableConfig table) { Set dbTypes = table.getDbTypes(); for (String dbType : dbTypes) { if (!"mysql".equalsIgnoreCase(dbType)) { return true; } } return false; } private void processChildTables(Map tables, TableConfig parentTable, String dataNodes, Element tableNode) { // parse child tables NodeList childNodeList = tableNode.getChildNodes(); for (int j = 0; j < childNodeList.getLength(); j++) { Node theNode = childNodeList.item(j); if (!theNode.getNodeName().equals("childTable")) { continue; } Element childTbElement = (Element) theNode; //读取子表信息 String cdTbName = childTbElement.getAttribute("name").toUpperCase(); String primaryKey = childTbElement.hasAttribute("primaryKey") ? childTbElement.getAttribute("primaryKey").toUpperCase() : null; boolean autoIncrement = false; if (childTbElement.hasAttribute("autoIncrement")) { autoIncrement = Boolean.parseBoolean(childTbElement.getAttribute("autoIncrement")); } boolean needAddLimit = true; if (childTbElement.hasAttribute("needAddLimit")) { needAddLimit = Boolean.parseBoolean(childTbElement.getAttribute("needAddLimit")); } String subTables = childTbElement.getAttribute("subTables"); //子表join键，和对应的parent的键，父子表通过这个关联 String joinKey = childTbElement.getAttribute("joinKey").toUpperCase(); String parentKey = childTbElement.getAttribute("parentKey").toUpperCase(); TableConfig table = new TableConfig(cdTbName, primaryKey, autoIncrement, needAddLimit, TableConfig.TYPE_GLOBAL_DEFAULT, dataNodes, getDbType(dataNodes), null, false, parentTable, true, joinKey, parentKey, subTables, false); if (tables.containsKey(table.getName())) { throw new ConfigException("table " + table.getName() + " duplicated!"); } tables.put(table.getName(), table); //对于子表的子表，递归处理 processChildTables(tables, table, dataNodes, childTbElement); } } private void checkDataNodeExists(Collection nodes) { if (nodes == null || nodes.size() < 1) { return; } for (String node : nodes) { if (!dataNodes.containsKey(node)) { throw new ConfigException("dataNode '" + node + "' is not found!"); } } } /** * 检查分片表分片规则配置, 目前主要检查分片表分片算法定义与分片dataNode是否匹配
* 例如分片表定义如下:
* {@code *

* } *
* 分片算法如下:
* {@code * * * 3 * * } *
* shard table datanode(2) < function count(3) 此时检测为不匹配 */ private void checkRuleSuitTable(TableConfig tableConf) { AbstractPartitionAlgorithm function = tableConf.getRule().getRuleAlgorithm(); int suitValue = function.suitableFor(tableConf); switch (suitValue) { case -1: // 少节点,给提示并抛异常 throw new ConfigException("Illegal table conf : table [ " + tableConf.getName() + " ] rule function [ " + tableConf.getRule().getFunctionName() + " ] partition size : " + tableConf.getRule().getRuleAlgorithm().getPartitionNum() + " > table datanode size : " + tableConf.getDataNodes().size() + ", please make sure table datanode size = function partition size"); case 0: // table datanode size == rule function partition size break; case 1: // 有些节点是多余的,给出warn log LOGGER.warn("table conf : table [ {} ] rule function [ {} ] partition size : {} < table datanode size : {} , this cause some datanode to be redundant", new String[]{ tableConf.getName(), tableConf.getRule().getFunctionName(), String.valueOf(tableConf.getRule().getRuleAlgorithm().getPartitionNum()), String.valueOf(tableConf.getDataNodes().size()) }); break; } } private void loadDataNodes(Element root) { //读取DataNode分支 NodeList list = root.getElementsByTagName("dataNode"); for (int i = 0, n = list.getLength(); i < n; i++) { Element element = (Element) list.item(i); String dnNamePre = element.getAttribute("name"); String databaseStr = element.getAttribute("database"); String host = element.getAttribute("dataHost"); //字符串不为空 if (empty(dnNamePre) || empty(databaseStr) || empty(host)) { throw new ConfigException("dataNode " + dnNamePre + " define error ,attribute can't be empty"); } //dnNames（name）,databases（database）,hostStrings（dataHost）都可以配置多个，以',', '$', '-'区分，但是需要保证database的个数*dataHost的个数=name的个数 //多个dataHost与多个database如果写在一个标签，则每个dataHost拥有所有database //例如： //则为：localhost1拥有dn1$0-75,localhost2也拥有dn1$0-75（对应db$76-151） String[] dnNames = io.mycat.util.SplitUtil.split(dnNamePre, ',', '$', '-'); String[] databases = io.mycat.util.SplitUtil.split(databaseStr, ',', '$', '-'); String[] hostStrings = io.mycat.util.SplitUtil.split(host, ',', '$', '-'); if (dnNames.length > 1 && dnNames.length != databases.length * hostStrings.length) { throw new ConfigException("dataNode " + dnNamePre + " define error ,dnNames.length must be=databases.length*hostStrings.length"); } if (dnNames.length > 1) { List mhdList = mergerHostDatabase(hostStrings, databases); for (int k = 0; k < dnNames.length; k++) { String[] hd = mhdList.get(k); String dnName = dnNames[k]; String databaseName = hd[1]; String hostName = hd[0]; createDataNode(dnName, databaseName, hostName); } } else { createDataNode(dnNamePre, databaseStr, host); } } } /** * 匹配DataHost和Database，每个DataHost拥有每个Database名字 * * @param hostStrings * @param databases * @return */ private List mergerHostDatabase(String[] hostStrings, String[] databases) { List mhdList = new ArrayList<>(); for (int i = 0; i < hostStrings.length; i++) { String hostString = hostStrings[i]; for (int i1 = 0; i1 < databases.length; i1++) { String database = databases[i1]; String[] hd = new String[2]; hd[0] = hostString; hd[1] = database; mhdList.add(hd); } } return mhdList; } private void createDataNode(String dnName, String database, String host) { DataNodeConfig conf = new DataNodeConfig(dnName, database, host); if (dataNodes.containsKey(conf.getName())) { throw new ConfigException("dataNode " + conf.getName() + " duplicated!"); } if (!dataHosts.containsKey(host)) { throw new ConfigException("dataNode " + dnName + " reference dataHost:" + host + " not exists!"); } dataHosts.get(host).addDataNode(conf.getName()); dataNodes.put(conf.getName(), conf); } private boolean empty(String dnName) { return dnName == null || dnName.length() == 0; } private DBHostConfig createDBHostConf(String dataHost, Element node, String dbType, String dbDriver, int maxCon, int minCon, String filters, long logTime) { String nodeHost = node.getAttribute("host"); String nodeUrl = node.getAttribute("url"); String user = node.getAttribute("user"); String password = node.getAttribute("password"); String usingDecrypt = node.getAttribute("usingDecrypt"); String checkAliveText = node.getAttribute("checkAlive"); if (checkAliveText == null)checkAliveText = Boolean.TRUE.toString(); boolean checkAlive = Boolean.parseBoolean(checkAliveText); String passwordEncryty = DecryptUtil.DBHostDecrypt(usingDecrypt, nodeHost, user, password); String weightStr = node.getAttribute("weight"); int weight = "".equals(weightStr) ? PhysicalDBPool.WEIGHT : Integer.parseInt(weightStr); String ip = null; int port = 0; if (empty(nodeHost) || empty(nodeUrl) || empty(user)) { throw new ConfigException( "dataHost " + dataHost + " define error,some attributes of this element is empty: " + nodeHost); } if ("native".equalsIgnoreCase(dbDriver)) { int colonIndex = nodeUrl.indexOf(':'); ip = nodeUrl.substring(0, colonIndex).trim(); port = Integer.parseInt(nodeUrl.substring(colonIndex + 1).trim()); } else { URI url; try { url = new URI(nodeUrl.substring(5)); } catch (Exception e) { throw new ConfigException("invalid jdbc url " + nodeUrl + " of " + dataHost); } ip = url.getHost(); port = url.getPort(); } DBHostConfig conf = new DBHostConfig(nodeHost, ip, port, nodeUrl, user, passwordEncryty, password,checkAlive); conf.setDbType(dbType); conf.setMaxCon(maxCon); conf.setMinCon(minCon); conf.setFilters(filters); conf.setLogTime(logTime); conf.setWeight(weight); //新增权重 return conf; } private void loadDataHosts(Element root) { NodeList list = root.getElementsByTagName("dataHost"); for (int i = 0, n = list.getLength(); i < n; ++i) { Element element = (Element) list.item(i); String name = element.getAttribute("name"); //判断是否重复 if (dataHosts.containsKey(name)) { throw new ConfigException("dataHost name " + name + "duplicated!"); } //读取最大连接数 int maxCon = Integer.parseInt(element.getAttribute("maxCon")); //读取最小连接数 int minCon = Integer.parseInt(element.getAttribute("minCon")); /** * 读取负载均衡配置 * 1. balance="0", 不开启分离机制，所有读操作都发送到当前可用的 writeHost 上。 * 2. balance="1"，全部的 readHost 和 stand by writeHost 参不 select 的负载均衡 * 3. balance="2"，所有读操作都随机的在 writeHost、readhost 上分发。 * 4. balance="3"，所有读请求随机的分发到 wiriterHost 对应的 readhost 执行，writerHost 不负担读压力 */ int balance = Integer.parseInt(element.getAttribute("balance")); /** * 负载均衡配置 * 1. balanceType=0, 随机 * 2. balanceType=1，加权轮询 * 3. balanceType=2，最少活跃 */ String balanceTypeStr = element.getAttribute("balanceType"); int balanceType = balanceTypeStr.equals("") ? 0 : Integer.parseInt(balanceTypeStr); /** * 读取切换类型 * -1 表示不自动切换 * 1 默认值，自动切换 * 2 基于MySQL主从同步的状态决定是否切换 * 心跳询句为 show slave status * 3 基于 MySQL galary cluster 的切换机制 */ String switchTypeStr = element.getAttribute("switchType"); int switchType = switchTypeStr.equals("") ? -1 : Integer.parseInt(switchTypeStr); //读取从延迟界限 String slaveThresholdStr = element.getAttribute("slaveThreshold"); int slaveThreshold = slaveThresholdStr.equals("") ? -1 : Integer.parseInt(slaveThresholdStr); //如果 tempReadHostAvailable 设置大于 0 则表示写主机如果挂掉， 临时的读服务依然可用 String tempReadHostAvailableStr = element.getAttribute("tempReadHostAvailable"); boolean tempReadHostAvailable = !tempReadHostAvailableStr.equals("") && Integer.parseInt(tempReadHostAvailableStr) > 0; /** * 读取 写类型 * 这里只支持 0 - 所有写操作仅配置的第一个 writeHost */ String writeTypStr = element.getAttribute("writeType"); int writeType = "".equals(writeTypStr) ? PhysicalDBPool.WRITE_ONLYONE_NODE : Integer.parseInt(writeTypStr); String dbDriver = element.getAttribute("dbDriver"); String dbType = element.getAttribute("dbType"); String filters = element.getAttribute("filters"); String logTimeStr = element.getAttribute("logTime"); String slaveIDs = element.getAttribute("slaveIDs"); String maxRetryCountStr = element.getAttribute("maxRetryCount"); int maxRetryCount; if (StringUtil.isEmpty(maxRetryCountStr)) { maxRetryCount = 3; } else { maxRetryCount = Integer.valueOf(maxRetryCountStr); } long logTime = "".equals(logTimeStr) ? PhysicalDBPool.LONG_TIME : Long.parseLong(logTimeStr); String notSwitch = element.getAttribute("notSwitch"); if(StringUtil.isEmpty(notSwitch)) { notSwitch = DataHostConfig.CAN_SWITCH_DS; } //读取心跳语句 String heartbeatSQL = element.getElementsByTagName("heartbeat").item(0).getTextContent(); //读取 初始化sql配置,用于oracle NodeList connectionInitSqlList = element.getElementsByTagName("connectionInitSql"); String initConSQL = null; if (connectionInitSqlList.getLength() > 0) { initConSQL = connectionInitSqlList.item(0).getTextContent(); } //读取writeHost NodeList writeNodes = element.getElementsByTagName("writeHost"); DBHostConfig[] writeDbConfs = new DBHostConfig[writeNodes.getLength()]; Map readHostsMap = new HashMap(2); Set writeHostNameSet = new HashSet(writeNodes.getLength()); for (int w = 0; w < writeDbConfs.length; w++) { Element writeNode = (Element) writeNodes.item(w); writeDbConfs[w] = createDBHostConf(name, writeNode, dbType, dbDriver, maxCon, minCon, filters, logTime); if (writeHostNameSet.contains(writeDbConfs[w].getHostName())) { throw new ConfigException("writeHost " + writeDbConfs[w].getHostName() + " duplicated!"); } else { writeHostNameSet.add(writeDbConfs[w].getHostName()); } NodeList readNodes = writeNode.getElementsByTagName("readHost"); //读取对应的每一个readHost if (readNodes.getLength() != 0) { DBHostConfig[] readDbConfs = new DBHostConfig[readNodes.getLength()]; Set readHostNameSet = new HashSet(readNodes.getLength()); for (int r = 0; r < readDbConfs.length; r++) { Element readNode = (Element) readNodes.item(r); readDbConfs[r] = createDBHostConf(name, readNode, dbType, dbDriver, maxCon, minCon, filters, logTime); if (readHostNameSet.contains(readDbConfs[r].getHostName())) { throw new ConfigException("readHost " + readDbConfs[r].getHostName() + " duplicated!"); } else { readHostNameSet.add(readDbConfs[r].getHostName()); } } readHostsMap.put(w, readDbConfs); } } DataHostConfig hostConf = new DataHostConfig(name, dbType, dbDriver, writeDbConfs, readHostsMap, switchType, slaveThreshold, tempReadHostAvailable); hostConf.setMaxCon(maxCon); hostConf.setMinCon(minCon); hostConf.setBalance(balance); hostConf.setBalanceType(balanceType); hostConf.setWriteType(writeType); hostConf.setHearbeatSQL(heartbeatSQL); hostConf.setConnectionInitSql(initConSQL); hostConf.setFilters(filters); hostConf.setLogTime(logTime); hostConf.setSlaveIDs(slaveIDs); hostConf.setNotSwitch(notSwitch); hostConf.setMaxRetryCount(maxRetryCount); dataHosts.put(hostConf.getName(), hostConf); } } } ================================================ FILE: src/main/java/io/mycat/config/loader/xml/XMLServerLoader.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.loader.xml; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.regex.Pattern; import io.mycat.config.Versions; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import com.alibaba.druid.wall.WallConfig; import io.mycat.config.model.ClusterConfig; import io.mycat.config.model.FirewallConfig; import io.mycat.config.model.SystemConfig; import io.mycat.config.model.UserConfig; import io.mycat.config.model.UserPrivilegesConfig; import io.mycat.config.util.ConfigException; import io.mycat.config.util.ConfigUtil; import io.mycat.config.util.ParameterMapping; import io.mycat.util.DecryptUtil; import io.mycat.util.SplitUtil; /** * @author mycat */ @SuppressWarnings("unchecked") public class XMLServerLoader { private final SystemConfig system; private final Map users; private final FirewallConfig firewall; private ClusterConfig cluster; public XMLServerLoader() { this.system = new SystemConfig(); this.users = new HashMap(); this.firewall = new FirewallConfig(); this.load(); } public SystemConfig getSystem() { return system; } public Map getUsers() { return (Map) (users.isEmpty() ? Collections.emptyMap() : Collections.unmodifiableMap(users)); } public FirewallConfig getFirewall() { return firewall; } public ClusterConfig getCluster() { return cluster; } private void load() { //读取server.xml配置 InputStream dtd = null; InputStream xml = null; try { dtd = XMLServerLoader.class.getResourceAsStream("/server.dtd"); xml = XMLServerLoader.class.getResourceAsStream("/server.xml"); Element root = ConfigUtil.getDocument(dtd, xml).getDocumentElement(); //加载System标签 loadSystem(root); //加载User标签 loadUsers(root); //加载集群配置 this.cluster = new ClusterConfig(root, system.getServerPort()); //加载全局SQL防火墙 loadFirewall(root); } catch (ConfigException e) { throw e; } catch (Exception e) { throw new ConfigException(e); } finally { if (dtd != null) { try { dtd.close(); } catch (IOException e) { } } if (xml != null) { try { xml.close(); } catch (IOException e) { } } } } /** * 初始载入配置获取防火墙配置，配置防火墙方法之一，一共有两处，另一处: * @see FirewallConfig * * @modification 修改增加网段白名单 * @date 2016/12/8 * @modifiedBy Hash Zhang */ private void loadFirewall(Element root) throws IllegalAccessException, InvocationTargetException { NodeList list = root.getElementsByTagName("host"); Map> whitehost = new HashMap<>(); Map> whitehostMask = new HashMap<>(); for (int i = 0, n = list.getLength(); i < n; i++) { Node node = list.item(i); if (node instanceof Element) { Element e = (Element) node; String hostStr = e.getAttribute("host").trim(); String userStr = e.getAttribute("user").trim(); String []hosts = hostStr.split(","); for (String host : hosts) { host = host.trim(); if (this.firewall.existsHost(host)) { throw new ConfigException("host duplicated : " + host); } } String []users = userStr.split(","); List userConfigs = new ArrayList(); for(String user : users){ user = user.trim(); UserConfig uc = this.users.get(user); if (null == uc) { throw new ConfigException("[user: " + user + "] doesn't exist in [host: " + hostStr + "]"); } if (uc.getSchemas() == null || uc.getSchemas().size() == 0) { throw new ConfigException("[host: " + hostStr + "] contains one root privileges user: " + user); } userConfigs.add(uc); } for (String host : hosts) { host = host.trim(); if (host.contains("*") || host.contains("%")) { whitehostMask.put(FirewallConfig.getMaskPattern(host), userConfigs); } else { whitehost.put(host, userConfigs); } } } } firewall.setWhitehost(whitehost); firewall.setWhitehostMask(whitehostMask); WallConfig wallConfig = new WallConfig(); NodeList blacklist = root.getElementsByTagName("blacklist"); for (int i = 0, n = blacklist.getLength(); i < n; i++) { Node node = blacklist.item(i); if (node instanceof Element) { Element e = (Element) node; String check = e.getAttribute("check"); if (null != check) { firewall.setCheck(Boolean.parseBoolean(check)); } Map props = ConfigUtil.loadElements((Element) node); ParameterMapping.mapping(wallConfig, props); } } firewall.setWallConfig(wallConfig); firewall.init(); } private void loadUsers(Element root) { NodeList list = root.getElementsByTagName("user"); for (int i = 0, n = list.getLength(); i < n; i++) { Node node = list.item(i); if (node instanceof Element) { Element e = (Element) node; String name = e.getAttribute("name"); //huangyiming add String defaultAccount = e.getAttribute("defaultAccount"); UserConfig user = new UserConfig(); Map props = ConfigUtil.loadElements(e); String password = (String)props.get("password"); String usingDecrypt = (String)props.get("usingDecrypt"); String passwordDecrypt = DecryptUtil.mycatDecrypt(usingDecrypt,name,password); user.setName(name); user.setDefaultAccount(Boolean.parseBoolean(defaultAccount)); user.setDefaultSchema((String)props.get("defaultSchema")); user.setPassword(passwordDecrypt); user.setEncryptPassword(password); String benchmark = (String) props.get("benchmark"); if(null != benchmark) { user.setBenchmark( Integer.parseInt(benchmark) ); } String readOnly = (String) props.get("readOnly"); if (null != readOnly) { user.setReadOnly(Boolean.parseBoolean(readOnly)); } String schemas = (String) props.get("schemas"); if (schemas != null) { String[] strArray = SplitUtil.split(schemas, ',', true); user.setSchemas(new HashSet(Arrays.asList(strArray))); } //加载用户 DML 权限 loadPrivileges(user, e); if (users.containsKey(name)) { throw new ConfigException("user " + name + " duplicated!"); } users.put(name, user); } } } private void loadPrivileges(UserConfig userConfig, Element node) { UserPrivilegesConfig privilegesConfig = new UserPrivilegesConfig(); NodeList privilegesNodes = node.getElementsByTagName("privileges"); int privilegesNodesLength = privilegesNodes.getLength(); for (int i = 0; i < privilegesNodesLength; ++i) { Element privilegesNode = (Element) privilegesNodes.item(i); String check = privilegesNode.getAttribute("check"); if (null != check) { privilegesConfig.setCheck(Boolean.valueOf(check)); } NodeList schemaNodes = privilegesNode.getElementsByTagName("schema"); int schemaNodeLength = schemaNodes.getLength(); for (int j = 0; j < schemaNodeLength; j++ ) { Element schemaNode = (Element) schemaNodes.item(j); String name1 = schemaNode.getAttribute("name"); String dml1 = schemaNode.getAttribute("dml"); int[] dml1Array = new int[ dml1.length() ]; for(int offset1 = 0; offset1 < dml1.length(); offset1++ ) { dml1Array[offset1] = Character.getNumericValue( dml1.charAt( offset1 ) ); } UserPrivilegesConfig.SchemaPrivilege schemaPrivilege = new UserPrivilegesConfig.SchemaPrivilege(); schemaPrivilege.setName( name1 ); schemaPrivilege.setDml( dml1Array ); NodeList tableNodes = schemaNode.getElementsByTagName("table"); int tableNodeLength = tableNodes.getLength(); for (int z = 0; z < tableNodeLength; z++) { UserPrivilegesConfig.TablePrivilege tablePrivilege = new UserPrivilegesConfig.TablePrivilege(); Element tableNode = (Element) tableNodes.item(z); String name2 = tableNode.getAttribute("name"); String dml2 = tableNode.getAttribute("dml"); int[] dml2Array = new int[ dml2.length() ]; for(int offset2 = 0; offset2 < dml2.length(); offset2++ ) { dml2Array[offset2] = Character.getNumericValue( dml2.charAt( offset2 ) ); } tablePrivilege.setName( name2 ); tablePrivilege.setDml( dml2Array ); schemaPrivilege.addTablePrivilege(name2, tablePrivilege); } privilegesConfig.addSchemaPrivilege(name1, schemaPrivilege); } // 获取 dataNode 权限 NodeList dataNodes = privilegesNode.getElementsByTagName("dataNode"); int dataNodeLength = dataNodes.getLength(); for(int k = 0; k < dataNodeLength; k++){ UserPrivilegesConfig.DataNodePrivilege dataNodePrivilege = new UserPrivilegesConfig.DataNodePrivilege(); Element dataNode = (Element) dataNodes.item(k); String dataNodeName = dataNode.getAttribute("name"); String dataNodeDml = dataNode.getAttribute("dml"); int[] dataNodeDmlArray = new int[ dataNodeDml.length() ]; for(int offset2 = 0; offset2 < dataNodeDml.length(); offset2++ ) { dataNodeDmlArray[offset2] = Character.getNumericValue( dataNodeDml.charAt( offset2 ) ); } dataNodePrivilege.setName( dataNodeName ); dataNodePrivilege.setDml( dataNodeDmlArray ); privilegesConfig.addDataNodePrivileges(dataNodeName, dataNodePrivilege); } } userConfig.setPrivilegesConfig(privilegesConfig); } private void loadSystem(Element root) throws IllegalAccessException, InvocationTargetException { NodeList list = root.getElementsByTagName("system"); for (int i = 0, n = list.getLength(); i < n; i++) { Node node = list.item(i); if (node instanceof Element) { Map props = ConfigUtil.loadElements((Element) node); ParameterMapping.mapping(system, props); } } if (system.getFakeMySQLVersion() != null) { boolean validVersion = true; String majorMySQLVersion = system.getFakeMySQLVersion(); /* * 注意！！！ 目前MySQL官方主版本号仍然是5.x, 以后万一前面的大版本号变成2位数字， * 比如 10.x...,下面获取主版本的代码要做修改 */ // majorMySQLVersion = majorMySQLVersion.substring(0, majorMySQLVersion.indexOf(".", 2)); // for (String ver : SystemConfig.MySQLVersions) { // // 这里只是比较mysql前面的大版本号 // if (majorMySQLVersion.equals(ver)) { // validVersion = true; // } // } if (validVersion) { Versions.setServerVersion(system.getFakeMySQLVersion()); } else { throw new ConfigException("The specified MySQL Version (" + system.getFakeMySQLVersion() + ") is not valid."); } } } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/comm/NotiflyService.java ================================================ package io.mycat.config.loader.zkprocess.comm; /** * 通过接口 * @author liujun * * @date 2015年2月4日 * @vsersion 0.0.1 */ public interface NotiflyService { /** * 进行通知接口 * @throws Exception 异常操作 * @return true 通知更新成功，false ，更新失败 */ public boolean notiflyProcess() throws Exception; } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/comm/ZkConfig.java ================================================ package io.mycat.config.loader.zkprocess.comm; import java.io.IOException; import java.io.InputStream; import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Strings; import io.mycat.config.loader.zkprocess.zktoxml.ZktoXmlMain; /** * 进行zk的配制信息 * 源文件名：ZkConfig.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class ZkConfig { /** * 日志信息 * @字段说明 LOGGER */ private static final Logger LOGGER = LoggerFactory.getLogger(ZkConfig.class); private static final String ZK_CONFIG_FILE_NAME = "/myid.properties"; private ZkConfig() { } /** * 实例对象信息 * @字段说明 ZKCFGINSTANCE */ private static ZkConfig ZKCFGINSTANCE = new ZkConfig(); /** * myid的属性文件信息 * @字段说明 ZKPROPERTIES */ private static Properties ZKPROPERTIES = null; static { ZKPROPERTIES = LoadMyidPropersites(); } public String getZkURL() { return ZKPROPERTIES==null?null:ZKPROPERTIES.getProperty(ZkParamCfg.ZK_CFG_URL.getKey()) ; } public void initZk() { try { if (Boolean.parseBoolean(ZKPROPERTIES.getProperty(ZkParamCfg.ZK_CFG_FLAG.getKey()))) { ZktoXmlMain.loadZktoFile(); } } catch (Exception e) { LOGGER.error("error:",e); } } /** * 获得实例对象信息 * 方法描述 * @return * @创建日期 2016年9月15日 */ public static ZkConfig getInstance() { return ZKCFGINSTANCE; } /** * 获取myid属性文件中的属性值 * 方法描述 * @param param 参数信息 * @return * @创建日期 2016年9月15日 */ public String getValue(ZkParamCfg param) { if (null != param) { return ZKPROPERTIES.getProperty(param.getKey()); } return null; } /** * 加载myid配制文件信息 * 方法描述 * @return * @创建日期 2016年9月15日 */ private static Properties LoadMyidPropersites() { Properties pros = new Properties(); try (InputStream configIS = ZkConfig.class.getResourceAsStream(ZK_CONFIG_FILE_NAME)) { if (configIS == null) { return null; } pros.load(configIS); } catch (IOException e) { LOGGER.error("ZkConfig LoadMyidPropersites error:", e); throw new RuntimeException("can't find myid properties file : " + ZK_CONFIG_FILE_NAME); } // validate String zkURL = pros.getProperty(ZkParamCfg.ZK_CFG_URL.getKey()); String myid = pros.getProperty(ZkParamCfg.ZK_CFG_MYID.getKey()); String clusterId = pros.getProperty(ZkParamCfg.ZK_CFG_CLUSTERID.getKey()); if (Strings.isNullOrEmpty(clusterId) ||Strings.isNullOrEmpty(zkURL) || Strings.isNullOrEmpty(myid)) { throw new RuntimeException("clusterId and zkURL and myid must not be null or empty!"); } return pros; } public static void main(String[] args) { String zk = ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_CLUSTERID); System.out.println(zk); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/comm/ZkParamCfg.java ================================================ package io.mycat.config.loader.zkprocess.comm; /** * 当前zk的配制参数信息 * 源文件名：ZkParamCfg.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月17日 * 修改作者：liujun * 修改日期：2016年9月17日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public enum ZkParamCfg { /** * zk是否启用标识 * @字段说明 ZK_CFG_OPEN */ ZK_CFG_FLAG("loadZk"), /** * zk配制的url地址信息 * @字段说明 ZK_CFG_URL */ ZK_CFG_URL("zkURL"), /** * 集群的id * @字段说明 ZK_CFG_CLUSTERID */ ZK_CFG_CLUSTERID("clusterId"), ZK_CFG_CLUSTER_SIZE("clusterSize"), /** * 当前mycat节点的id * @字段说明 zk_CFG_MYID */ ZK_CFG_MYID("myid"), MYCAT_SERVER_TYPE("type"), MYCAT_BOOSTER_DATAHOSTS("boosterDataHosts"), /** * 集群中所有节点的名称信息 * @字段说明 ZK_CFG_CLUSTER_NODES */ ZK_CFG_CLUSTER_NODES("clusterNodes"), /** * 集群中所有节点的名称信息的分隔符 * @字段说明 ZK_CFG_CLUSTER_NODES */ ZK_CFG_CLUSTER_NODES_SEPARATE(","), ; private ZkParamCfg(String key) { this.key = key; } private String key; public String getKey() { return key; } public void setKey(String key) { this.key = key; } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/comm/ZookeeperProcessListen.java ================================================ package io.mycat.config.loader.zkprocess.comm; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.config.loader.console.ZookeeperPath; import io.mycat.config.loader.zkprocess.console.ZkNofiflyCfg; /** * 进行zookeeper操作的监控器器父类信息 * * @author liujun * * @date 2015年2月4日 * @vsersion 0.0.1 */ public class ZookeeperProcessListen { /** * 日志 * @字段说明 LOGGER */ private static final Logger lOG = LoggerFactory.getLogger(ZookeeperProcessListen.class); /** * 所有更新缓存操作的集合 */ private Map listenCache = new HashMap(); /** * 监控的路径信息 * @字段说明 watchPath */ private Map> watchPathMap = new HashMap<>(); /** * 监控路径对应的缓存key的对应表 * @字段说明 watchToListen */ private Map watchToListenMap = new HashMap<>(); /** * 基本路径信息 * @字段说明 basePath */ private String basePath; public String getBasePath() { return basePath; } public void setBasePath(String basePath) { this.basePath = basePath; } /** * 添加缓存更新操作 * * @param key * @param cacheNotiflySercie */ public void addListen(String key, NotiflyService cacheNotiflySercie) { listenCache.put(key, cacheNotiflySercie); } /** * 专门针对zk设置的监控路径 * 方法描述 * @param key * @param path * @param cacheNotiflySercie * @创建日期 2016年9月19日 */ public void watchPath(String key, String path) { Set watchPaths = watchPathMap.get(key); if (null == watchPaths) { watchPaths = new HashSet<>(); } watchPaths.add(path); watchPathMap.put(key, watchPaths); } /** * 进行监控路径的转换 * 方法描述 * @创建日期 2016年9月20日 */ public void watchToParse() { if (null != watchPathMap && !watchPathMap.isEmpty()) { for (Entry> watchPathEntry : watchPathMap.entrySet()) { for (String path : watchPathEntry.getValue()) { watchToListenMap.put(watchPathEntry.getKey() + ZookeeperPath.ZK_SEPARATOR.getKey() + path, watchPathEntry.getKey()); } } } } /** * 返回路径集合 * 方法描述 * @return * @创建日期 2016年9月19日 */ public Set getWatchPath() { if (watchToListenMap.isEmpty()) { this.watchToParse(); } return watchToListenMap.keySet(); } /** * 进行缓存更新通知 * * @param key * 缓存模块的key * @return true 当前缓存模块数据更新成功，false，当前缓存数据更新失败 */ public boolean notifly(String key) { boolean result = false; if (null != key && !"".equals(key)) { // 进行配制加载所有 if (ZkNofiflyCfg.ZK_NOTIFLY_LOAD_ALL.getKey().equals(key)) { this.notiflyAll(); } // 如果是具体的单独更新，则进行单业务的业务刷新 else { String watchListen = watchToListenMap.get(key); if (null != watchListen) { // 取得具体的业务监听信息 NotiflyService cacheService = listenCache.get(watchListen); if (null != cacheService) { try { result = cacheService.notiflyProcess(); } catch (Exception e) { lOG.error("ZookeeperProcessListen notifly key :" + key + " error:Exception info:", e); } } } } } return result; } /** * 进行通知所有缓存进行更新操作 */ private void notiflyAll() { Iterator> notiflyIter = listenCache.entrySet().iterator(); Entry item = null; while (notiflyIter.hasNext()) { item = notiflyIter.next(); // 进行缓存更新通知操作 if (null != item.getValue()) { try { item.getValue().notiflyProcess(); } catch (Exception e) { lOG.error("ZookeeperProcessListen notiflyAll key :" + item.getKey() + ";value " + item.getValue() + ";error:Exception info:", e); } } } } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/console/ParseParamEnum.java ================================================ package io.mycat.config.loader.zkprocess.console; /** * 转换的流程参数配制信息 * 源文件名：ParseParamEnum.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月18日 * 修改作者：liujun * 修改日期：2016年9月18日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public enum ParseParamEnum { /** * mapfile配制的参数名 * @字段说明 ZK_PATH_RULE_MAPFILE_NAME */ ZK_PATH_RULE_MAPFILE_NAME("mapFile"), ; /** * 配制的key的信息 * @字段说明 key */ private String key; private ParseParamEnum(String key) { this.key = key; } public String getKey() { return key; } public void setKey(String key) { this.key = key; } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/console/ZkNofiflyCfg.java ================================================ package io.mycat.config.loader.zkprocess.console; /** * 进行zk通知的参数配制信息 * 源文件名：ZkNofiflyCfg.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public enum ZkNofiflyCfg { /** * 通知更新所有节点的信息 * @字段说明 ZK_NOTIFLY_LOAD_ALL */ ZK_NOTIFLY_LOAD_ALL("all"), ; /** * 配制的key的信息 * @字段说明 key */ private String key; private ZkNofiflyCfg(String key) { this.key = key; } public String getKey() { return key; } public void setKey(String key) { this.key = key; } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/Named.java ================================================ package io.mycat.config.loader.zkprocess.entity; /** * presentation a object have a filed name. */ public interface Named { /** * 获得属性的名称 * 方法描述 * @return * @创建日期 2016年9月15日 */ String getName(); } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/Propertied.java ================================================ package io.mycat.config.loader.zkprocess.entity; /** * Created by lion on 12/8/15. */ public interface Propertied { void addProperty(Property property); } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/Property.java ================================================ package io.mycat.config.loader.zkprocess.entity; import java.util.Objects; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlType; import javax.xml.bind.annotation.XmlValue; /** * 键值对信息 * 源文件名：Property.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月16日 * 修改作者：liujun * 修改日期：2016年9月16日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ @XmlAccessorType(XmlAccessType.FIELD) @XmlType(name = "Property") public class Property implements Named { @XmlValue protected String value; @XmlAttribute(name = "name") protected String name; public String getValue() { return value; } public Property setValue(String value) { this.value = value; return this; } public String getName() { return name; } public Property setName(String value) { this.name = value; return this; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } Property property = (Property) o; return value.equals(property.value) && name.equals(property.name); } @Override public int hashCode() { return Objects.hash(value, name); } @Override public String toString() { return "Property{" + "value='" + value + '\'' + ", name='" + name + '\'' + '}'; } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/Rules.java ================================================ package io.mycat.config.loader.zkprocess.entity; import java.util.ArrayList; import java.util.List; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; import io.mycat.config.loader.zkprocess.entity.rule.function.Function; import io.mycat.config.loader.zkprocess.entity.rule.tablerule.TableRule; @XmlAccessorType(XmlAccessType.FIELD) @XmlRootElement(namespace = "http://io.mycat/", name = "rule") public class Rules { /** * 表的路由配制信息 * @字段说明 tableRule */ protected List tableRule; /** * 指定的方法信息 * @字段说明 function */ protected List function; public List getTableRule() { if (this.tableRule == null) { tableRule = new ArrayList<>(); } return tableRule; } public void setTableRule(List tableRule) { this.tableRule = tableRule; } public List getFunction() { if (this.function == null) { function = new ArrayList<>(); } return function; } public void setFunction(List function) { this.function = function; } @Override public String toString() { StringBuilder builder = new StringBuilder(); builder.append("Rules [tableRule="); builder.append(tableRule); builder.append(", function="); builder.append(function); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/Schemas.java ================================================ package io.mycat.config.loader.zkprocess.entity; import java.util.ArrayList; import java.util.List; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; import io.mycat.config.loader.zkprocess.entity.schema.datahost.DataHost; import io.mycat.config.loader.zkprocess.entity.schema.datanode.DataNode; import io.mycat.config.loader.zkprocess.entity.schema.schema.Schema; @XmlAccessorType(XmlAccessType.FIELD) @XmlRootElement(namespace = "http://io.mycat/", name = "schema") public class Schemas { /** * 配制的逻辑表信息 * @字段说明 schema */ private List schema; /** * 配制的表对应的数据库信息 * @字段说明 dataNode */ private List dataNode; /** * 用于指定数据信息 * @字段说明 dataHost */ private List dataHost; public List getSchema() { if (this.schema == null) { schema = new ArrayList<>(); } return schema; } public void setSchema(List schema) { this.schema = schema; } public List getDataNode() { if (this.dataNode == null) { dataNode = new ArrayList<>(); } return dataNode; } public void setDataNode(List dataNode) { this.dataNode = dataNode; } public List getDataHost() { if (this.dataHost == null) { dataHost = new ArrayList<>(); } return dataHost; } public void setDataHost(List dataHost) { this.dataHost = dataHost; } @Override public String toString() { StringBuilder builder = new StringBuilder(); builder.append("Schemas [schema="); builder.append(schema); builder.append(", dataNode="); builder.append(dataNode); builder.append(", dataHost="); builder.append(dataHost); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/Server.java ================================================ package io.mycat.config.loader.zkprocess.entity; import java.util.List; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; import io.mycat.config.loader.zkprocess.entity.server.System; import io.mycat.config.loader.zkprocess.entity.server.user.User; @XmlAccessorType(XmlAccessType.FIELD) @XmlRootElement(namespace = "http://io.mycat/", name = "server") public class Server { @XmlElement(required = true) protected System system; @XmlElement(required = true) protected List user; public System getSystem() { return system; } public void setSystem(System value) { this.system = value; } public List getUser() { return user; } public void setUser(List user) { this.user = user; } @Override public String toString() { StringBuilder builder = new StringBuilder(); builder.append("Server [system="); builder.append(system); builder.append(", user="); builder.append(user); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/cache/CacheInfo.java ================================================ package io.mycat.config.loader.zkprocess.entity.cache; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlRootElement; /** * 缓存配制信息 * 源文件名：CacheInfo.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月19日 * 修改作者：liujun * 修改日期：2016年9月19日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ @XmlAccessorType(XmlAccessType.NONE) @XmlRootElement(name = "defaultCache") public class CacheInfo { /** * maxElementsInMemory:在内存中最大的对象数量 * @字段说明 maxEntriesLocalHeap */ @XmlAttribute private int maxElementsInMemory; /** * eternal：设置元素是否永久的，如果为永久，则timeout忽略 * @字段说明 maxBytesLocalDisk */ @XmlAttribute private boolean eternal; /** * overflowToDisk：是否当memory中的数量达到限制后，保存到Disk * @字段说明 updateCheck */ @XmlAttribute private boolean overflowToDisk; /** * diskSpoolBufferSizeMB：这个参数设置DiskStore（磁盘缓存）的缓存区大小。默认是30MB。每个Cache都应该有自己的一个缓冲区。 * @字段说明 diskSpoolBufferSizeMB */ @XmlAttribute private int diskSpoolBufferSizeMB; /** * maxElementsOnDisk：硬盘最大缓存个数。 * @字段说明 maxElementsOnDisk */ @XmlAttribute private int maxElementsOnDisk; /** * diskPersistent：是否缓存虚拟机重启期数据 * @字段说明 diskPersistent */ @XmlAttribute private boolean diskPersistent; /** * diskExpiryThreadIntervalSeconds：磁盘失效线程运行时间间隔，默认是120秒。 * @字段说明 diskExpiryThreadIntervalSeconds */ @XmlAttribute private int diskExpiryThreadIntervalSeconds; /** * memoryStoreEvictionPolicy：当达到maxElementsInMemory限制时， * Ehcache将会根据指定的策略去清理内存。默认策略是LRU（最近最少使用）。 * 你可以设置为FIFO（先进先出）或是LFU（较少使用）。 * @字段说明 memoryStoreEvictionPolicy */ @XmlAttribute private String memoryStoreEvictionPolicy; public int getMaxElementsInMemory() { return maxElementsInMemory; } public void setMaxElementsInMemory(int maxElementsInMemory) { this.maxElementsInMemory = maxElementsInMemory; } public boolean isEternal() { return eternal; } public void setEternal(boolean eternal) { this.eternal = eternal; } public boolean isOverflowToDisk() { return overflowToDisk; } public void setOverflowToDisk(boolean overflowToDisk) { this.overflowToDisk = overflowToDisk; } public int getDiskSpoolBufferSizeMB() { return diskSpoolBufferSizeMB; } public void setDiskSpoolBufferSizeMB(int diskSpoolBufferSizeMB) { this.diskSpoolBufferSizeMB = diskSpoolBufferSizeMB; } public int getMaxElementsOnDisk() { return maxElementsOnDisk; } public void setMaxElementsOnDisk(int maxElementsOnDisk) { this.maxElementsOnDisk = maxElementsOnDisk; } public boolean isDiskPersistent() { return diskPersistent; } public void setDiskPersistent(boolean diskPersistent) { this.diskPersistent = diskPersistent; } public int getDiskExpiryThreadIntervalSeconds() { return diskExpiryThreadIntervalSeconds; } public void setDiskExpiryThreadIntervalSeconds(int diskExpiryThreadIntervalSeconds) { this.diskExpiryThreadIntervalSeconds = diskExpiryThreadIntervalSeconds; } public String getMemoryStoreEvictionPolicy() { return memoryStoreEvictionPolicy; } public void setMemoryStoreEvictionPolicy(String memoryStoreEvictionPolicy) { this.memoryStoreEvictionPolicy = memoryStoreEvictionPolicy; } @Override public String toString() { StringBuilder builder = new StringBuilder(); builder.append("CacheInfo [maxElementsInMemory="); builder.append(maxElementsInMemory); builder.append(", eternal="); builder.append(eternal); builder.append(", overflowToDisk="); builder.append(overflowToDisk); builder.append(", diskSpoolBufferSizeMB="); builder.append(diskSpoolBufferSizeMB); builder.append(", maxElementsOnDisk="); builder.append(maxElementsOnDisk); builder.append(", diskPersistent="); builder.append(diskPersistent); builder.append(", diskExpiryThreadIntervalSeconds="); builder.append(diskExpiryThreadIntervalSeconds); builder.append(", memoryStoreEvictionPolicy="); builder.append(memoryStoreEvictionPolicy); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/cache/Ehcache.java ================================================ package io.mycat.config.loader.zkprocess.entity.cache; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; /** * ehcache配制信息 * 源文件名：Ehcache.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月19日 * 修改作者：liujun * 修改日期：2016年9月19日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ @XmlAccessorType(XmlAccessType.FIELD) @XmlRootElement(name = "ehcache") public class Ehcache { /** * * @字段说明 maxEntriesLocalHeap */ @XmlAttribute private int maxEntriesLocalHeap; /** * @字段说明 maxBytesLocalDisk */ @XmlAttribute private String maxBytesLocalDisk; /** * @字段说明 updateCheck */ @XmlAttribute private boolean updateCheck; /** * 缓存信息 * @字段说明 defaultCache */ @XmlElement private CacheInfo defaultCache; public int getMaxEntriesLocalHeap() { return maxEntriesLocalHeap; } public void setMaxEntriesLocalHeap(int maxEntriesLocalHeap) { this.maxEntriesLocalHeap = maxEntriesLocalHeap; } public String getMaxBytesLocalDisk() { return maxBytesLocalDisk; } public void setMaxBytesLocalDisk(String maxBytesLocalDisk) { this.maxBytesLocalDisk = maxBytesLocalDisk; } public boolean isUpdateCheck() { return updateCheck; } public void setUpdateCheck(boolean updateCheck) { this.updateCheck = updateCheck; } public CacheInfo getDefaultCache() { return defaultCache; } public void setDefaultCache(CacheInfo defaultCache) { this.defaultCache = defaultCache; } @Override public String toString() { StringBuilder builder = new StringBuilder(); builder.append("Ehcache [maxEntriesLocalHeap="); builder.append(maxEntriesLocalHeap); builder.append(", maxBytesLocalDisk="); builder.append(maxBytesLocalDisk); builder.append(", updateCheck="); builder.append(updateCheck); builder.append(", defaultCache="); builder.append(defaultCache); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/package-info.java ================================================ @XmlSchema(xmlns = @XmlNs(prefix = "mycat", namespaceURI = "http://io.mycat/") , elementFormDefault = javax.xml.bind.annotation.XmlNsForm.QUALIFIED) package io.mycat.config.loader.zkprocess.entity; import javax.xml.bind.annotation.XmlNs; import javax.xml.bind.annotation.XmlSchema; ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/rule/function/Function.java ================================================ package io.mycat.config.loader.zkprocess.entity.rule.function; import java.util.ArrayList; import java.util.List; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlType; import io.mycat.config.loader.zkprocess.entity.Named; import io.mycat.config.loader.zkprocess.entity.Propertied; import io.mycat.config.loader.zkprocess.entity.Property; /** * * * 3 * * 源文件名：Function.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月18日 * 修改作者：liujun * 修改日期：2016年9月18日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ @XmlAccessorType(XmlAccessType.FIELD) @XmlType(name = "function") public class Function implements Propertied, Named { @XmlAttribute(required = true) protected String name; @XmlAttribute(required = true, name = "class") protected String clazz; protected List property; public String getName() { return name; } public void setName(String name) { this.name = name; } public String getClazz() { return clazz; } public void setClazz(String clazz) { this.clazz = clazz; } public List getProperty() { if (this.property == null) { property = new ArrayList<>(); } return property; } public void setProperty(List property) { this.property = property; } @Override public void addProperty(Property property) { this.getProperty().add(property); } @Override public String toString() { StringBuilder builder = new StringBuilder(); builder.append("Function [name="); builder.append(name); builder.append(", clazz="); builder.append(clazz); builder.append(", property="); builder.append(property); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/rule/tablerule/Rule.java ================================================ package io.mycat.config.loader.zkprocess.entity.rule.tablerule; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlType; /** * * * * *id * * *func1 * * * 源文件名：Rule.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月18日 * 修改作者：liujun * 修改日期：2016年9月18日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ @XmlAccessorType(XmlAccessType.FIELD) @XmlType(name = "rule", propOrder = { "columns", "algorithm" }) public class Rule { protected String columns; protected String algorithm; public String getColumns() { return columns; } public Rule setColumns(String columns) { this.columns = columns; return this; } public String getAlgorithm() { return algorithm; } public Rule setAlgorithm(String algorithm) { this.algorithm = algorithm; return this; } @Override public String toString() { StringBuilder builder = new StringBuilder(); builder.append("Rule [columns="); builder.append(columns); builder.append(", algorithm="); builder.append(algorithm); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/rule/tablerule/TableRule.java ================================================ package io.mycat.config.loader.zkprocess.entity.rule.tablerule; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlType; import io.mycat.config.loader.zkprocess.entity.Named; /** * * * * * * * * *id * * * *func1 * * * * 源文件名：TableRule.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月18日 * 修改作者：liujun * 修改日期：2016年9月18日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ @XmlAccessorType(XmlAccessType.FIELD) @XmlType(name = "tableRule") public class TableRule implements Named { @XmlElement(required = true, name = "rule") protected Rule rule; @XmlAttribute(required = true) protected String name; public Rule getRule() { return rule; } public TableRule setRule(Rule rule) { this.rule = rule; return this; } public String getName() { return name; } public TableRule setName(String name) { this.name = name; return this; } @Override public String toString() { StringBuilder builder = new StringBuilder(); builder.append("TableRule [rule="); builder.append(rule); builder.append(", name="); builder.append(name); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/schema/datahost/DataHost.java ================================================ package io.mycat.config.loader.zkprocess.entity.schema.datahost; import java.util.ArrayList; import java.util.List; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlType; import io.mycat.config.loader.zkprocess.entity.Named; /** * * * 源文件名：DataHost.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ @XmlAccessorType(XmlAccessType.FIELD) @XmlType(name = "dataHost") public class DataHost implements Named { @XmlAttribute(required = true) protected Integer balance; @XmlAttribute(required = true) protected Integer balanceType; @XmlAttribute(required = true) protected Integer maxCon; @XmlAttribute(required = true) protected Integer minCon; @XmlAttribute(required = true) protected String name; @XmlAttribute protected Integer writeType; @XmlAttribute protected Integer switchType; @XmlAttribute protected Integer slaveThreshold; @XmlAttribute(required = true) protected String dbType; @XmlAttribute(required = true) protected String dbDriver; @XmlAttribute() protected String slaveIDs; @XmlAttribute(required = true) protected String maxRetryCount; protected String heartbeat; protected String connectionInitSql; protected List writeHost; public String getMaxRetryCount() { return maxRetryCount; } public void setMaxRetryCount(String maxRetryCount) { this.maxRetryCount = maxRetryCount; } public String getHeartbeat() { return heartbeat; } public void setHeartbeat(String heartbeat) { this.heartbeat = heartbeat; } public String getConnectionInitSql() { return connectionInitSql; } public void setConnectionInitSql(String connectionInitSql) { this.connectionInitSql = connectionInitSql; } public List getWriteHost() { if (this.writeHost == null) { writeHost = new ArrayList<>(); } return writeHost; } public String getSlaveIDs() { return slaveIDs; } public void setSlaveIDs(String slaveIDs) { this.slaveIDs = slaveIDs; } public void setWriteHost(List writeHost) { this.writeHost = writeHost; } public String getName() { return name; } public void setName(String name) { this.name = name; } public Integer getMaxCon() { return maxCon; } public void setMaxCon(Integer maxCon) { this.maxCon = maxCon; } public Integer getMinCon() { return minCon; } public void setMinCon(Integer minCon) { this.minCon = minCon; } public Integer getBalance() { return balance; } public void setBalance(Integer balance) { this.balance = balance; } public Integer getBalanceType() { return balanceType; } public void setBalanceType(Integer balanceType) { this.balanceType = balanceType; } public String getDbType() { return dbType; } public void setDbType(String dbType) { this.dbType = dbType; } public String getDbDriver() { return dbDriver; } public void setDbDriver(String dbDriver) { this.dbDriver = dbDriver; } public Integer getWriteType() { return writeType; } public void setWriteType(Integer writeType) { this.writeType = writeType; } public Integer getSwitchType() { return switchType; } public void setSwitchType(Integer switchType) { this.switchType = switchType; } public Integer getSlaveThreshold() { return slaveThreshold; } public void setSlaveThreshold(Integer slaveThreshold) { this.slaveThreshold = slaveThreshold; } @Override public String toString() { StringBuilder builder = new StringBuilder(); builder.append("DataHost [balance="); builder.append(balance); builder.append(", balanceType="); builder.append(balanceType); builder.append(", maxCon="); builder.append(maxCon); builder.append(", minCon="); builder.append(minCon); builder.append(", name="); builder.append(name); builder.append(", writeType="); builder.append(writeType); builder.append(", switchType="); builder.append(switchType); builder.append(", slaveThreshold="); builder.append(slaveThreshold); builder.append(", dbType="); builder.append(dbType); builder.append(", dbDriver="); builder.append(dbDriver); builder.append(", heartbeat="); builder.append(heartbeat); builder.append(", connectionInitSql="); builder.append(connectionInitSql); builder.append(", writeHost="); builder.append(writeHost); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/schema/datahost/ReadHost.java ================================================ package io.mycat.config.loader.zkprocess.entity.schema.datahost; import java.util.List; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.XmlType; /** * * 源文件名：ReadHost.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ @XmlAccessorType(XmlAccessType.FIELD) @XmlType(name = "readHost") public class ReadHost extends WriteHost { @XmlAttribute protected String weight; public String getWeight() { return weight; } public void setWeight(String weight) { this.weight = weight; } @XmlTransient @Override public List getReadHost() { return super.getReadHost(); } @Override public String toString() { StringBuilder builder = new StringBuilder(); builder.append("ReadHost [weight="); builder.append(weight); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/schema/datahost/WriteHost.java ================================================ package io.mycat.config.loader.zkprocess.entity.schema.datahost; import java.util.ArrayList; import java.util.List; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlType; /** * * 源文件名：WriteHost.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ @XmlAccessorType(XmlAccessType.FIELD) @XmlType(name = "writeHost") public class WriteHost { @XmlAttribute(required = true) protected String host; @XmlAttribute(required = true) protected String url; @XmlAttribute(required = true) protected String password; @XmlAttribute(required = true) protected String user; @XmlAttribute protected Boolean usingDecrypt; @XmlAttribute protected Boolean checkAlive; private List readHost; public String getHost() { return host; } public void setHost(String host) { this.host = host; } public String getUrl() { return url; } public void setUrl(String url) { this.url = url; } public String getPassword() { return password; } public void setPassword(String password) { this.password = password; } public String getUser() { return user; } public void setUser(String user) { this.user = user; } public Boolean isUsingDecrypt() { return usingDecrypt; } public void setUsingDecrypt(Boolean usingDecrypt) { this.usingDecrypt = usingDecrypt; } public List getReadHost() { if (this.readHost == null) { readHost = new ArrayList<>(); } return readHost; } public void setReadHost(List readHost) { this.readHost = readHost; } @Override public String toString() { StringBuilder builder = new StringBuilder(); builder.append("WriteHost [host="); builder.append(host); builder.append(", url="); builder.append(url); builder.append(", password="); builder.append(password); builder.append(", user="); builder.append(user); builder.append(", usingDecrypt="); builder.append(usingDecrypt); builder.append(", readHost="); builder.append(readHost); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/schema/datanode/DataNode.java ================================================ package io.mycat.config.loader.zkprocess.entity.schema.datanode; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlType; import io.mycat.config.loader.zkprocess.entity.Named; /** * * 源文件名：DataNode.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ @XmlAccessorType(XmlAccessType.FIELD) @XmlType(name = "dataNode") public class DataNode implements Named { @XmlAttribute(required = true) private String name; @XmlAttribute(required = true) private String dataHost; @XmlAttribute(required = true) private String database; public String getName() { return name; } public void setName(String name) { this.name = name; } public String getDataHost() { return dataHost; } public void setDataHost(String dataHost) { this.dataHost = dataHost; } public String getDatabase() { return database; } public void setDatabase(String database) { this.database = database; } @Override public String toString() { StringBuilder builder = new StringBuilder(); builder.append("DataNode [name="); builder.append(name); builder.append(", dataHost="); builder.append(dataHost); builder.append(", database="); builder.append(database); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/schema/schema/ChildTable.java ================================================ package io.mycat.config.loader.zkprocess.entity.schema.schema; import java.util.ArrayList; import java.util.List; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlType; import io.mycat.config.loader.zkprocess.entity.Named; /** * * * 配制子表信息 * 源文件名：ChildTable.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ @XmlAccessorType(XmlAccessType.FIELD) @XmlType(name = "childTable") public class ChildTable implements Named { @XmlAttribute(required = true) protected String name; @XmlAttribute(required = true) protected String joinKey; @XmlAttribute(required = true) protected String parentKey; @XmlAttribute protected String primaryKey; @XmlAttribute protected Boolean autoIncrement; protected List childTable; public String getName() { return name; } public void setName(String name) { this.name = name; } public String getJoinKey() { return joinKey; } public void setJoinKey(String joinKey) { this.joinKey = joinKey; } public String getParentKey() { return parentKey; } public void setParentKey(String parentKey) { this.parentKey = parentKey; } public String getPrimaryKey() { return primaryKey; } public void setPrimaryKey(String primaryKey) { this.primaryKey = primaryKey; } public Boolean isAutoIncrement() { return autoIncrement; } public void setAutoIncrement(Boolean autoIncrement) { this.autoIncrement = autoIncrement; } public List getChildTable() { if (this.childTable == null) { childTable = new ArrayList<>(); } return childTable; } public void setChildTable(List childTable) { this.childTable = childTable; } @Override public String toString() { StringBuilder builder = new StringBuilder(); builder.append("ChildTable [name="); builder.append(name); builder.append(", joinKey="); builder.append(joinKey); builder.append(", parentKey="); builder.append(parentKey); builder.append(", primaryKey="); builder.append(primaryKey); builder.append(", autoIncrement="); builder.append(autoIncrement); builder.append(", childTable="); builder.append(childTable); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/schema/schema/Schema.java ================================================ package io.mycat.config.loader.zkprocess.entity.schema.schema; import java.util.ArrayList; import java.util.List; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlType; import io.mycat.config.loader.zkprocess.entity.Named; /** * * *
* * * * * 源文件名：Schema.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ @XmlAccessorType(XmlAccessType.FIELD) @XmlType(name = "schema") public class Schema implements Named { /** * schema的名称 * @字段说明 name */ @XmlAttribute(required = true) protected String name; /** * 当诠值讴置为 true 时， * 如果我们执行询句**select * from TESTDB.travelrecord; * **则MyCat会把询句修改为**select * from travelrecord;** * @字段说明 checkSQLschema */ @XmlAttribute protected Boolean checkSQLschema; /** * 当诠值设置为某个数值时。每条执行癿SQL询句，如果没有加上limit询句，MyCat也会自劢癿加上所对应癿 * @字段说明 sqlMaxLimit */ @XmlAttribute protected Integer sqlMaxLimit; /** * 诠属性用二绊定逡辑库刡某个具体癿database上， * 1.3版本如果配置了dataNode，则不可以配置分片表， * 1.4可以配置默讣分片，叧雹要配置需要分片的表即可 * @字段说明 dataNode */ @XmlAttribute protected String dataNode; /** * 配制表信息 * @字段说明 table */ protected List
table; public String getName() { return name; } public void setName(String name) { this.name = name; } public Boolean isCheckSQLschema() { return checkSQLschema; } public void setCheckSQLschema(Boolean checkSQLschema) { this.checkSQLschema = checkSQLschema; } public Integer getSqlMaxLimit() { return sqlMaxLimit; } public void setSqlMaxLimit(Integer sqlMaxLimit) { this.sqlMaxLimit = sqlMaxLimit; } public String getDataNode() { return dataNode; } public void setDataNode(String dataNode) { this.dataNode = dataNode; } public List
getTable() { if (this.table == null) { table = new ArrayList<>(); } return table; } public void setTable(List
table) { this.table = table; } @Override public String toString() { StringBuilder builder = new StringBuilder(); builder.append("Schema [name="); builder.append(name); builder.append(", checkSQLschema="); builder.append(checkSQLschema); builder.append(", sqlMaxLimit="); builder.append(sqlMaxLimit); builder.append(", dataNode="); builder.append(dataNode); builder.append(", table="); builder.append(table); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/schema/schema/Table.java ================================================ package io.mycat.config.loader.zkprocess.entity.schema.schema; import java.util.ArrayList; import java.util.List; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlType; import io.mycat.config.loader.zkprocess.entity.Named; /** *
* 用于具体的表信息 * 源文件名：Table.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ @XmlAccessorType(XmlAccessType.FIELD) @XmlType(name = "table") public class Table implements Named { @XmlAttribute(required = true) protected String name; @XmlAttribute protected String nameSuffix; @XmlAttribute(required = true) protected String dataNode; @XmlAttribute protected String rule; @XmlAttribute protected Boolean ruleRequired; @XmlAttribute protected String primaryKey; @XmlAttribute protected Boolean autoIncrement; @XmlAttribute protected Boolean needAddLimit; @XmlAttribute protected String type; @XmlAttribute protected String subTables; /** * 子节点信息 * @字段说明 childTable */ protected List childTable; public String getName() { return name; } public void setName(String name) { this.name = name; } public String getDataNode() { return dataNode; } public void setDataNode(String dataNode) { this.dataNode = dataNode; } public String getRule() { return rule; } public void setRule(String rule) { this.rule = rule; } public List getChildTable() { if (this.childTable == null) { childTable = new ArrayList<>(); } return childTable; } public void setChildTable(List childTable) { this.childTable = childTable; } public String getNameSuffix() { return nameSuffix; } public void setNameSuffix(String nameSuffix) { this.nameSuffix = nameSuffix; } public Boolean isRuleRequired() { return ruleRequired; } public void setRuleRequired(Boolean ruleRequired) { this.ruleRequired = ruleRequired; } public String getPrimaryKey() { return primaryKey; } public void setPrimaryKey(String primaryKey) { this.primaryKey = primaryKey; } public Boolean isAutoIncrement() { return autoIncrement; } public void setAutoIncrement(Boolean autoIncrement) { this.autoIncrement = autoIncrement; } public Boolean isNeedAddLimit() { return needAddLimit; } public void setNeedAddLimit(Boolean needAddLimit) { this.needAddLimit = needAddLimit; } public String getType() { return type; } public void setType(String type) { this.type = type; } public String getSubTables() { return subTables; } public void setSubTables(String subTables) { this.subTables = subTables; } @Override public String toString() { StringBuilder builder = new StringBuilder(); builder.append("Table [name="); builder.append(name); builder.append(", nameSuffix="); builder.append(nameSuffix); builder.append(", dataNode="); builder.append(dataNode); builder.append(", rule="); builder.append(rule); builder.append(", ruleRequired="); builder.append(ruleRequired); builder.append(", primaryKey="); builder.append(primaryKey); builder.append(", autoIncrement="); builder.append(autoIncrement); builder.append(", needAddLimit="); builder.append(needAddLimit); builder.append(", type="); builder.append(type); builder.append(", childTable="); builder.append(childTable); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/server/System.java ================================================ package io.mycat.config.loader.zkprocess.entity.server; import java.util.ArrayList; import java.util.List; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlType; import io.mycat.config.loader.zkprocess.entity.Propertied; import io.mycat.config.loader.zkprocess.entity.Property; /** * 系统信息 * 源文件名：System.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月16日 * 修改作者：liujun * 修改日期：2016年9月16日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ @XmlAccessorType(XmlAccessType.FIELD) @XmlType(name = "system") public class System implements Propertied { protected List property; public List getProperty() { if (this.property == null) { property = new ArrayList<>(); } return property; } public void setProperty(List property) { this.property = property; } @Override public void addProperty(Property property) { this.getProperty().add(property); } /** * 设置最新的方法值 * 方法描述 * @param newSet * @创建日期 2016年9月17日 */ public void setNewValue(System newSet) { if (null != newSet) { List valuePro = newSet.getProperty(); // 最新设置的属性值 for (Property netsetProper : valuePro) { // 当前已经设置的属性值 for (Property property : this.getProperty()) { // 如果新设置的属性名称与当前的已经存在的名称相同，则设置为新值 if (netsetProper.getName().equals(property.getName())) { property.setValue(netsetProper.getValue()); } } } } } @Override public String toString() { StringBuilder builder = new StringBuilder(); builder.append("System [property="); builder.append(property); builder.append("]"); return builder.toString(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/entity/server/user/User.java ================================================ package io.mycat.config.loader.zkprocess.entity.server.user; import java.util.ArrayList; import java.util.List; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlType; import io.mycat.config.loader.zkprocess.entity.Named; import io.mycat.config.loader.zkprocess.entity.Propertied; import io.mycat.config.loader.zkprocess.entity.Property; @XmlAccessorType(XmlAccessType.FIELD) @XmlType(name = "user") public class User implements Propertied, Named { @XmlAttribute(required = true) protected String name; protected List property; public String getName() { return name; } public void setName(String name) { this.name = name; } public List getProperty() { if (this.property == null) { property = new ArrayList<>(); } return property; } public void setProperty(List property) { this.property = property; } @Override public void addProperty(Property property) { this.getProperty().add(property); } @Override public String toString() { return "User{" + "name='" + name + '\'' + ", property=" + property + '}'; } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/parse/JsonProcessBase.java ================================================ package io.mycat.config.loader.zkprocess.parse; import java.lang.reflect.Type; import java.util.ArrayList; import java.util.List; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import io.mycat.config.loader.zkprocess.entity.Schemas; import io.mycat.config.loader.zkprocess.entity.schema.datanode.DataNode; /** * json数据与实体类的类的信息 * 源文件名：XmlProcessBase.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class JsonProcessBase { /** * 进行消息转换的类的信息 * @字段说明 gson */ private Gson gson = new Gson(); /** * 进行json字符串化 * 方法描述 * @param obj * @return * @创建日期 2016年9月17日 */ public String toJsonFromBean(Object obj) { if (null != obj) { return gson.toJson(obj); } return null; } /** * 将json字符串至类，根据指定的类型信息,一般用于集合的转换 * 方法描述 * @param json * @param typeSchema * @return * @创建日期 2016年9月17日 */ public T toBeanformJson(String json, Type typeSchema) { T result = this.gson.fromJson(json, typeSchema); return result; } /** * 将json字符串至类，根据指定的类型信息,用于转换单对象实体 * 方法描述 * @param * @param json * @param typeSchema * @return * @创建日期 2016年9月17日 */ public T toBeanformJson(String json, Class classinfo) { T result = this.gson.fromJson(json, classinfo); return result; } public static void main(String[] args) { DataNode datanode = new DataNode(); datanode.setDatabase("db1"); datanode.setDataHost("os1"); datanode.setName("dn1"); JsonProcessBase jsonParse = new JsonProcessBase(); String jsonStr = jsonParse.toJsonFromBean(datanode); System.out.println("单对象当前的json:" + jsonStr); // 转换实体 DataNode node = jsonParse.toBeanformJson(jsonStr, DataNode.class); System.out.println("单对象:" + node); List listNode = new ArrayList<>(); listNode.add(datanode); listNode.add(datanode); String listJson = jsonParse.toJsonFromBean(listNode); System.out.println("当前集合的json:" + listJson); // 转换为集合的bean Type parseType = new TypeToken>() { }.getType(); List list = jsonParse.toBeanformJson(listJson, parseType); System.out.println("集合对象:" + list); // 复杂对象的转换 Schemas schema = new Schemas(); schema.setDataNode(listNode); String jsonMultStr = jsonParse.toJsonFromBean(schema); System.out.println("复杂单对象当前的json:" + jsonMultStr); // 转换实体 Schemas nodeMult = jsonParse.toBeanformJson(jsonMultStr, Schemas.class); System.out.println("复杂单对象:" + nodeMult); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/parse/ParseJsonServiceInf.java ================================================ package io.mycat.config.loader.zkprocess.parse; /** * json转化服务 * 源文件名：JsonParseServiceInf.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月16日 * 修改作者：liujun * 修改日期：2016年9月16日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public interface ParseJsonServiceInf { /** * 将对象T转换为json字符串 * 方法描述 * @param data * @return * @创建日期 2016年9月16日 */ public String parseBeanToJson(T t); /** * 将json字符串转换为javabean对象 * 方法描述 * @param json * @return * @创建日期 2016年9月16日 */ public T parseJsonToBean(String json); } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/parse/ParseXmlServiceInf.java ================================================ package io.mycat.config.loader.zkprocess.parse; /** *xml转化服务 * 源文件名：JsonParseServiceInf.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月16日 * 修改作者：liujun * 修改日期：2016年9月16日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public interface ParseXmlServiceInf { /** * 将对象T写入xml文件 * 方法描述 * @param data * @return * @创建日期 2016年9月16日 */ public void parseToXmlWrite(T data, String outputPath, String dataName); /** * 将指定的xml转换为javabean对象 * 方法描述 * @param path xml文件路径信息 * @return * @创建日期 2016年9月16日 */ public T parseXmlToBean(String path); } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/parse/XmlProcessBase.java ================================================ package io.mycat.config.loader.zkprocess.parse; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Map.Entry; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Marshaller; import javax.xml.bind.Unmarshaller; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import javax.xml.transform.stream.StreamSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * xml文件操作转换的类的信息 * 源文件名：XmlProcessBase.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class XmlProcessBase { /** * 日志 * @字段说明 LOGGER */ private static final Logger lOG = LoggerFactory.getLogger(XmlProcessBase.class); /** * 转换对象 * @字段说明 jaxContext */ private JAXBContext jaxContext; /** * 反序列化xml文件的对象 * @字段说明 unmarshaller */ private Unmarshaller unmarshaller; /** * 转换的实体对象的class信息 * @字段说明 parseXmlClass */ @SuppressWarnings("rawtypes") public List parseXmlClass = new ArrayList(); /** * 添加转换的class信息 * 方法描述 * @param parseClass * @创建日期 2016年9月15日 */ @SuppressWarnings("rawtypes") public void addParseClass(Class parseClass) { this.parseXmlClass.add(parseClass); } /** * 进行jaxb对象的初始化 * 方法描述 * @throws JAXBException * @创建日期 2016年9月15日 */ @SuppressWarnings("rawtypes") public void initJaxbClass() throws JAXBException { // 将集合转换为数组 Class[] classArray = new Class[parseXmlClass.size()]; parseXmlClass.toArray(classArray); try { this.jaxContext = JAXBContext.newInstance(classArray, Collections. emptyMap()); } catch (JAXBException e) { lOG.error("ZookeeperProcessListen initJaxbClass error:Exception info:", e); throw e; } // 创建解反序化对象 unmarshaller = jaxContext.createUnmarshaller(); } /** * 默认将bean序列化为xml对象信息并写入文件 * 方法描述 * @param user 用户对象 * @param inputPath * @param name 当前的转换xml的dtd文件的信息 * @创建日期 2016年9月15日 */ public void baseParseAndWriteToXml(Object user, String inputPath, String name) throws IOException { try { Marshaller marshaller = this.jaxContext.createMarshaller(); marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); marshaller.setProperty(Marshaller.JAXB_FRAGMENT, Boolean.TRUE); if (null != name) { marshaller.setProperty("com.sun.xml.internal.bind.xmlHeaders", String.format("", name)); } Path path = Paths.get(inputPath); OutputStream out = Files.newOutputStream(path, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); marshaller.marshal(user, out); } catch (JAXBException e) { lOG.error("ZookeeperProcessListen parseToXml error:Exception info:", e); } catch (IOException e) { lOG.error("ZookeeperProcessListen parseToXml error:Exception info:", e); } } /** * 默认将bean序列化为xml对象信息并写入文件 * 方法描述 * @param user 用户对象 * @param inputPath * @param name 当前的转换xml的dtd文件的信息 * @创建日期 2016年9月15日 */ @SuppressWarnings("restriction") public void baseParseAndWriteToXml(Object user, String inputPath, String name, Map map) throws IOException { try { Marshaller marshaller = this.jaxContext.createMarshaller(); marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); marshaller.setProperty(Marshaller.JAXB_FRAGMENT, Boolean.TRUE); if (null != name) { marshaller.setProperty("com.sun.xml.internal.bind.xmlHeaders", String.format("", name)); } if (null != map && !map.isEmpty()) { for (Entry entry : map.entrySet()) { marshaller.setProperty(entry.getKey(), entry.getValue()); } } Path path = Paths.get(inputPath); OutputStream out = Files.newOutputStream(path, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); marshaller.marshal(user, out); } catch (JAXBException e) { lOG.error("ZookeeperProcessListen parseToXml error:Exception info:", e); } catch (IOException e) { lOG.error("ZookeeperProcessListen parseToXml error:Exception info:", e); } } /** * 默认转换将指定的xml转化为 * 方法描述 * @param inputStream * @param fileName * @return * @throws JAXBException * @throws XMLStreamException * @创建日期 2016年9月16日 */ public Object baseParseXmlToBean(String fileName) throws JAXBException, XMLStreamException { // 搜索当前转化的文件 InputStream inputStream = XmlProcessBase.class.getResourceAsStream(fileName); // 如果能够搜索到文件 if (inputStream != null) { // 进行文件反序列化信息 XMLInputFactory xif = XMLInputFactory.newFactory(); xif.setProperty(XMLInputFactory.SUPPORT_DTD, false); XMLStreamReader xmlRead = xif.createXMLStreamReader(new StreamSource(inputStream)); return unmarshaller.unmarshal(xmlRead); } return null; } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/parse/entryparse/cache/json/EhcacheJsonParse.java ================================================ package io.mycat.config.loader.zkprocess.parse.entryparse.cache.json; import io.mycat.config.loader.zkprocess.entity.cache.Ehcache; import io.mycat.config.loader.zkprocess.parse.JsonProcessBase; import io.mycat.config.loader.zkprocess.parse.ParseJsonServiceInf; /** * 进行Function节点的转换 * 源文件名：FunctionJsonParse.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月17日 * 修改作者：liujun * 修改日期：2016年9月17日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class EhcacheJsonParse extends JsonProcessBase implements ParseJsonServiceInf { @Override public String parseBeanToJson(Ehcache t) { return this.toJsonFromBean(t); } @Override public Ehcache parseJsonToBean(String json) { return this.toBeanformJson(json, Ehcache.class); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/parse/entryparse/cache/xml/EhcacheParseXmlImpl.java ================================================ package io.mycat.config.loader.zkprocess.parse.entryparse.cache.xml; import java.io.IOException; import java.util.HashMap; import java.util.Map; import javax.xml.bind.JAXBException; import javax.xml.bind.Marshaller; import javax.xml.stream.XMLStreamException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.config.loader.zkprocess.entity.cache.Ehcache; import io.mycat.config.loader.zkprocess.parse.ParseXmlServiceInf; import io.mycat.config.loader.zkprocess.parse.XmlProcessBase; /** * rule.xml与javabean之间的转化 * 源文件名：SchemasParseXmlImpl.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月16日 * 修改作者：liujun * 修改日期：2016年9月16日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class EhcacheParseXmlImpl implements ParseXmlServiceInf { /** * 日志 * @字段说明 LOGGER */ private static final Logger lOG = LoggerFactory.getLogger(EhcacheParseXmlImpl.class); /** * 基本的转换类的信息 * @字段说明 parseBean */ private XmlProcessBase parseBean; /** * 转换的类的信息 * 构造方法 * @param parseBase */ public EhcacheParseXmlImpl(XmlProcessBase parseBase) { this.parseBean = parseBase; // 添加xml的转换的实体类信息 parseBean.addParseClass(Ehcache.class); } @Override public Ehcache parseXmlToBean(String path) { Ehcache schema = null; try { schema = (Ehcache) this.parseBean.baseParseXmlToBean(path); } catch (JAXBException e) { e.printStackTrace(); lOG.error("EhcacheParseXmlImpl parseXmlToBean JAXBException", e); } catch (XMLStreamException e) { e.printStackTrace(); lOG.error("EhcacheParseXmlImpl parseXmlToBean XMLStreamException", e); } return schema; } @Override public void parseToXmlWrite(Ehcache data, String outputFile, String dataName) { try { // 设置 Map paramMap = new HashMap<>(); paramMap.put(Marshaller.JAXB_NO_NAMESPACE_SCHEMA_LOCATION, "ehcache.xsd"); this.parseBean.baseParseAndWriteToXml(data, outputFile, dataName, paramMap); } catch (IOException e) { e.printStackTrace(); lOG.error("EhcacheParseXmlImpl parseToXmlWrite IOException", e); } } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/parse/entryparse/rule/json/FunctionJsonParse.java ================================================ package io.mycat.config.loader.zkprocess.parse.entryparse.rule.json; import java.lang.reflect.Type; import java.util.List; import com.google.gson.reflect.TypeToken; import io.mycat.config.loader.zkprocess.entity.rule.function.Function; import io.mycat.config.loader.zkprocess.parse.JsonProcessBase; import io.mycat.config.loader.zkprocess.parse.ParseJsonServiceInf; /** * 进行Function节点的转换 * 源文件名：FunctionJsonParse.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月17日 * 修改作者：liujun * 修改日期：2016年9月17日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class FunctionJsonParse extends JsonProcessBase implements ParseJsonServiceInf> { @Override public String parseBeanToJson(List t) { return this.toJsonFromBean(t); } @Override public List parseJsonToBean(String json) { // 转换为集合的bean Type parseType = new TypeToken>() { }.getType(); return this.toBeanformJson(json, parseType); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/parse/entryparse/rule/json/TableRuleJsonParse.java ================================================ package io.mycat.config.loader.zkprocess.parse.entryparse.rule.json; import java.lang.reflect.Type; import java.util.List; import com.google.gson.reflect.TypeToken; import io.mycat.config.loader.zkprocess.entity.rule.tablerule.TableRule; import io.mycat.config.loader.zkprocess.parse.JsonProcessBase; import io.mycat.config.loader.zkprocess.parse.ParseJsonServiceInf; /** * 进行TableRule节点的转换 * 源文件名：TableRuleJsonParse.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月17日 * 修改作者：liujun * 修改日期：2016年9月17日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class TableRuleJsonParse extends JsonProcessBase implements ParseJsonServiceInf> { @Override public String parseBeanToJson(List t) { return this.toJsonFromBean(t); } @Override public List parseJsonToBean(String json) { // 转换为集合的bean Type parseType = new TypeToken>() { }.getType(); return this.toBeanformJson(json, parseType); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/parse/entryparse/rule/xml/RuleParseXmlImpl.java ================================================ package io.mycat.config.loader.zkprocess.parse.entryparse.rule.xml; import java.io.IOException; import javax.xml.bind.JAXBException; import javax.xml.stream.XMLStreamException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.config.loader.zkprocess.entity.Rules; import io.mycat.config.loader.zkprocess.parse.ParseXmlServiceInf; import io.mycat.config.loader.zkprocess.parse.XmlProcessBase; /** * rule.xml与javabean之间的转化 * 源文件名：SchemasParseXmlImpl.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月16日 * 修改作者：liujun * 修改日期：2016年9月16日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class RuleParseXmlImpl implements ParseXmlServiceInf { /** * 日志 * @字段说明 LOGGER */ private static final Logger lOG = LoggerFactory.getLogger(RuleParseXmlImpl.class); /** * 基本的转换类的信息 * @字段说明 parseBean */ private XmlProcessBase parseBean; /** * 转换的类的信息 * 构造方法 * @param parseBase */ public RuleParseXmlImpl(XmlProcessBase parseBase) { this.parseBean = parseBase; // 添加xml的转换的实体类信息 parseBean.addParseClass(Rules.class); } @Override public Rules parseXmlToBean(String path) { Rules schema = null; try { schema = (Rules) this.parseBean.baseParseXmlToBean(path); } catch (JAXBException e) { e.printStackTrace(); lOG.error("RulesParseXmlImpl parseXmlToBean JAXBException", e); } catch (XMLStreamException e) { e.printStackTrace(); lOG.error("RulesParseXmlImpl parseXmlToBean XMLStreamException", e); } return schema; } @Override public void parseToXmlWrite(Rules data, String outputFile, String dataName) { try { this.parseBean.baseParseAndWriteToXml(data, outputFile, dataName); } catch (IOException e) { e.printStackTrace(); lOG.error("RulesParseXmlImpl parseToXmlWrite IOException", e); } } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/parse/entryparse/schema/json/DataHostJsonParse.java ================================================ package io.mycat.config.loader.zkprocess.parse.entryparse.schema.json; import java.lang.reflect.Type; import java.util.List; import com.google.gson.reflect.TypeToken; import io.mycat.config.loader.zkprocess.entity.schema.datahost.DataHost; import io.mycat.config.loader.zkprocess.parse.JsonProcessBase; import io.mycat.config.loader.zkprocess.parse.ParseJsonServiceInf; /** * 进行datahost节点的转换 * 源文件名：DataHostJsonParse.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月17日 * 修改作者：liujun * 修改日期：2016年9月17日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class DataHostJsonParse extends JsonProcessBase implements ParseJsonServiceInf> { @Override public String parseBeanToJson(List t) { return this.toJsonFromBean(t); } @Override public List parseJsonToBean(String json) { // 转换为集合的bean Type parseType = new TypeToken>() { }.getType(); return this.toBeanformJson(json, parseType); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/parse/entryparse/schema/json/DataNodeJsonParse.java ================================================ package io.mycat.config.loader.zkprocess.parse.entryparse.schema.json; import java.lang.reflect.Type; import java.util.List; import com.google.gson.reflect.TypeToken; import io.mycat.config.loader.zkprocess.entity.schema.datanode.DataNode; import io.mycat.config.loader.zkprocess.parse.JsonProcessBase; import io.mycat.config.loader.zkprocess.parse.ParseJsonServiceInf; /** * 进行将datanode数据与json的转化 * 源文件名：DataNodeJsonParse.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月17日 * 修改作者：liujun * 修改日期：2016年9月17日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class DataNodeJsonParse extends JsonProcessBase implements ParseJsonServiceInf> { @Override public String parseBeanToJson(List t) { return this.toJsonFromBean(t); } @Override public List parseJsonToBean(String json) { // 转换为集合的bean Type parseType = new TypeToken>() { }.getType(); return this.toBeanformJson(json, parseType); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/parse/entryparse/schema/json/SchemaJsonParse.java ================================================ package io.mycat.config.loader.zkprocess.parse.entryparse.schema.json; import java.lang.reflect.Type; import java.util.List; import com.google.gson.reflect.TypeToken; import io.mycat.config.loader.zkprocess.entity.schema.schema.Schema; import io.mycat.config.loader.zkprocess.parse.JsonProcessBase; import io.mycat.config.loader.zkprocess.parse.ParseJsonServiceInf; /** * 进行schema部分的转换 * 源文件名：SchemaJsonParse.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月17日 * 修改作者：liujun * 修改日期：2016年9月17日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class SchemaJsonParse extends JsonProcessBase implements ParseJsonServiceInf> { @Override public String parseBeanToJson(List t) { return this.toJsonFromBean(t); } @Override public List parseJsonToBean(String json) { // 转换为集合的bean Type parseType = new TypeToken>() { }.getType(); return this.toBeanformJson(json, parseType); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/parse/entryparse/schema/xml/SchemasParseXmlImpl.java ================================================ package io.mycat.config.loader.zkprocess.parse.entryparse.schema.xml; import java.io.IOException; import javax.xml.bind.JAXBException; import javax.xml.stream.XMLStreamException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.config.loader.zkprocess.entity.Schemas; import io.mycat.config.loader.zkprocess.parse.ParseXmlServiceInf; import io.mycat.config.loader.zkprocess.parse.XmlProcessBase; /** * schema.xml与javabean之间的转化 * 源文件名：SchemasParseXmlImpl.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月16日 * 修改作者：liujun * 修改日期：2016年9月16日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class SchemasParseXmlImpl implements ParseXmlServiceInf { /** * 日志 * @字段说明 LOGGER */ private static final Logger lOG = LoggerFactory.getLogger(SchemasParseXmlImpl.class); /** * 基本的转换类的信息 * @字段说明 parseBean */ private XmlProcessBase parseBean; /** * 转换的类的信息 * 构造方法 * @param parseBase */ public SchemasParseXmlImpl(XmlProcessBase parseBase) { this.parseBean = parseBase; // 添加xml的转换的实体类信息 parseBean.addParseClass(Schemas.class); } @Override public Schemas parseXmlToBean(String path) { Schemas schema = null; try { schema = (Schemas) this.parseBean.baseParseXmlToBean(path); } catch (JAXBException e) { e.printStackTrace(); lOG.error("SchemasParseXmlImpl parseXmlToBean JAXBException", e); } catch (XMLStreamException e) { e.printStackTrace(); lOG.error("SchemasParseXmlImpl parseXmlToBean XMLStreamException", e); } return schema; } @Override public void parseToXmlWrite(Schemas data, String outputFile, String dataName) { try { this.parseBean.baseParseAndWriteToXml(data, outputFile, dataName); } catch (IOException e) { e.printStackTrace(); lOG.error("SchemasParseXmlImpl parseToXmlWrite IOException", e); } } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/parse/entryparse/server/json/SystemJsonParse.java ================================================ package io.mycat.config.loader.zkprocess.parse.entryparse.server.json; import io.mycat.config.loader.zkprocess.entity.server.System; import io.mycat.config.loader.zkprocess.parse.JsonProcessBase; import io.mycat.config.loader.zkprocess.parse.ParseJsonServiceInf; /** * 进行datahost节点的转换 * 源文件名：DataHostJsonParse.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月17日 * 修改作者：liujun * 修改日期：2016年9月17日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class SystemJsonParse extends JsonProcessBase implements ParseJsonServiceInf { @Override public String parseBeanToJson(System t) { return this.toJsonFromBean(t); } @Override public System parseJsonToBean(String json) { return this.toBeanformJson(json, System.class); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/parse/entryparse/server/json/UserJsonParse.java ================================================ package io.mycat.config.loader.zkprocess.parse.entryparse.server.json; import java.lang.reflect.Type; import java.util.List; import com.google.gson.reflect.TypeToken; import io.mycat.config.loader.zkprocess.entity.server.user.User; import io.mycat.config.loader.zkprocess.parse.JsonProcessBase; import io.mycat.config.loader.zkprocess.parse.ParseJsonServiceInf; /** * 进行datahost节点的转换 * 源文件名：DataHostJsonParse.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月17日 * 修改作者：liujun * 修改日期：2016年9月17日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class UserJsonParse extends JsonProcessBase implements ParseJsonServiceInf> { @Override public String parseBeanToJson(List t) { return this.toJsonFromBean(t); } @Override public List parseJsonToBean(String json) { // 转换为集合的bean Type parseType = new TypeToken>() { }.getType(); return this.toBeanformJson(json, parseType); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/parse/entryparse/server/xml/ServerParseXmlImpl.java ================================================ package io.mycat.config.loader.zkprocess.parse.entryparse.server.xml; import java.io.IOException; import javax.xml.bind.JAXBException; import javax.xml.stream.XMLStreamException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.config.loader.zkprocess.entity.Server; import io.mycat.config.loader.zkprocess.parse.ParseXmlServiceInf; import io.mycat.config.loader.zkprocess.parse.XmlProcessBase; /** * schema.xml与javabean之间的转化 * 源文件名：ServerParseXmlImpl.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月16日 * 修改作者：liujun * 修改日期：2016年9月16日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class ServerParseXmlImpl implements ParseXmlServiceInf { /** * 日志 * @字段说明 LOGGER */ private static final Logger lOG = LoggerFactory.getLogger(ServerParseXmlImpl.class); /** * 基本的转换类的信息 * @字段说明 parseBean */ private XmlProcessBase parseBean; /** * 转换的类的信息 * 构造方法 * @param parseBase */ public ServerParseXmlImpl(XmlProcessBase parseBase) { this.parseBean = parseBase; // 添加xml的转换的实体类信息 parseBean.addParseClass(Server.class); } @Override public Server parseXmlToBean(String path) { Server server = null; try { server = (Server) this.parseBean.baseParseXmlToBean(path); } catch (JAXBException e) { e.printStackTrace(); lOG.error("ServerParseXmlImpl parseXmlToBean JAXBException", e); } catch (XMLStreamException e) { e.printStackTrace(); lOG.error("ServerParseXmlImpl parseXmlToBean XMLStreamException", e); } return server; } @Override public void parseToXmlWrite(Server data, String outputFile, String dataName) { try { this.parseBean.baseParseAndWriteToXml(data, outputFile, dataName); } catch (IOException e) { e.printStackTrace(); lOG.error("ServerParseXmlImpl parseToXmlWrite IOException", e); } } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/xmltozk/BindataToZK.java ================================================ package io.mycat.config.loader.zkprocess.xmltozk; import com.google.common.io.Files; import io.mycat.config.loader.console.ZookeeperPath; import io.mycat.config.loader.zkprocess.zktoxml.listen.RuleszkToxmlLoader; import io.mycat.config.model.SystemConfig; import io.mycat.util.ZKUtils; import org.apache.curator.framework.CuratorFramework; import java.io.File; import java.nio.file.Paths; /** * Created by magicdoom on 2016/10/26. * only for test */ public class BindataToZK { public static void main(String[] args) { File file = new File(SystemConfig.getHomePath()+ "/conf","ruledata" ); if(file.exists()&&file.isDirectory()) { File[] binFiles=file.listFiles(); for (File binFile : binFiles) { String path= ZKUtils.getZKBasePath()+"ruledata/"+binFile.getName(); CuratorFramework zk= ZKUtils.getConnection(); try { zk.create().creatingParentsIfNeeded().forPath(path) ; zk.setData().forPath(path, Files.toByteArray(binFile)) ; } catch (Exception e) { e.printStackTrace(); } } } } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/xmltozk/XmltoZkMain.java ================================================ package io.mycat.config.loader.zkprocess.xmltozk; import javax.xml.bind.JAXBException; import com.alibaba.fastjson.JSON; import io.mycat.config.loader.zkprocess.zookeeper.ClusterInfo; import org.apache.curator.framework.CuratorFramework; import io.mycat.config.loader.console.ZookeeperPath; import io.mycat.config.loader.zkprocess.comm.ZkConfig; import io.mycat.config.loader.zkprocess.comm.ZkParamCfg; import io.mycat.config.loader.zkprocess.comm.ZookeeperProcessListen; import io.mycat.config.loader.zkprocess.console.ZkNofiflyCfg; import io.mycat.config.loader.zkprocess.parse.XmlProcessBase; import io.mycat.config.loader.zkprocess.xmltozk.listen.EcachesxmlTozkLoader; import io.mycat.config.loader.zkprocess.xmltozk.listen.OthermsgTozkLoader; import io.mycat.config.loader.zkprocess.xmltozk.listen.RulesxmlTozkLoader; import io.mycat.config.loader.zkprocess.xmltozk.listen.SchemasxmlTozkLoader; import io.mycat.config.loader.zkprocess.xmltozk.listen.SequenceTozkLoader; import io.mycat.config.loader.zkprocess.xmltozk.listen.ServerxmlTozkLoader; import io.mycat.util.ZKUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class XmltoZkMain { private static final Logger LOGGER = LoggerFactory.getLogger(XmltoZkMain.class); public static void main(String[] args) throws JAXBException, InterruptedException { // 加载zk总服务 ZookeeperProcessListen zkListen = new ZookeeperProcessListen(); // 得到集群名称 String custerName = ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_CLUSTERID); // 得到基本路径 String basePath = ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_BASE.getKey(); basePath = basePath + ZookeeperPath.ZK_SEPARATOR.getKey() + custerName; zkListen.setBasePath(basePath); // 获得zk的连接信息 CuratorFramework zkConn = buildConnection(ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_URL)); // 获得公共的xml转换器对象 XmlProcessBase xmlProcess = new XmlProcessBase(); // 进行xmltozk的schema文件的操作 new SchemasxmlTozkLoader(zkListen, zkConn, xmlProcess); // 进行xmltozk的server文件的操作 new ServerxmlTozkLoader(zkListen, zkConn, xmlProcess); // 进行rule文件到zk的操作 new RulesxmlTozkLoader(zkListen, zkConn, xmlProcess); // 进行序列信息入zk中 new SequenceTozkLoader(zkListen, zkConn, xmlProcess); // 缓存配制信息 new EcachesxmlTozkLoader(zkListen, zkConn, xmlProcess); // 将其他信息加载的zk中 new OthermsgTozkLoader(zkListen, zkConn, xmlProcess); // 初始化xml转换操作 xmlProcess.initJaxbClass(); // 加载通知进程 zkListen.notifly(ZkNofiflyCfg.ZK_NOTIFLY_LOAD_ALL.getKey()); String clusterNodes= ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_CLUSTER_NODES); String clusterSize= ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_CLUSTER_SIZE); ClusterInfo info=new ClusterInfo(); info.setClusterNodes(clusterNodes); info.setClusterSize(Integer.parseInt(clusterSize)); try { zkConn.setData().forPath(basePath, JSON.toJSONBytes(info)); } catch (Exception e) { LOGGER.error("error",e); } } private static CuratorFramework buildConnection(String url) { return ZKUtils.getConnection(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/xmltozk/listen/EcachesxmlTozkLoader.java ================================================ package io.mycat.config.loader.zkprocess.xmltozk.listen; import static com.google.common.base.Preconditions.checkNotNull; import java.io.IOException; import java.io.InputStream; import org.apache.curator.framework.CuratorFramework; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.fastjson.util.IOUtils; import io.mycat.config.loader.console.ZookeeperPath; import io.mycat.config.loader.zkprocess.comm.NotiflyService; import io.mycat.config.loader.zkprocess.comm.ZookeeperProcessListen; import io.mycat.config.loader.zkprocess.entity.cache.Ehcache; import io.mycat.config.loader.zkprocess.parse.ParseJsonServiceInf; import io.mycat.config.loader.zkprocess.parse.ParseXmlServiceInf; import io.mycat.config.loader.zkprocess.parse.XmlProcessBase; import io.mycat.config.loader.zkprocess.parse.entryparse.cache.json.EhcacheJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.cache.xml.EhcacheParseXmlImpl; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkMultLoader; /** * 进行从ecache.xml加载到zk中加载 * 源文件名：SchemasLoader.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class EcachesxmlTozkLoader extends ZkMultLoader implements NotiflyService { /** * 日志 * @字段说明 LOGGER */ private static final Logger LOGGER = LoggerFactory.getLogger(EcachesxmlTozkLoader.class); /** * 当前文件中的zkpath信息 * @字段说明 currZkPath */ private final String currZkPath; /** * Ehcache文件的路径信息 * @字段说明 SCHEMA_PATH */ private static final String EHCACHE_PATH = ZookeeperPath.ZK_LOCAL_CFG_PATH.getKey() + "ehcache.xml"; /** * 缓存文件名称 * @字段说明 CACHESERVER_NAME */ private static final String CACHESERVER_NAME = "cacheservice.properties"; /** * 缓存的xml文件配制信息 * @字段说明 EHCACHE_NAME */ private static final String EHCACHE_NAME = "ehcache.xml"; /** * ehcache的xml的转换信息 * @字段说明 parseEhcacheXMl */ private final ParseXmlServiceInf parseEcacheXMl; /** * 表的路由信息 * @字段说明 parseJsonService */ private ParseJsonServiceInf parseJsonEhcacheService = new EhcacheJsonParse(); public EcachesxmlTozkLoader(ZookeeperProcessListen zookeeperListen, CuratorFramework curator, XmlProcessBase xmlParseBase) { this.setCurator(curator); // 获得当前集群的名称 String schemaPath = zookeeperListen.getBasePath(); schemaPath = schemaPath + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_CACHE.getKey(); currZkPath = schemaPath; // 将当前自己注册为事件接收对象 zookeeperListen.addListen(schemaPath, this); // 生成xml与类的转换信息 parseEcacheXMl = new EhcacheParseXmlImpl(xmlParseBase); } @Override public boolean notiflyProcess() throws Exception { // 1,读取本地的xml文件 Ehcache Ehcache = this.parseEcacheXMl.parseXmlToBean(EHCACHE_PATH); LOGGER.info("EhcachexmlTozkLoader notiflyProcessxml to zk Ehcache Object :" + Ehcache); // 将实体信息写入至zk中 this.xmlTozkEhcacheJson(currZkPath, Ehcache); LOGGER.info("EhcachexmlTozkLoader notiflyProcess xml to zk is success"); return true; } /** * 将xml文件的信息写入到zk中 * 方法描述 * @param basePath 基本路径 * @param schema schema文件的信息 * @throws Exception 异常信息 * @创建日期 2016年9月17日 */ private void xmlTozkEhcacheJson(String basePath, Ehcache ehcache) throws Exception { // ehcache节点信息 String ehcacheFile = ZookeeperPath.ZK_SEPARATOR.getKey() + EHCACHE_NAME; String ehcacheJson = this.parseJsonEhcacheService.parseBeanToJson(ehcache); this.checkAndwriteString(basePath, ehcacheFile, ehcacheJson); // 读取文件信息 String cacheServicePath = ZookeeperPath.ZK_SEPARATOR.getKey() + CACHESERVER_NAME; String serviceValue = this.readSeqFile(CACHESERVER_NAME); this.checkAndwriteString(basePath, cacheServicePath, serviceValue); } /** * 读取 mapFile文件的信息 * 方法描述 * @param name 名称信息 * @return * @创建日期 2016年9月18日 */ private String readSeqFile(String name) { StringBuilder mapFileStr = new StringBuilder(); String path = ZookeeperPath.ZK_LOCAL_CFG_PATH.getKey() + name; // 加载数据 InputStream input = EcachesxmlTozkLoader.class.getResourceAsStream(path); checkNotNull(input, "read SeqFile file curr Path :" + path + " is null! must is not null"); byte[] buffers = new byte[256]; try { int readIndex = -1; while ((readIndex = input.read(buffers)) != -1) { mapFileStr.append(new String(buffers, 0, readIndex)); } } catch (IOException e) { e.printStackTrace(); LOGGER.error("EhcachexmlTozkLoader readMapFile IOException", e); } finally { IOUtils.close(input); } return mapFileStr.toString(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/xmltozk/listen/OthermsgTozkLoader.java ================================================ package io.mycat.config.loader.zkprocess.xmltozk.listen; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.utils.ZKPaths; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.config.loader.console.ZookeeperPath; import io.mycat.config.loader.zkprocess.comm.NotiflyService; import io.mycat.config.loader.zkprocess.comm.ZookeeperProcessListen; import io.mycat.config.loader.zkprocess.parse.XmlProcessBase; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkMultLoader; /** * 其他一些信息加载到zk中 * 源文件名：SchemasLoader.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class OthermsgTozkLoader extends ZkMultLoader implements NotiflyService { /** * 日志 * @字段说明 LOGGER */ private static final Logger LOGGER = LoggerFactory.getLogger(OthermsgTozkLoader.class); /** * 当前文件中的zkpath信息 * @字段说明 currZkPath */ private final String currZkPath; public OthermsgTozkLoader(ZookeeperProcessListen zookeeperListen, CuratorFramework curator, XmlProcessBase xmlParseBase) { this.setCurator(curator); // 获得当前集群的名称 String schemaPath = zookeeperListen.getBasePath(); currZkPath = schemaPath; // 将当前自己注册为事件接收对象 zookeeperListen.addListen(schemaPath, this); } @Override public boolean notiflyProcess() throws Exception { // 添加line目录，用作集群中节点，在线的基本目录信息 String line = currZkPath + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_LINE.getKey(); ZKPaths.mkdirs(this.getCurator().getZookeeperClient().getZooKeeper(), line); LOGGER.info("OthermsgTozkLoader zookeeper mkdir " + line + " success"); // 添加序列目录信息 String seqLine = currZkPath + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SEQUENCE.getKey(); seqLine = seqLine + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SEQUENCE_INSTANCE.getKey(); ZKPaths.mkdirs(this.getCurator().getZookeeperClient().getZooKeeper(), seqLine); String seqLeader = currZkPath + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SEQUENCE.getKey(); seqLeader = seqLeader + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SEQUENCE_LEADER.getKey(); ZKPaths.mkdirs(this.getCurator().getZookeeperClient().getZooKeeper(), seqLeader); String incrSeq = currZkPath + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SEQUENCE.getKey(); incrSeq = incrSeq + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SEQUENCE_INCREMENT_SEQ.getKey(); ZKPaths.mkdirs(this.getCurator().getZookeeperClient().getZooKeeper(), incrSeq); LOGGER.info("OthermsgTozkLoader zookeeper mkdir " + seqLine + " success"); LOGGER.info("OthermsgTozkLoader zookeeper mkdir " + seqLeader + " success"); LOGGER.info("OthermsgTozkLoader zookeeper mkdir " + incrSeq + " success"); return true; } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/xmltozk/listen/RulesxmlTozkLoader.java ================================================ package io.mycat.config.loader.zkprocess.xmltozk.listen; import static com.google.common.base.Preconditions.checkNotNull; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import org.apache.curator.framework.CuratorFramework; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.fastjson.util.IOUtils; import io.mycat.config.loader.console.ZookeeperPath; import io.mycat.config.loader.zkprocess.comm.NotiflyService; import io.mycat.config.loader.zkprocess.comm.ZookeeperProcessListen; import io.mycat.config.loader.zkprocess.console.ParseParamEnum; import io.mycat.config.loader.zkprocess.entity.Property; import io.mycat.config.loader.zkprocess.entity.Rules; import io.mycat.config.loader.zkprocess.entity.rule.function.Function; import io.mycat.config.loader.zkprocess.entity.rule.tablerule.TableRule; import io.mycat.config.loader.zkprocess.parse.ParseJsonServiceInf; import io.mycat.config.loader.zkprocess.parse.ParseXmlServiceInf; import io.mycat.config.loader.zkprocess.parse.XmlProcessBase; import io.mycat.config.loader.zkprocess.parse.entryparse.rule.json.FunctionJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.rule.json.TableRuleJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.rule.xml.RuleParseXmlImpl; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkMultLoader; /** * 进行从rule.xml加载到zk中加载 * 源文件名：SchemasLoader.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class RulesxmlTozkLoader extends ZkMultLoader implements NotiflyService { /** * 日志 * @字段说明 LOGGER */ private static final Logger LOGGER = LoggerFactory.getLogger(RulesxmlTozkLoader.class); /** * 当前文件中的zkpath信息 * @字段说明 currZkPath */ private final String currZkPath; /** * Rules文件的路径信息 * @字段说明 SCHEMA_PATH */ private static final String RULE_PATH = ZookeeperPath.ZK_LOCAL_CFG_PATH.getKey() + "rule.xml"; /** * Rules的xml的转换信息 * @字段说明 parseRulesXMl */ private ParseXmlServiceInf parseRulesXMl; /** * 表的路由信息 * @字段说明 parseJsonService */ private ParseJsonServiceInf> parseJsonTableRuleService = new TableRuleJsonParse(); /** * 表对应的字段信息 * @字段说明 parseJsonFunctionService */ private ParseJsonServiceInf> parseJsonFunctionService = new FunctionJsonParse(); public RulesxmlTozkLoader(ZookeeperProcessListen zookeeperListen, CuratorFramework curator, XmlProcessBase xmlParseBase) { this.setCurator(curator); // 获得当前集群的名称 String schemaPath = zookeeperListen.getBasePath(); schemaPath = schemaPath + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_RULE.getKey(); currZkPath = schemaPath; // 将当前自己注册为事件接收对象 zookeeperListen.addListen(schemaPath, this); // 生成xml与类的转换信息 parseRulesXMl = new RuleParseXmlImpl(xmlParseBase); } @Override public boolean notiflyProcess() throws Exception { // 1,读取本地的xml文件 Rules Rules = this.parseRulesXMl.parseXmlToBean(RULE_PATH); LOGGER.info("RulesxmlTozkLoader notiflyProcessxml to zk Rules Object :" + Rules); // 将实体信息写入至zk中 this.xmlTozkRulesJson(currZkPath, Rules); LOGGER.info("RulesxmlTozkLoader notiflyProcess xml to zk is success"); return true; } /** * 将xml文件的信息写入到zk中 * 方法描述 * @param basePath 基本路径 * @param schema schema文件的信息 * @throws Exception 异常信息 * @创建日期 2016年9月17日 */ private void xmlTozkRulesJson(String basePath, Rules Rules) throws Exception { // tablerune节点信息 String tableRulePath = ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_RULE_TABLERULE.getKey(); String tableRuleJson = this.parseJsonTableRuleService.parseBeanToJson(Rules.getTableRule()); this.checkAndwriteString(basePath, tableRulePath, tableRuleJson); // 读取mapFile文件,并加入到function中 this.readMapFileAddFunction(Rules.getFunction()); // 方法设置信息 String functionPath = ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_RULE_FUNCTION.getKey(); String functionJson = this.parseJsonFunctionService.parseBeanToJson(Rules.getFunction()); this.checkAndwriteString(basePath, functionPath, functionJson); } /** * 读取序列配制文件便利店 * 方法描述 * @param functionList * @创建日期 2016年9月18日 */ private void readMapFileAddFunction(List functionList) { List tempData = new ArrayList<>(); for (Function function : functionList) { List proList = function.getProperty(); if (null != proList && !proList.isEmpty()) { // 进行数据遍历 for (Property property : proList) { // 如果为mapfile，则需要去读取数据信息，并存到json中 if (ParseParamEnum.ZK_PATH_RULE_MAPFILE_NAME.getKey().equals(property.getName())) { Property mapFilePro = new Property(); mapFilePro.setName(property.getValue()); // 加载属性的值信息 mapFilePro.setValue(this.readMapFile(property.getValue())); tempData.add(mapFilePro); } } // 将数据添加的集合中 proList.addAll(tempData); // 清空，以进行下一次的添加 tempData.clear(); } } } /** * 读取 mapFile文件的信息 * 方法描述 * @param name 名称信息 * @return * @创建日期 2016年9月18日 */ private String readMapFile(String name) { StringBuilder mapFileStr = new StringBuilder(); String path = ZookeeperPath.ZK_LOCAL_CFG_PATH.getKey() + name; // 加载数据 InputStream input = RulesxmlTozkLoader.class.getResourceAsStream(path); checkNotNull(input, "read Map file curr Path :" + path + " is null! must is not null"); byte[] buffers = new byte[256]; try { int readIndex = -1; while ((readIndex = input.read(buffers)) != -1) { mapFileStr.append(new String(buffers, 0, readIndex)); } } catch (IOException e) { e.printStackTrace(); LOGGER.error("RulesxmlTozkLoader readMapFile IOException", e); } finally { IOUtils.close(input); } return mapFileStr.toString(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/xmltozk/listen/SchemasxmlTozkLoader.java ================================================ package io.mycat.config.loader.zkprocess.xmltozk.listen; import java.util.List; import org.apache.curator.framework.CuratorFramework; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.config.loader.console.ZookeeperPath; import io.mycat.config.loader.zkprocess.comm.ZookeeperProcessListen; import io.mycat.config.loader.zkprocess.entity.Schemas; import io.mycat.config.loader.zkprocess.entity.schema.datahost.DataHost; import io.mycat.config.loader.zkprocess.entity.schema.datanode.DataNode; import io.mycat.config.loader.zkprocess.entity.schema.schema.Schema; import io.mycat.config.loader.zkprocess.comm.NotiflyService; import io.mycat.config.loader.zkprocess.parse.ParseJsonServiceInf; import io.mycat.config.loader.zkprocess.parse.ParseXmlServiceInf; import io.mycat.config.loader.zkprocess.parse.XmlProcessBase; import io.mycat.config.loader.zkprocess.parse.entryparse.schema.json.DataHostJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.schema.json.DataNodeJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.schema.json.SchemaJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.schema.xml.SchemasParseXmlImpl; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkMultLoader; /** * 进行从xml加载到zk中加载 * 源文件名：SchemasLoader.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class SchemasxmlTozkLoader extends ZkMultLoader implements NotiflyService { /** * 日志 * @字段说明 LOGGER */ private static final Logger LOGGER = LoggerFactory.getLogger(SchemasxmlTozkLoader.class); /** * 当前文件中的zkpath信息 * @字段说明 currZkPath */ private final String currZkPath; /** * schema文件的路径信息 * @字段说明 SCHEMA_PATH */ private static final String SCHEMA_PATH = ZookeeperPath.ZK_LOCAL_CFG_PATH.getKey() + "schema.xml"; /** * schema类与xml转换服务 * @字段说明 parseSchemaService */ private ParseXmlServiceInf parseSchemaXmlService; /** * 进行将schema * @字段说明 parseJsonSchema */ private ParseJsonServiceInf> parseJsonSchema = new SchemaJsonParse(); /** * 进行将dataNode * @字段说明 parseJsonSchema */ private ParseJsonServiceInf> parseJsonDataNode = new DataNodeJsonParse(); /** * 进行将dataNode * @字段说明 parseJsonSchema */ private ParseJsonServiceInf> parseJsonDataHost = new DataHostJsonParse(); public SchemasxmlTozkLoader(ZookeeperProcessListen zookeeperListen, CuratorFramework curator, XmlProcessBase xmlParseBase) { this.setCurator(curator); // 获得当前集群的名称 String schemaPath = zookeeperListen.getBasePath(); schemaPath = schemaPath + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FOW_ZK_PATH_SCHEMA.getKey(); currZkPath = schemaPath; // 将当前自己注册为事件接收对象 zookeeperListen.addListen(schemaPath, this); // 生成xml与类的转换信息 this.parseSchemaXmlService = new SchemasParseXmlImpl(xmlParseBase); } @Override public boolean notiflyProcess() throws Exception { // 1,读取本地的xml文件 Schemas schema = this.parseSchemaXmlService.parseXmlToBean(SCHEMA_PATH); LOGGER.info("SchemasxmlTozkLoader notiflyProcessxml to zk schema Object :" + schema); // 将实体信息写入至zk中 this.xmlTozkSchemasJson(currZkPath, schema); LOGGER.info("SchemasxmlTozkLoader notiflyProcess xml to zk is success"); return true; } /** * 将xml文件的信息写入到zk中 * 方法描述 * @param basePath 基本路径 * @param schema schema文件的信息 * @throws Exception 异常信息 * @创建日期 2016年9月17日 */ private void xmlTozkSchemasJson(String basePath, Schemas schema) throws Exception { // 设置schema目录的值 String schemaStr = ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SCHEMA_SCHEMA.getKey(); String schemaValueStr = this.parseJsonSchema.parseBeanToJson(schema.getSchema()); this.checkAndwriteString(basePath, schemaStr, schemaValueStr); // 设置datanode String dataNodeStr = ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SCHEMA_DATANODE.getKey(); String dataNodeValueStr = this.parseJsonDataNode.parseBeanToJson(schema.getDataNode()); this.checkAndwriteString(basePath, dataNodeStr, dataNodeValueStr); // 设置dataHost String dataHostStr = ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SCHEMA_DATAHOST.getKey(); String dataHostValueStr = this.parseJsonDataHost.parseBeanToJson(schema.getDataHost()); this.checkAndwriteString(basePath, dataHostStr, dataHostValueStr); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/xmltozk/listen/SequenceTozkLoader.java ================================================ package io.mycat.config.loader.zkprocess.xmltozk.listen; import java.io.IOException; import java.io.InputStream; import org.apache.curator.framework.CuratorFramework; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.fastjson.util.IOUtils; import io.mycat.config.loader.console.ZookeeperPath; import io.mycat.config.loader.zkprocess.comm.NotiflyService; import io.mycat.config.loader.zkprocess.comm.ZkConfig; import io.mycat.config.loader.zkprocess.comm.ZkParamCfg; import io.mycat.config.loader.zkprocess.comm.ZookeeperProcessListen; import io.mycat.config.loader.zkprocess.parse.XmlProcessBase; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkMultLoader; /** * 进行从sequence加载到zk中加载 * 源文件名：SchemasLoader.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class SequenceTozkLoader extends ZkMultLoader implements NotiflyService { /** * 日志 * @字段说明 LOGGER */ private static final Logger LOGGER = LoggerFactory.getLogger(SequenceTozkLoader.class); /** * 当前文件中的zkpath信息 * @字段说明 currZkPath */ private final String currZkPath; /** * 后缀名 * @字段说明 PROPERTIES_SUFFIX */ private static final String PROPERTIES_SUFFIX = ".properties"; /** * 序列配制信息 * @字段说明 PROPERTIES_SEQUENCE_CONF */ private static final String PROPERTIES_SEQUENCE_CONF = "sequence_conf"; /** * db序列配制信息 * @字段说明 PROPERTIES_SEQUENCE_CONF */ private static final String PROPERTIES_SEQUENCE_DB_CONF = "sequence_db_conf"; /** * 分布式的序列配制 * @字段说明 PROPERTIES_SEQUENCE_CONF */ private static final String PROPERTIES_SEQUENCE_DISTRIBUTED_CONF = "sequence_distributed_conf"; /** * 时间的序列配制 * @字段说明 PROPERTIES_SEQUENCE_CONF */ private static final String PROPERTIES_SEQUENCE_TIME_CONF = "sequence_time_conf"; public SequenceTozkLoader(ZookeeperProcessListen zookeeperListen, CuratorFramework curator, XmlProcessBase xmlParseBase) { this.setCurator(curator); // 获得当前集群的名称 String schemaPath = zookeeperListen.getBasePath(); schemaPath = schemaPath + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SEQUENCE.getKey(); currZkPath = schemaPath; // 将当前自己注册为事件接收对象 zookeeperListen.addListen(schemaPath, this); } @Override public boolean notiflyProcess() throws Exception { // 将zk序列配配制信息入zk this.sequenceTozk(currZkPath, PROPERTIES_SEQUENCE_CONF); LOGGER.info("SequenceTozkLoader notiflyProcess sequence_conf to zk success"); // 将zk的db方式信息入zk this.sequenceTozk(currZkPath, PROPERTIES_SEQUENCE_DB_CONF); LOGGER.info("SequenceTozkLoader notiflyProcess sequence_db_conf to zk success"); // 将zk的分布式信息入zk this.sequenceTozk(currZkPath, PROPERTIES_SEQUENCE_DISTRIBUTED_CONF); LOGGER.info("SequenceTozkLoader notiflyProcess sequence_distributed_conf to zk success"); // 将时间序列入zk this.sequenceTozk(currZkPath, PROPERTIES_SEQUENCE_TIME_CONF); LOGGER.info("SequenceTozkLoader notiflyProcess sequence_time_conf to zk success"); LOGGER.info("SequenceTozkLoader notiflyProcess xml to zk is success"); return true; } /** * 将xml文件的信息写入到zk中 * 方法描述 * @param basePath 基本路径 * @param schema schema文件的信息 * @throws Exception 异常信息 * @创建日期 2016年9月17日 */ private void sequenceTozk(String basePath, String name) throws Exception { // 读取当前节的信息 String commPath = ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SEQUENCE_COMMON.getKey() + ZookeeperPath.ZK_SEPARATOR.getKey(); String readFile = name + PROPERTIES_SUFFIX; // 读取公共节点的信息 String commSequence = this.readSequenceCfg(readFile); String sequenceZkPath = commPath + readFile; this.checkAndwriteString(basePath, sequenceZkPath, commSequence); // 集群中特有的节点的配制信息 String culsterPath = ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SEQUENCE_CLUSTER.getKey() + ZookeeperPath.ZK_SEPARATOR.getKey(); String[] clusters = ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_CLUSTER_NODES) .split(ZkParamCfg.ZK_CFG_CLUSTER_NODES_SEPARATE.getKey()); if (null != clusters) { String nodeName = null; for (String clusterName : clusters) { nodeName = name + "-" + clusterName + PROPERTIES_SUFFIX; // 读取当前集群中特有的节点的信息 String clusterSequence = this.readSequenceCfg(nodeName); // 如果配制了特定节点的信息,则将往上入zk中 if (null != clusterSequence) { String seqclusterZkPath = culsterPath + nodeName; this.checkAndwriteString(basePath, seqclusterZkPath, clusterSequence); } } } } /** * 读取 sequence配制文件的信息 * 方法描述 * @param name 名称信息 * @return * @创建日期 2016年9月18日 */ private String readSequenceCfg(String name) { String path = ZookeeperPath.ZK_LOCAL_CFG_PATH.getKey() + name; // 加载数据 InputStream input = SequenceTozkLoader.class.getResourceAsStream(path); if (null != input) { StringBuilder mapFileStr = new StringBuilder(); byte[] buffers = new byte[256]; try { int readIndex = -1; while ((readIndex = input.read(buffers)) != -1) { mapFileStr.append(new String(buffers, 0, readIndex)); } } catch (IOException e) { e.printStackTrace(); LOGGER.error("SequenceTozkLoader readMapFile IOException", e); } finally { IOUtils.close(input); } return mapFileStr.toString(); } return null; } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/xmltozk/listen/ServerxmlTozkLoader.java ================================================ package io.mycat.config.loader.zkprocess.xmltozk.listen; import java.io.IOException; import java.io.InputStream; import java.util.List; import org.apache.curator.framework.CuratorFramework; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.fastjson.util.IOUtils; import io.mycat.config.loader.console.ZookeeperPath; import io.mycat.config.loader.zkprocess.comm.NotiflyService; import io.mycat.config.loader.zkprocess.comm.ZkConfig; import io.mycat.config.loader.zkprocess.comm.ZkParamCfg; import io.mycat.config.loader.zkprocess.comm.ZookeeperProcessListen; import io.mycat.config.loader.zkprocess.entity.Server; import io.mycat.config.loader.zkprocess.entity.server.System; import io.mycat.config.loader.zkprocess.entity.server.user.User; import io.mycat.config.loader.zkprocess.parse.ParseJsonServiceInf; import io.mycat.config.loader.zkprocess.parse.ParseXmlServiceInf; import io.mycat.config.loader.zkprocess.parse.XmlProcessBase; import io.mycat.config.loader.zkprocess.parse.entryparse.server.json.SystemJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.server.json.UserJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.server.xml.ServerParseXmlImpl; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkMultLoader; /** * 进行从server.xml加载到zk中加载 * 源文件名：SchemasLoader.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class ServerxmlTozkLoader extends ZkMultLoader implements NotiflyService { /** * 日志 * @字段说明 LOGGER */ private static final Logger LOGGER = LoggerFactory.getLogger(ServerxmlTozkLoader.class); /** * 当前文件中的zkpath信息 * @字段说明 currZkPath */ private final String currZkPath; /** * server文件的路径信息 * @字段说明 SCHEMA_PATH */ private static final String SERVER_PATH = ZookeeperPath.ZK_LOCAL_CFG_PATH.getKey() + "server.xml"; /** * index_to_charset文件的路径信息 * @字段说明 SCHEMA_PATH */ private static final String INDEX_TOCHARSET_PATH = "index_to_charset.properties"; /** * server的xml的转换信息 * @字段说明 parseServerXMl */ private ParseXmlServiceInf parseServerXMl; /** * system信息 * @字段说明 parseJsonSchema */ private ParseJsonServiceInf parseJsonSystem = new SystemJsonParse(); /** * system信息 * @字段说明 parseJsonSchema */ private ParseJsonServiceInf> parseJsonUser = new UserJsonParse(); public ServerxmlTozkLoader(ZookeeperProcessListen zookeeperListen, CuratorFramework curator, XmlProcessBase xmlParseBase) { this.setCurator(curator); // 获得当前集群的名称 String schemaPath = zookeeperListen.getBasePath(); schemaPath = schemaPath + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SERVER.getKey(); currZkPath = schemaPath; // 将当前自己注册为事件接收对象 zookeeperListen.addListen(schemaPath, this); // 生成xml与类的转换信息 parseServerXMl = new ServerParseXmlImpl(xmlParseBase); } @Override public boolean notiflyProcess() throws Exception { // 1,读取本地的xml文件 Server server = this.parseServerXMl.parseXmlToBean(SERVER_PATH); LOGGER.info("ServerxmlTozkLoader notiflyProcessxml to zk server Object :" + server); // 将实体信息写入至zk中 this.xmlTozkServerJson(currZkPath, server); // 2,读取集群中的节点信息 this.writeClusterNode(currZkPath); // 读取properties String charSetValue = readProperties(INDEX_TOCHARSET_PATH); // 将文件上传 this.checkAndwriteString(currZkPath, INDEX_TOCHARSET_PATH, charSetValue); LOGGER.info("ServerxmlTozkLoader notiflyProcess xml to zk is success"); return true; } /** * 写入集群节点的信息 * 方法描述 * @throws Exception * @创建日期 2016年9月17日 */ private void writeClusterNode(String basePath) throws Exception { // 1，读取集群节点信息 String[] zkNodes = ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_CLUSTER_NODES) .split(ZkParamCfg.ZK_CFG_CLUSTER_NODES_SEPARATE.getKey()); if (null != zkNodes && zkNodes.length > 0) { for (String node : zkNodes) { String nodePath = ZookeeperPath.ZK_LOCAL_CFG_PATH.getKey() + "server-" + node + ".xml"; // 将当前的xml文件写入到zk中 Server serverNode = this.parseServerXMl.parseXmlToBean(nodePath); LOGGER.info("ServerxmlTozkLoader writeClusterNode to zk server Object :" + serverNode); // 如果当前不存在此配制文件则不写入 if (null != serverNode) { // 以集群的节点的名称写入 this.xmlTozkClusterNodeJson(basePath, node, serverNode); LOGGER.info("ServerxmlTozkLoader writeClusterNode xml to zk is success"); } } } } /** * 将xml文件的信息写入到zk中 * 方法描述 * @param basePath 基本路径 * @param schema schema文件的信息 * @throws Exception 异常信息 * @创建日期 2016年9月17日 */ private void xmlTozkServerJson(String basePath, Server server) throws Exception { // 设置默认的节点信息 String defaultSystem = ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SERVER_DEFAULT.getKey(); String defaultSystemValue = this.parseJsonSystem.parseBeanToJson(server.getSystem()); this.checkAndwriteString(basePath, defaultSystem, defaultSystemValue); // 设置用户信息 String userStr = ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SERVER_USER.getKey(); String userValueStr = this.parseJsonUser.parseBeanToJson(server.getUser()); this.checkAndwriteString(basePath, userStr, userValueStr); } /** * 将xml文件的信息写入到zk中 * 方法描述 * @param basePath 基本路径 * @param schema schema文件的信息 * @throws Exception 异常信息 * @创建日期 2016年9月17日 */ private void xmlTozkClusterNodeJson(String basePath, String node, Server server) throws Exception { // 设置集群中的节点信息 basePath = basePath + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SERVER_CLUSTER.getKey(); String clusterSystemValue = this.parseJsonSystem.parseBeanToJson(server.getSystem()); this.checkAndwriteString(basePath, node, clusterSystemValue); } /** * 读取 properties配制文件的信息 * 方法描述 * @param name 名称信息 * @return * @创建日期 2016年9月18日 */ private String readProperties(String name) { String path = ZookeeperPath.ZK_LOCAL_CFG_PATH.getKey() + name; // 加载数据 InputStream input = SequenceTozkLoader.class.getResourceAsStream(path); if (null != input) { StringBuilder mapFileStr = new StringBuilder(); byte[] buffers = new byte[256]; try { int readIndex = -1; while ((readIndex = input.read(buffers)) != -1) { mapFileStr.append(new String(buffers, 0, readIndex)); } } catch (IOException e) { e.printStackTrace(); LOGGER.error("SequenceTozkLoader readMapFile IOException", e); } finally { IOUtils.close(input); } return mapFileStr.toString(); } return null; } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/zktoxml/ZktoXmlMain.java ================================================ package io.mycat.config.loader.zkprocess.zktoxml; import java.util.Set; import io.mycat.config.loader.zkprocess.zktoxml.command.CommandPathListener; import io.mycat.config.loader.zkprocess.zktoxml.listen.*; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.NodeCache; import org.apache.curator.framework.recipes.cache.NodeCacheListener; import org.apache.curator.framework.recipes.cache.PathChildrenCache; import org.apache.curator.utils.ZKPaths; import org.apache.zookeeper.CreateMode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.config.loader.console.ZookeeperPath; import io.mycat.config.loader.zkprocess.comm.ZkConfig; import io.mycat.config.loader.zkprocess.comm.ZkParamCfg; import io.mycat.config.loader.zkprocess.comm.ZookeeperProcessListen; import io.mycat.config.loader.zkprocess.console.ZkNofiflyCfg; import io.mycat.config.loader.zkprocess.parse.XmlProcessBase; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkMultLoader; import io.mycat.manager.handler.ZKHandler; import io.mycat.migrate.MigrateTaskWatch; import io.mycat.util.ZKUtils; /** * 将xk的信息转换为xml文件的操作 * 源文件名：ZktoxmlMain.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月20日 * 修改作者：liujun * 修改日期：2016年9月20日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class ZktoXmlMain { /** * 日志 * @字段说明 LOGGER */ private static final Logger LOGGER = LoggerFactory.getLogger(ZkMultLoader.class); /** * 加载zk监听服务 */ public static final ZookeeperProcessListen ZKLISTENER = new ZookeeperProcessListen(); public static void main(String[] args) throws Exception { loadZktoFile(); System.out.println(Long.MAX_VALUE); } /** * 将zk数据放到到本地 * 方法描述 * @throws Exception * @创建日期 2016年9月21日 */ public static void loadZktoFile() throws Exception { // 得到集群名称 String custerName = ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_CLUSTERID); // 得到基本路径 String basePath = ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_BASE.getKey(); basePath = basePath + ZookeeperPath.ZK_SEPARATOR.getKey() + custerName; ZKLISTENER.setBasePath(basePath); // 获得zk的连接信息 CuratorFramework zkConn = buildConnection(ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_URL)); // 获得公共的xml转换器对象 XmlProcessBase xmlProcess = new XmlProcessBase(); // 加载以接收者 new SchemaszkToxmlLoader(ZKLISTENER, zkConn, xmlProcess); // server加载 new ServerzkToxmlLoader(ZKLISTENER, zkConn, xmlProcess); // rule文件加载 // new RuleszkToxmlLoader(zkListen, zkConn, xmlProcess); ZKUtils.addChildPathCache(ZKUtils.getZKBasePath() + "rules", new RuleFunctionCacheListener()); // 将序列配制信息加载 new SequenceTopropertiesLoader(ZKLISTENER, zkConn, xmlProcess); // 进行ehcache转换 new EcacheszkToxmlLoader(ZKLISTENER, zkConn, xmlProcess); // 将bindata目录的数据进行转换到本地文件 ZKUtils.addChildPathCache(ZKUtils.getZKBasePath() + "bindata", new BinDataPathChildrenCacheListener()); // ruledata ZKUtils.addChildPathCache(ZKUtils.getZKBasePath() + "ruledata", new RuleDataPathChildrenCacheListener()); // 初始化xml转换操作 xmlProcess.initJaxbClass(); // 通知所有人 ZKLISTENER.notifly(ZkNofiflyCfg.ZK_NOTIFLY_LOAD_ALL.getKey()); // 加载watch loadZkWatch(ZKLISTENER.getWatchPath(), zkConn, ZKLISTENER); // 创建临时节点 createTempNode(ZKUtils.getZKBasePath() + "line", ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_MYID), zkConn, ZkConfig.getInstance().getValue(ZkParamCfg.MYCAT_SERVER_TYPE)); // 接收zk发送过来的命令 runCommandWatch(zkConn, ZKUtils.getZKBasePath() + ZKHandler.ZK_NODE_PATH); MigrateTaskWatch.start(); } private static void loadZkWatch(Set setPaths, final CuratorFramework zkConn, final ZookeeperProcessListen zkListen) throws Exception { if (null != setPaths && !setPaths.isEmpty()) { for (String path : setPaths) { // 进行本地节点的监控操作 NodeCache node = runWatch(zkConn, path, zkListen); node.start(); LOGGER.info("ZktoxmlMain loadZkWatch path:" + path + " regist success"); } } } /** * 进行命令的监听操作 * 方法描述 * @param zkConn zk的连接信息 * @param path 路径信息 * @param ZKLISTENER 监控路径信息 * @throws Exception * @创建日期 2016年9月20日 */ @SuppressWarnings("resource") private static void runCommandWatch(final CuratorFramework zkConn, final String path) throws Exception { PathChildrenCache children = new PathChildrenCache(zkConn, path, true); CommandPathListener commandListener = new CommandPathListener(); // 移除原来的监听再进行添加 children.getListenable().addListener(commandListener); children.start(); } /** * 创建临时节点测试 * 方法描述 * @param parent * @param node * @param zkConn * @throws Exception * @创建日期 2016年9月20日 */ private static void createTempNode(String parent, String node, final CuratorFramework zkConn, String type) throws Exception { String path = ZKPaths.makePath(parent, node); zkConn.create().withMode(CreateMode.EPHEMERAL).inBackground().forPath(path, type.getBytes("UTF-8")); } /** * 进行zk的watch操作 * 方法描述 * @param zkConn zk的连接信息 * @param path 路径信息 * @param zkListen 监控路径信息 * @throws Exception * @创建日期 2016年9月20日 */ private static NodeCache runWatch(final CuratorFramework zkConn, final String path, final ZookeeperProcessListen zkListen) throws Exception { final NodeCache cache = new NodeCache(zkConn, path); NodeCacheListener listen = new NodeCacheListener() { @Override public void nodeChanged() throws Exception { LOGGER.info("ZktoxmlMain runWatch process path event start "); LOGGER.info("NodeCache changed, path is: " + cache.getCurrentData().getPath()); String notPath = cache.getCurrentData().getPath(); // 进行通知更新 zkListen.notifly(notPath); LOGGER.info("ZktoxmlMain runWatch process path event over"); } }; // 添加监听 cache.getListenable().addListener(listen); return cache; } private static CuratorFramework buildConnection(String url) { return ZKUtils.getConnection(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/zktoxml/command/CommandPathListener.java ================================================ package io.mycat.config.loader.zkprocess.zktoxml.command; import java.util.concurrent.Callable; import java.util.concurrent.locks.ReentrantLock; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import io.mycat.MycatServer; import io.mycat.config.loader.zkprocess.comm.ZkConfig; import io.mycat.config.loader.zkprocess.comm.ZkParamCfg; import io.mycat.config.loader.zkprocess.console.ZkNofiflyCfg; import io.mycat.config.loader.zkprocess.zktoxml.ZktoXmlMain; import io.mycat.manager.handler.ZKHandler; import io.mycat.manager.response.ReloadConfig; import io.mycat.net.NIOProcessor; import io.mycat.util.ZKUtils; /** * zk命令监听器 * @author kk * @date 2017年1月18日 * @version 0.0.1 */ public class CommandPathListener implements PathChildrenCacheListener { private static final Logger LOGGER = LoggerFactory.getLogger(CommandPathListener.class); @Override public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception { switch (event.getType()) { case CHILD_ADDED: // 在发生节点添加的时候，则执行接收命令并执行 // 1,首先检查 String path = event.getData().getPath(); String basePath = ZKUtils.getZKBasePath() + ZKHandler.ZK_NODE_PATH + "/"; // 检查节点与当前的节点是否一致 String node = path.substring(basePath.length()); if (node.equals(ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_MYID))) { // 检查命令内容是否为 if (ZKHandler.RELOAD_FROM_ZK.equals(new String(client.getData().forPath(path)))) { // 从服务器上下载最新的配制文件信息 ZktoXmlMain.ZKLISTENER.notifly(ZkNofiflyCfg.ZK_NOTIFLY_LOAD_ALL.getKey()); // 重新加载配制信息 reload(path); // 完成之后，删除命令信息， 以供下次读取 client.delete().forPath(event.getData().getPath()); LOGGER.info("CommandPathListener path:" + path + " reload success"); } } break; case CHILD_UPDATED: break; case CHILD_REMOVED: break; default: break; } } public void reload(final String path) { // reload @@config_all 校验前一次的事务完成情况 if (!NIOProcessor.backends_old.isEmpty()) { return; } final ReentrantLock lock = MycatServer.getInstance().getConfig().getLock(); lock.lock(); try { ListenableFuture listenableFuture = MycatServer.getInstance().getListeningExecutorService() .submit(new Callable() { @Override public Boolean call() throws Exception { return ReloadConfig.reload_all(); } }); Futures.addCallback(listenableFuture, new FutureCallback() { @Override public void onSuccess(Boolean result) { LOGGER.info("CommandPathListener path:" + path + " reload success"); } @Override public void onFailure(Throwable t) { LOGGER.error("CommandPathListener path:" + path + " reload error", t); } }, MycatServer.getInstance().getListeningExecutorService()); } finally { lock.unlock(); } } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/zktoxml/listen/BinDataPathChildrenCacheListener.java ================================================ package io.mycat.config.loader.zkprocess.zktoxml.listen; import java.io.File; import java.io.IOException; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import com.google.common.io.Files; import io.mycat.MycatServer; import io.mycat.config.model.SystemConfig; /** * Created by magicdoom on 2016/10/27. */ public class BinDataPathChildrenCacheListener implements PathChildrenCacheListener { @Override public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception { ChildData data = event.getData(); switch (event.getType()) { case CHILD_ADDED: add(data.getPath().substring(data.getPath().lastIndexOf("/")+1),event.getData().getData()) ; break; case CHILD_REMOVED: delete(data.getPath().substring(data.getPath().lastIndexOf("/")+1),event.getData().getData()); ; break; case CHILD_UPDATED: add(data.getPath().substring(data.getPath().lastIndexOf("/")+1),event.getData().getData()) ; break; default: break; } } private void add(String name,byte[] data) throws IOException { File file = new File( SystemConfig.getHomePath() + File.separator + "conf" , name); Files.write(data,file); //try to reload dnindex if("dnindex.properties".equals(name)) { MycatServer.getInstance().reloadDnIndex(); } } private void delete(String name,byte[] data) throws IOException { File file = new File( SystemConfig.getHomePath() + File.separator + "conf" , name); if(file.exists()) file.delete(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/zktoxml/listen/EcacheszkToxmlLoader.java ================================================ package io.mycat.config.loader.zkprocess.zktoxml.listen; import static com.google.common.base.Preconditions.checkNotNull; import java.io.File; import java.io.IOException; import org.apache.curator.framework.CuratorFramework; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.io.Files; import io.mycat.config.loader.console.ZookeeperPath; import io.mycat.config.loader.zkprocess.comm.NotiflyService; import io.mycat.config.loader.zkprocess.comm.ZookeeperProcessListen; import io.mycat.config.loader.zkprocess.entity.cache.Ehcache; import io.mycat.config.loader.zkprocess.parse.ParseJsonServiceInf; import io.mycat.config.loader.zkprocess.parse.ParseXmlServiceInf; import io.mycat.config.loader.zkprocess.parse.XmlProcessBase; import io.mycat.config.loader.zkprocess.parse.entryparse.cache.json.EhcacheJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.cache.xml.EhcacheParseXmlImpl; import io.mycat.config.loader.zkprocess.zookeeper.DataInf; import io.mycat.config.loader.zkprocess.zookeeper.DiretoryInf; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkDataImpl; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkDirectoryImpl; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkMultLoader; /** * 进行从ecache.xml加载到zk中加载 * 源文件名：SchemasLoader.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class EcacheszkToxmlLoader extends ZkMultLoader implements NotiflyService { /** * 日志 * @字段说明 LOGGER */ private static final Logger LOGGER = LoggerFactory.getLogger(EcacheszkToxmlLoader.class); /** * 当前文件中的zkpath信息 * @字段说明 currZkPath */ private final String currZkPath; /** * 缓存文件名称 * @字段说明 CACHESERVER_NAME */ private static final String CACHESERVER_NAME = "cacheservice.properties"; /** * 缓存的xml文件配制信息 * @字段说明 EHCACHE_NAME */ private static final String EHCACHE_NAME = "ehcache.xml"; /** * ehcache的xml的转换信息 * @字段说明 parseEhcacheXMl */ private final ParseXmlServiceInf parseEcacheXMl; /** * 表的路由信息 * @字段说明 parseJsonService */ private ParseJsonServiceInf parseJsonEhcacheService = new EhcacheJsonParse(); /** * 监控类信息 * @字段说明 zookeeperListen */ private ZookeeperProcessListen zookeeperListen; public EcacheszkToxmlLoader(ZookeeperProcessListen zookeeperListen, CuratorFramework curator, XmlProcessBase xmlParseBase) { this.setCurator(curator); this.zookeeperListen = zookeeperListen; // 获得当前集群的名称 String schemaPath = zookeeperListen.getBasePath(); schemaPath = schemaPath + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_CACHE.getKey(); currZkPath = schemaPath; // 将当前自己注册为事件接收对象 this.zookeeperListen.addListen(schemaPath, this); // 生成xml与类的转换信息 parseEcacheXMl = new EhcacheParseXmlImpl(xmlParseBase); } @Override public boolean notiflyProcess() throws Exception { // 通过组合模式进行zk目录树的加载 DiretoryInf RulesDirectory = new ZkDirectoryImpl(currZkPath, null); // 进行递归的数据获取 this.getTreeDirectory(currZkPath, ZookeeperPath.FLOW_ZK_PATH_CACHE.getKey(), RulesDirectory); // 从当前的下一级开始进行遍历,获得到 ZkDirectoryImpl zkDirectory = (ZkDirectoryImpl) RulesDirectory.getSubordinateInfo().get(0); // 进行写入操作 zktoEhcacheWrite(zkDirectory); LOGGER.info("EcacheszkToxmlLoader notiflyProcess zk ehcache write success "); return true; } /** * 将zk上面的信息转换为javabean对象 * 方法描述 * @param zkDirectory * @return * @创建日期 2016年9月17日 */ private void zktoEhcacheWrite(ZkDirectoryImpl zkDirectory) { // 得到schema对象的目录信息 DataInf ehcacheZkDirectory = this.getZkData(zkDirectory, EHCACHE_NAME); Ehcache ehcache = parseJsonEhcacheService.parseJsonToBean(ehcacheZkDirectory.getDataValue()); String outputPath = EcacheszkToxmlLoader.class.getClassLoader() .getResource(ZookeeperPath.ZK_LOCAL_WRITE_PATH.getKey()).getPath(); outputPath = new File(outputPath).getPath() + File.separator; outputPath += EHCACHE_NAME; parseEcacheXMl.parseToXmlWrite(ehcache, outputPath, null); // 设置zk监控的路径信息 String watchPath = zkDirectory.getName(); watchPath = watchPath + ZookeeperPath.ZK_SEPARATOR.getKey() + EHCACHE_NAME; this.zookeeperListen.watchPath(currZkPath, watchPath); // 写入cacheservice.properties的信息 DataInf cacheserZkDirectory = this.getZkData(zkDirectory, CACHESERVER_NAME); if (null != cacheserZkDirectory) { ZkDataImpl cacheData = (ZkDataImpl) cacheserZkDirectory; // 写入文件cacheservice.properties this.writeCacheservice(cacheData.getName(), cacheData.getValue()); String watchServerPath = zkDirectory.getName(); watchServerPath = watchPath + ZookeeperPath.ZK_SEPARATOR.getKey() + CACHESERVER_NAME; this.zookeeperListen.watchPath(currZkPath, watchServerPath); } } /** * 读取 mapFile文件的信息 * 方法描述 * @param name 名称信息 * @return * @创建日期 2016年9月18日 */ private void writeCacheservice(String name, String value) { // 加载数据 String path = RuleszkToxmlLoader.class.getClassLoader().getResource(ZookeeperPath.ZK_LOCAL_WRITE_PATH.getKey()) .getPath(); checkNotNull(path, "write ecache file curr Path :" + path + " is null! must is not null"); path = new File(path).getPath() + File.separator; path += name; // 进行数据写入 try { Files.write(value.getBytes(), new File(path)); } catch (IOException e1) { e1.printStackTrace(); } } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/zktoxml/listen/RuleDataPathChildrenCacheListener.java ================================================ package io.mycat.config.loader.zkprocess.zktoxml.listen; import io.mycat.MycatServer; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.config.model.TableConfig; import io.mycat.config.model.rule.RuleConfig; import io.mycat.route.function.AbstractPartitionAlgorithm; import io.mycat.route.function.ReloadFunction; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import java.util.Map; /** * Created by magicdoom on 2016/10/27. */ public class RuleDataPathChildrenCacheListener implements PathChildrenCacheListener { @Override public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception { ChildData data = event.getData(); switch (event.getType()) { case CHILD_ADDED: add(data.getPath().substring(data.getPath().lastIndexOf("/") + 1), event.getData().getData()); break; case CHILD_REMOVED: delete(data.getPath().substring(data.getPath().lastIndexOf("/") + 1), event.getData().getData()); ; break; case CHILD_UPDATED: add(data.getPath().substring(data.getPath().lastIndexOf("/") + 1), event.getData().getData()); break; default: break; } } private void reloadRuleData(String name) { String tableName = name.substring(name.lastIndexOf("_") + 1, name.indexOf(".")); String ruleName = name.substring(0, name.indexOf(".")); Map schemaConfigMap = MycatServer.getInstance().getConfig().getSchemas(); for (SchemaConfig schemaConfig : schemaConfigMap.values()) { TableConfig tableConfig = schemaConfig.getTables().get(tableName.toUpperCase()); if (tableConfig == null) continue; RuleConfig rule = tableConfig.getRule(); AbstractPartitionAlgorithm function = rule.getRuleAlgorithm(); if (function instanceof ReloadFunction) { ((ReloadFunction) function).reload(); } } } private void add(String name, byte[] data) throws IOException { Path ruledataPath = Paths.get(SystemConfig.getHomePath(), "conf", "ruledata"); if (!Files.exists(ruledataPath)) { Files.createDirectory(ruledataPath); } Path file = ruledataPath.resolve(name); Files.write(file, data, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); reloadRuleData(name); } private void delete(String name, byte[] data) throws IOException { File file = new File( SystemConfig.getHomePath() + File.separator + "conf" + File.separator + "ruledata", name); if (file.exists()) file.delete(); } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/zktoxml/listen/RuleFunctionCacheListener.java ================================================ package io.mycat.config.loader.zkprocess.zktoxml.listen; import com.google.common.io.Files; import io.mycat.MycatServer; import io.mycat.config.loader.console.ZookeeperPath; import io.mycat.config.loader.zkprocess.console.ParseParamEnum; import io.mycat.config.loader.zkprocess.entity.Property; import io.mycat.config.loader.zkprocess.entity.Rules; import io.mycat.config.loader.zkprocess.entity.rule.function.Function; import io.mycat.config.loader.zkprocess.entity.rule.tablerule.TableRule; import io.mycat.config.loader.zkprocess.parse.ParseJsonServiceInf; import io.mycat.config.loader.zkprocess.parse.ParseXmlServiceInf; import io.mycat.config.loader.zkprocess.parse.XmlProcessBase; import io.mycat.config.loader.zkprocess.parse.entryparse.rule.json.FunctionJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.rule.json.TableRuleJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.rule.xml.RuleParseXmlImpl; import io.mycat.config.loader.zkprocess.zookeeper.DataInf; import io.mycat.config.loader.zkprocess.zookeeper.DiretoryInf; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkDataImpl; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.config.model.TableConfig; import io.mycat.config.model.rule.RuleConfig; import io.mycat.manager.response.ReloadConfig; import io.mycat.route.function.AbstractPartitionAlgorithm; import io.mycat.route.function.ReloadFunction; import io.mycat.util.ZKUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.xml.bind.JAXBException; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; import static com.google.common.base.Preconditions.checkNotNull; /** * Created by magicdoom on 2016/10/27. */ public class RuleFunctionCacheListener implements PathChildrenCacheListener { private static final Logger LOGGER = LoggerFactory.getLogger(RuleFunctionCacheListener.class); @Override public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception { ChildData data = event.getData(); switch (event.getType()) { case CHILD_ADDED: addOrUpdate(); break; case CHILD_UPDATED: addOrUpdate(); break; default: break; } } public RuleFunctionCacheListener() { XmlProcessBase xmlProcessBase = new XmlProcessBase(); parseRulesXMl = new RuleParseXmlImpl(xmlProcessBase) ; try { xmlProcessBase.initJaxbClass(); } catch (JAXBException e) { LOGGER.error("error",e); } } private void addOrUpdate() { Rules Rules = null; try { Rules = this.zktoRulesBean(); } catch (Exception e) { LOGGER.error("error",e); } LOGGER.info("RuleszkToxmlLoader notiflyProcess zk to object zk Rules Object :" + Rules); // 将mapfile信息写入到文件 中 writeMapFileAddFunction(Rules.getFunction()); LOGGER.info("RuleszkToxmlLoader notiflyProcess write mapFile is success "); // 数配制信息写入文件 String path = RuleszkToxmlLoader.class.getClassLoader().getResource(ZookeeperPath.ZK_LOCAL_WRITE_PATH.getKey()) .getPath(); path = new File(path).getPath() + File.separator; path = path + WRITEPATH; LOGGER.info("RuleszkToxmlLoader notiflyProcess zk to object writePath :" + path); this.parseRulesXMl.parseToXmlWrite(Rules, path, "rule"); LOGGER.info("RuleszkToxmlLoader notiflyProcess zk to object zk Rules write :" + path + " is success"); if (MycatServer.getInstance().getProcessors() != null) ReloadConfig.reload(); } private static final String WRITEPATH = "rule.xml"; /** * Rules的xml的转换信息 * @字段说明 parseRulesXMl */ private ParseXmlServiceInf parseRulesXMl;; /** * 表的路由信息 * @字段说明 parseJsonService */ private ParseJsonServiceInf> parseJsonTableRuleService = new TableRuleJsonParse(); /** * 表对应的字段信息 * @字段说明 parseJsonFunctionService */ private ParseJsonServiceInf> parseJsonFunctionService = new FunctionJsonParse(); private Rules zktoRulesBean() throws Exception { Rules Rules = new Rules(); // tablerule信息 String value= new String( ZKUtils.getConnection().getData().forPath(ZKUtils.getZKBasePath()+"rules/tableRule"),"UTF-8") ; DataInf RulesZkData = new ZkDataImpl("tableRule",value); List tableRuleData = parseJsonTableRuleService.parseJsonToBean(RulesZkData.getDataValue()); Rules.setTableRule(tableRuleData); // 得到function信息 String fucValue= new String( ZKUtils.getConnection().getData().forPath(ZKUtils.getZKBasePath()+"rules/function"),"UTF-8") ; DataInf functionZkData =new ZkDataImpl("function",fucValue) ; List functionList = parseJsonFunctionService.parseJsonToBean(functionZkData.getDataValue()); Rules.setFunction(functionList); return Rules; } /** * 读取序列配制文件便利店 * 方法描述 * @param functionList * @创建日期 2016年9月18日 */ private void writeMapFileAddFunction(List functionList) { List tempData = new ArrayList<>(); List writeData = new ArrayList<>(); for (Function function : functionList) { List proList = function.getProperty(); if (null != proList && !proList.isEmpty()) { // 进行数据遍历 for (Property property : proList) { // 如果为mapfile，则需要去读取数据信息，并存到json中 if (ParseParamEnum.ZK_PATH_RULE_MAPFILE_NAME.getKey().equals(property.getName())) { tempData.add(property); } } // 通过mapfile的名称，找到对应的数据信息 if (!tempData.isEmpty()) { for (Property property : tempData) { for (Property prozkdownload : proList) { // 根据mapfile的文件名去提取数据 if (property.getValue().equals(prozkdownload.getName())) { writeData.add(prozkdownload); } } } } // 将对应的数据信息写入到磁盘中 if (!writeData.isEmpty()) { for (Property writeMsg : writeData) { this.writeMapFile(writeMsg.getName(), writeMsg.getValue()); } } // 将数据添加的集合中 proList.removeAll(writeData); // 清空，以进行下一次的添加 tempData.clear(); writeData.clear(); } } } /** * 读取 mapFile文件的信息 * 方法描述 * @param name 名称信息 * @return * @创建日期 2016年9月18日 */ private void writeMapFile(String name, String value) { // 加载数据 String path = RuleszkToxmlLoader.class.getClassLoader().getResource(ZookeeperPath.ZK_LOCAL_WRITE_PATH.getKey()) .getPath(); checkNotNull(path, "write Map file curr Path :" + path + " is null! must is not null"); path = new File(path).getPath() + File.separator; path += name; // 进行数据写入 try { Files.write(value.getBytes(), new File(path)); } catch (IOException e1) { e1.printStackTrace(); } } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/zktoxml/listen/RuleszkToxmlLoader.java ================================================ package io.mycat.config.loader.zkprocess.zktoxml.listen; import static com.google.common.base.Preconditions.checkNotNull; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; import org.apache.curator.framework.CuratorFramework; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.io.Files; import io.mycat.MycatServer; import io.mycat.config.loader.console.ZookeeperPath; import io.mycat.config.loader.zkprocess.comm.NotiflyService; import io.mycat.config.loader.zkprocess.comm.ZookeeperProcessListen; import io.mycat.config.loader.zkprocess.console.ParseParamEnum; import io.mycat.config.loader.zkprocess.entity.Property; import io.mycat.config.loader.zkprocess.entity.Rules; import io.mycat.config.loader.zkprocess.entity.rule.function.Function; import io.mycat.config.loader.zkprocess.entity.rule.tablerule.TableRule; import io.mycat.config.loader.zkprocess.parse.ParseJsonServiceInf; import io.mycat.config.loader.zkprocess.parse.ParseXmlServiceInf; import io.mycat.config.loader.zkprocess.parse.XmlProcessBase; import io.mycat.config.loader.zkprocess.parse.entryparse.rule.json.FunctionJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.rule.json.TableRuleJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.rule.xml.RuleParseXmlImpl; import io.mycat.config.loader.zkprocess.zookeeper.DataInf; import io.mycat.config.loader.zkprocess.zookeeper.DiretoryInf; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkDirectoryImpl; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkMultLoader; import io.mycat.manager.response.ReloadConfig; /** * 进行rule的文件从zk中加载 * 源文件名：RuleszkToxmlLoader.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class RuleszkToxmlLoader extends ZkMultLoader implements NotiflyService { /** * 日志 * @字段说明 LOGGER */ private static final Logger LOGGER = LoggerFactory.getLogger(RuleszkToxmlLoader.class); /** * 当前文件中的zkpath信息 * @字段说明 currZkPath */ private final String currZkPath; /** * 写入本地的文件路径 * @字段说明 WRITEPATH */ private static final String WRITEPATH = "rule.xml"; /** * Rules的xml的转换信息 * @字段说明 parseRulesXMl */ private ParseXmlServiceInf parseRulesXMl; /** * 表的路由信息 * @字段说明 parseJsonService */ private ParseJsonServiceInf> parseJsonTableRuleService = new TableRuleJsonParse(); /** * 表对应的字段信息 * @字段说明 parseJsonFunctionService */ private ParseJsonServiceInf> parseJsonFunctionService = new FunctionJsonParse(); /** * zk的监控路径信息 * @字段说明 zookeeperListen */ private ZookeeperProcessListen zookeeperListen; public RuleszkToxmlLoader(ZookeeperProcessListen zookeeperListen, CuratorFramework curator, XmlProcessBase xmlParseBase) { this.setCurator(curator); this.zookeeperListen = zookeeperListen; // 获得当前集群的名称 String RulesPath = zookeeperListen.getBasePath(); RulesPath = RulesPath + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_RULE.getKey(); currZkPath = RulesPath; // 将当前自己注册为事件接收对象 zookeeperListen.addListen(RulesPath, this); // 生成xml与类的转换信息 parseRulesXMl = new RuleParseXmlImpl(xmlParseBase); } @Override public boolean notiflyProcess() throws Exception { // 1,将集群Rules目录下的所有集群按层次结构加载出来 // 通过组合模式进行zk目录树的加载 DiretoryInf RulesDirectory = new ZkDirectoryImpl(currZkPath, null); // 进行递归的数据获取 this.getTreeDirectory(currZkPath, ZookeeperPath.FLOW_ZK_PATH_RULE.getKey(), RulesDirectory); // 从当前的下一级开始进行遍历,获得到 ZkDirectoryImpl zkDirectory = (ZkDirectoryImpl) RulesDirectory.getSubordinateInfo().get(0); Rules Rules = this.zktoRulesBean(zkDirectory); LOGGER.info("RuleszkToxmlLoader notiflyProcess zk to object zk Rules Object :" + Rules); // 将mapfile信息写入到文件 中 writeMapFileAddFunction(Rules.getFunction()); LOGGER.info("RuleszkToxmlLoader notiflyProcess write mapFile is success "); // 数配制信息写入文件 String path = RuleszkToxmlLoader.class.getClassLoader().getResource(ZookeeperPath.ZK_LOCAL_WRITE_PATH.getKey()) .getPath(); path = new File(path).getPath() + File.separator; path = path + WRITEPATH; LOGGER.info("RuleszkToxmlLoader notiflyProcess zk to object writePath :" + path); this.parseRulesXMl.parseToXmlWrite(Rules, path, "rule"); LOGGER.info("RuleszkToxmlLoader notiflyProcess zk to object zk Rules write :" + path + " is success"); if (MycatServer.getInstance().getProcessors() != null) ReloadConfig.reload(); return true; } /** * 将zk上面的信息转换为javabean对象 * 方法描述 * @param zkDirectory * @return * @创建日期 2016年9月17日 */ private Rules zktoRulesBean(DiretoryInf zkDirectory) { Rules Rules = new Rules(); // tablerule信息 DataInf RulesZkData = this.getZkData(zkDirectory, ZookeeperPath.FLOW_ZK_PATH_RULE_TABLERULE.getKey()); List tableRuleData = parseJsonTableRuleService.parseJsonToBean(RulesZkData.getDataValue()); Rules.setTableRule(tableRuleData); // tablerule的监控路径信息 String watchPath = ZookeeperPath.FLOW_ZK_PATH_RULE.getKey(); watchPath = watchPath + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_RULE_TABLERULE.getKey(); this.zookeeperListen.watchPath(currZkPath, watchPath); // 得到function信息 DataInf functionZkData = this.getZkData(zkDirectory, ZookeeperPath.FLOW_ZK_PATH_RULE_FUNCTION.getKey()); List functionList = parseJsonFunctionService.parseJsonToBean(functionZkData.getDataValue()); Rules.setFunction(functionList); // function的监控路径信息 String functionWatchPath = ZookeeperPath.FLOW_ZK_PATH_RULE.getKey(); functionWatchPath = functionWatchPath + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_RULE_FUNCTION.getKey(); this.zookeeperListen.watchPath(currZkPath, functionWatchPath); return Rules; } /** * 读取序列配制文件便利店 * 方法描述 * @param functionList * @创建日期 2016年9月18日 */ private void writeMapFileAddFunction(List functionList) { List tempData = new ArrayList<>(); List writeData = new ArrayList<>(); for (Function function : functionList) { List proList = function.getProperty(); if (null != proList && !proList.isEmpty()) { // 进行数据遍历 for (Property property : proList) { // 如果为mapfile，则需要去读取数据信息，并存到json中 if (ParseParamEnum.ZK_PATH_RULE_MAPFILE_NAME.getKey().equals(property.getName())) { tempData.add(property); } } // 通过mapfile的名称，找到对应的数据信息 if (!tempData.isEmpty()) { for (Property property : tempData) { for (Property prozkdownload : proList) { // 根据mapfile的文件名去提取数据 if (property.getValue().equals(prozkdownload.getName())) { writeData.add(prozkdownload); } } } } // 将对应的数据信息写入到磁盘中 if (!writeData.isEmpty()) { for (Property writeMsg : writeData) { this.writeMapFile(writeMsg.getName(), writeMsg.getValue()); } } // 将数据添加的集合中 proList.removeAll(writeData); // 清空，以进行下一次的添加 tempData.clear(); writeData.clear(); } } } /** * 读取 mapFile文件的信息 * 方法描述 * @param name 名称信息 * @return * @创建日期 2016年9月18日 */ private void writeMapFile(String name, String value) { // 加载数据 String path = RuleszkToxmlLoader.class.getClassLoader().getResource(ZookeeperPath.ZK_LOCAL_WRITE_PATH.getKey()) .getPath(); checkNotNull(path, "write Map file curr Path :" + path + " is null! must is not null"); path = new File(path).getPath() + File.separator; path += name; // 进行数据写入 try { Files.write(value.getBytes(), new File(path)); } catch (IOException e1) { e1.printStackTrace(); } } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/zktoxml/listen/SchemaszkToxmlLoader.java ================================================ package io.mycat.config.loader.zkprocess.zktoxml.listen; import java.io.File; import java.util.List; import io.mycat.MycatServer; import io.mycat.manager.response.ReloadConfig; import org.apache.curator.framework.CuratorFramework; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.config.loader.console.ZookeeperPath; import io.mycat.config.loader.zkprocess.comm.NotiflyService; import io.mycat.config.loader.zkprocess.comm.ZookeeperProcessListen; import io.mycat.config.loader.zkprocess.entity.Schemas; import io.mycat.config.loader.zkprocess.entity.schema.datahost.DataHost; import io.mycat.config.loader.zkprocess.entity.schema.datanode.DataNode; import io.mycat.config.loader.zkprocess.entity.schema.schema.Schema; import io.mycat.config.loader.zkprocess.parse.ParseJsonServiceInf; import io.mycat.config.loader.zkprocess.parse.ParseXmlServiceInf; import io.mycat.config.loader.zkprocess.parse.XmlProcessBase; import io.mycat.config.loader.zkprocess.parse.entryparse.schema.json.DataHostJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.schema.json.DataNodeJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.schema.json.SchemaJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.schema.xml.SchemasParseXmlImpl; import io.mycat.config.loader.zkprocess.zookeeper.DataInf; import io.mycat.config.loader.zkprocess.zookeeper.DiretoryInf; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkDirectoryImpl; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkMultLoader; /** * 进行schema的文件从zk中加载 * 源文件名：SchemasLoader.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class SchemaszkToxmlLoader extends ZkMultLoader implements NotiflyService { /** * 日志 * @字段说明 LOGGER */ private static final Logger LOGGER = LoggerFactory.getLogger(SchemaszkToxmlLoader.class); /** * 当前文件中的zkpath信息 * @字段说明 currZkPath */ private final String currZkPath; /** * 写入本地的文件路径 * @字段说明 WRITEPATH */ private static final String WRITEPATH = "schema.xml"; /** * schema类与xml转换服务 * @字段说明 parseSchemaService */ private ParseXmlServiceInf parseSchemaXmlService; /** * 进行将schema * @字段说明 parseJsonSchema */ private ParseJsonServiceInf> parseJsonSchema = new SchemaJsonParse(); /** * 进行将dataNode * @字段说明 parseJsonSchema */ private ParseJsonServiceInf> parseJsonDataNode = new DataNodeJsonParse(); /** * 进行将dataNode * @字段说明 parseJsonSchema */ private ParseJsonServiceInf> parseJsonDataHost = new DataHostJsonParse(); /** * zk的监控路径信息 * @字段说明 zookeeperListen */ private ZookeeperProcessListen zookeeperListen; public SchemaszkToxmlLoader(ZookeeperProcessListen zookeeperListen, CuratorFramework curator, XmlProcessBase xmlParseBase) { this.setCurator(curator); this.zookeeperListen = zookeeperListen; // 获得当前集群的名称 String schemaPath = zookeeperListen.getBasePath(); schemaPath = schemaPath + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FOW_ZK_PATH_SCHEMA.getKey(); currZkPath = schemaPath; // 将当前自己注册为事件接收对象 this.zookeeperListen.addListen(schemaPath, this); // 生成xml与类的转换信息 this.parseSchemaXmlService = new SchemasParseXmlImpl(xmlParseBase); } @Override public boolean notiflyProcess() throws Exception { // 1,将集群schema目录下的所有集群按层次结构加载出来 // 通过组合模式进行zk目录树的加载 DiretoryInf schemaDirectory = new ZkDirectoryImpl(currZkPath, null); // 进行递归的数据获取 this.getTreeDirectory(currZkPath, ZookeeperPath.FLOW_ZK_PATH_SCHEMA_SCHEMA.getKey(), schemaDirectory); // 从当前的下一级开始进行遍历,获得到 ZkDirectoryImpl zkDirectory = (ZkDirectoryImpl) schemaDirectory.getSubordinateInfo().get(0); Schemas schema = this.zktoSchemasBean(zkDirectory); LOGGER.info("SchemasLoader notiflyProcess zk to object zk schema Object :" + schema); String path = SchemaszkToxmlLoader.class.getClassLoader() .getResource(ZookeeperPath.ZK_LOCAL_WRITE_PATH.getKey()).getPath(); path=new File(path).getPath()+File.separator; path += WRITEPATH; LOGGER.info("SchemasLoader notiflyProcess zk to object writePath :" + path); this.parseSchemaXmlService.parseToXmlWrite(schema, path, "schema"); LOGGER.info("SchemasLoader notiflyProcess zk to object zk schema write :" + path + " is success"); if(MycatServer.getInstance().getStartup().get()) { ReloadConfig.reload_all(); } return true; } /** * 将zk上面的信息转换为javabean对象 * 方法描述 * @param zkDirectory * @return * @创建日期 2016年9月17日 */ private Schemas zktoSchemasBean(ZkDirectoryImpl zkDirectory) { Schemas schema = new Schemas(); // 得到schema对象的目录信息 DataInf schemaZkDirectory = this.getZkData(zkDirectory, ZookeeperPath.FLOW_ZK_PATH_SCHEMA_SCHEMA.getKey()); List schemaList = parseJsonSchema.parseJsonToBean(schemaZkDirectory.getDataValue()); schema.setSchema(schemaList); this.zookeeperListen.watchPath(currZkPath, ZookeeperPath.FLOW_ZK_PATH_SCHEMA_SCHEMA.getKey()); // 得到dataNode的信息 DataInf dataNodeZkDirectory = this.getZkData(zkDirectory, ZookeeperPath.FLOW_ZK_PATH_SCHEMA_DATANODE.getKey()); List dataNodeList = parseJsonDataNode.parseJsonToBean(dataNodeZkDirectory.getDataValue()); schema.setDataNode(dataNodeList); this.zookeeperListen.watchPath(currZkPath, ZookeeperPath.FLOW_ZK_PATH_SCHEMA_DATANODE.getKey()); // 得到dataNode的信息 DataInf dataHostZkDirectory = this.getZkData(zkDirectory, ZookeeperPath.FLOW_ZK_PATH_SCHEMA_DATAHOST.getKey()); List dataHostList = parseJsonDataHost.parseJsonToBean(dataHostZkDirectory.getDataValue()); schema.setDataHost(dataHostList); this.zookeeperListen.watchPath(currZkPath, ZookeeperPath.FLOW_ZK_PATH_SCHEMA_DATAHOST.getKey()); return schema; } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/zktoxml/listen/SequenceTopropertiesLoader.java ================================================ package io.mycat.config.loader.zkprocess.zktoxml.listen; import static com.google.common.base.Preconditions.checkNotNull; import java.io.File; import java.io.IOException; import org.apache.curator.framework.CuratorFramework; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.io.Files; import io.mycat.MycatServer; import io.mycat.config.loader.console.ZookeeperPath; import io.mycat.config.loader.zkprocess.comm.NotiflyService; import io.mycat.config.loader.zkprocess.comm.ZkConfig; import io.mycat.config.loader.zkprocess.comm.ZkParamCfg; import io.mycat.config.loader.zkprocess.comm.ZookeeperProcessListen; import io.mycat.config.loader.zkprocess.parse.XmlProcessBase; import io.mycat.config.loader.zkprocess.zookeeper.DiretoryInf; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkDataImpl; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkDirectoryImpl; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkMultLoader; import io.mycat.manager.response.ReloadConfig; /** * 进行从sequence加载到zk中加载 * 源文件名：SchemasLoader.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class SequenceTopropertiesLoader extends ZkMultLoader implements NotiflyService { /** * 日志 * @字段说明 LOGGER */ private static final Logger LOGGER = LoggerFactory.getLogger(SequenceTopropertiesLoader.class); /** * 当前文件中的zkpath信息 * @字段说明 currZkPath */ private final String currZkPath; /** * 后缀名 * @字段说明 PROPERTIES_SUFFIX */ private static final String PROPERTIES_SUFFIX = ".properties"; /** * 序列配制信息 * @字段说明 PROPERTIES_SEQUENCE_CONF */ private static final String PROPERTIES_SEQUENCE_CONF = "sequence_conf"; /** * db序列配制信息 * @字段说明 PROPERTIES_SEQUENCE_CONF */ private static final String PROPERTIES_SEQUENCE_DB_CONF = "sequence_db_conf"; /** * 分布式的序列配制 * @字段说明 PROPERTIES_SEQUENCE_CONF */ private static final String PROPERTIES_SEQUENCE_DISTRIBUTED_CONF = "sequence_distributed_conf"; /** * 时间的序列配制 * @字段说明 PROPERTIES_SEQUENCE_CONF */ private static final String PROPERTIES_SEQUENCE_TIME_CONF = "sequence_time_conf"; /** * 监控路径信息 * @字段说明 zookeeperListen */ private ZookeeperProcessListen zookeeperListen; public SequenceTopropertiesLoader(ZookeeperProcessListen zookeeperListen, CuratorFramework curator, XmlProcessBase xmlParseBase) { this.setCurator(curator); this.zookeeperListen = zookeeperListen; // 获得当前集群的名称 String schemaPath = zookeeperListen.getBasePath(); schemaPath = schemaPath + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SEQUENCE.getKey(); currZkPath = schemaPath; // 将当前自己注册为事件接收对象 this.zookeeperListen.addListen(schemaPath, this); } @Override public boolean notiflyProcess() throws Exception { // 1,将集群server目录下的所有集群按层次结构加载出来 // 通过组合模式进行zk目录树的加载 DiretoryInf sequenceDirectory = new ZkDirectoryImpl(currZkPath, null); // 进行递归的数据获取 this.getTreeDirectory(currZkPath, ZookeeperPath.FLOW_ZK_PATH_SEQUENCE.getKey(), sequenceDirectory); // 取到当前根目录 信息 sequenceDirectory = (DiretoryInf) sequenceDirectory.getSubordinateInfo().get(0); // 将zk序列配配制信息入本地文件 this.sequenceZkToProperties(currZkPath, PROPERTIES_SEQUENCE_CONF, sequenceDirectory); LOGGER.info("SequenceTozkLoader notiflyProcess sequence_conf to local properties success"); // 将zk的db方式信息入本地文件 this.sequenceZkToProperties(currZkPath, PROPERTIES_SEQUENCE_DB_CONF, sequenceDirectory); LOGGER.info("SequenceTozkLoader notiflyProcess sequence_db_conf to local properties success"); // 将zk的分布式信息入本地文件 this.seqWriteOneZkToProperties(currZkPath, PROPERTIES_SEQUENCE_DISTRIBUTED_CONF, sequenceDirectory); LOGGER.info("SequenceTozkLoader notiflyProcess sequence_distributed_conf to local properties success"); // 将zk时间序列入本地文件 this.seqWriteOneZkToProperties(currZkPath, PROPERTIES_SEQUENCE_TIME_CONF, sequenceDirectory); LOGGER.info("SequenceTozkLoader notiflyProcess sequence_time_conf to local properties success"); LOGGER.info("SequenceTozkLoader notiflyProcess xml to local properties is success"); if (MycatServer.getInstance().getProcessors() != null) ReloadConfig.reload(); return true; } /** * 将xml文件的信息写入到zk中 * 方法描述 * @param basePath 基本路径 * @param schema schema文件的信息 * @throws Exception 异常信息 * @创建日期 2016年9月17日 */ private void sequenceZkToProperties(String basePath, String name, DiretoryInf seqDirectory) throws Exception { // 读取当前节的信息 ZkDirectoryImpl zkDirectory = (ZkDirectoryImpl) this.getZkDirectory(seqDirectory, ZookeeperPath.FLOW_ZK_PATH_SEQUENCE_COMMON.getKey()); if (null != zkDirectory) { String writeFile = name + PROPERTIES_SUFFIX; // 读取common目录下的数据 ZkDataImpl commData = (ZkDataImpl) this.getZkData(zkDirectory, writeFile); // 读取公共节点的信息 this.writeMapFile(commData.getName(), commData.getValue()); String seqComm = ZookeeperPath.FLOW_ZK_PATH_SEQUENCE_COMMON.getKey(); seqComm = seqComm + ZookeeperPath.ZK_SEPARATOR.getKey() + commData.getName(); this.zookeeperListen.watchPath(currZkPath, seqComm); } // 集群中特有的节点的配制信息 ZkDirectoryImpl zkClusterDir = (ZkDirectoryImpl) this.getZkDirectory(seqDirectory, ZookeeperPath.FLOW_ZK_PATH_SEQUENCE_CLUSTER.getKey()); if (null != zkClusterDir) { String clusterName = ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_MYID); String nodeName = name + "-" + clusterName + PROPERTIES_SUFFIX; // 读取cluster目录下的数据 ZkDataImpl clusterData = (ZkDataImpl) this.getZkData(zkClusterDir, nodeName); if (null != clusterData) { // 读取当前集群中特有的节点的信息 this.writeMapFile(clusterData.getName(), clusterData.getValue()); String seqCluster = ZookeeperPath.FLOW_ZK_PATH_SEQUENCE_COMMON.getKey(); seqCluster = seqCluster + ZookeeperPath.ZK_SEPARATOR.getKey() + clusterData.getName(); this.zookeeperListen.watchPath(currZkPath, seqCluster); } } } /** * 将xml文件的信息写入到zk中 * 方法描述 * @param basePath 基本路径 * @param schema schema文件的信息 * @throws Exception 异常信息 * @创建日期 2016年9月17日 */ private void seqWriteOneZkToProperties(String basePath, String name, DiretoryInf seqDirectory) throws Exception { // 读取当前节的信息 ZkDirectoryImpl zkDirectory = (ZkDirectoryImpl) this.getZkDirectory(seqDirectory, ZookeeperPath.FLOW_ZK_PATH_SEQUENCE_COMMON.getKey()); ZkDataImpl commData = null; if (null != zkDirectory) { String writeFile = name + PROPERTIES_SUFFIX; // 读取common目录下的数据 commData = (ZkDataImpl) this.getZkData(zkDirectory, writeFile); // comm路径的监控路径 String seqComm = ZookeeperPath.FLOW_ZK_PATH_SEQUENCE_COMMON.getKey(); seqComm = seqComm + ZookeeperPath.ZK_SEPARATOR.getKey() + commData.getName(); this.zookeeperListen.watchPath(currZkPath, seqComm); } // 集群中特有的节点的配制信息 ZkDirectoryImpl zkClusterDir = (ZkDirectoryImpl) this.getZkDirectory(seqDirectory, ZookeeperPath.FLOW_ZK_PATH_SEQUENCE_CLUSTER.getKey()); ZkDataImpl clusterData = null; if (null != zkClusterDir) { String clusterName = ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_MYID); String nodeName = name + "-" + clusterName + PROPERTIES_SUFFIX; // 读取cluster目录下的数据 clusterData = (ZkDataImpl) this.getZkData(zkClusterDir, nodeName); if (null != clusterData) { // comm路径的监控路径 String seqCluster = ZookeeperPath.FLOW_ZK_PATH_SEQUENCE_CLUSTER.getKey(); seqCluster = seqCluster + ZookeeperPath.ZK_SEPARATOR.getKey() + clusterData.getName(); this.zookeeperListen.watchPath(currZkPath, seqCluster); } } // 如果配制了单独节点的信息,以公共的名称，写入当前的值 if (clusterData != null && commData != null) { // 读取公共节点的信息 this.writeMapFile(commData.getName(), clusterData.getValue()); } else if (commData != null) { // 读取当前集群中特有的节点的信息 this.writeMapFile(commData.getName(), commData.getValue()); } } /** * 读取 mapFile文件的信息 * 方法描述 * @param name 名称信息 * @return * @创建日期 2016年9月18日 */ private void writeMapFile(String name, String value) { // 加载数据 String path = RuleszkToxmlLoader.class.getClassLoader().getResource(ZookeeperPath.ZK_LOCAL_WRITE_PATH.getKey()) .getPath(); checkNotNull(path, "write Map file curr Path :" + path + " is null! must is not null"); path = new File(path).getPath() + File.separator; path += name; // 进行数据写入 try { Files.write(value.getBytes(), new File(path)); } catch (IOException e1) { e1.printStackTrace(); } } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/zktoxml/listen/ServerzkToxmlLoader.java ================================================ package io.mycat.config.loader.zkprocess.zktoxml.listen; import static com.google.common.base.Preconditions.checkNotNull; import java.io.File; import java.io.IOException; import java.util.List; import org.apache.curator.framework.CuratorFramework; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.io.Files; import io.mycat.MycatServer; import io.mycat.config.loader.console.ZookeeperPath; import io.mycat.config.loader.zkprocess.comm.NotiflyService; import io.mycat.config.loader.zkprocess.comm.ZkConfig; import io.mycat.config.loader.zkprocess.comm.ZkParamCfg; import io.mycat.config.loader.zkprocess.comm.ZookeeperProcessListen; import io.mycat.config.loader.zkprocess.entity.Server; import io.mycat.config.loader.zkprocess.entity.server.System; import io.mycat.config.loader.zkprocess.entity.server.user.User; import io.mycat.config.loader.zkprocess.parse.ParseJsonServiceInf; import io.mycat.config.loader.zkprocess.parse.ParseXmlServiceInf; import io.mycat.config.loader.zkprocess.parse.XmlProcessBase; import io.mycat.config.loader.zkprocess.parse.entryparse.server.json.SystemJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.server.json.UserJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.server.xml.ServerParseXmlImpl; import io.mycat.config.loader.zkprocess.zookeeper.DataInf; import io.mycat.config.loader.zkprocess.zookeeper.DiretoryInf; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkDataImpl; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkDirectoryImpl; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkMultLoader; import io.mycat.manager.response.ReloadConfig; /** * 进行server的文件从zk中加载 * 源文件名：ServerzkToxmlLoader.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class ServerzkToxmlLoader extends ZkMultLoader implements NotiflyService { /** * 日志 * @字段说明 LOGGER */ private static final Logger LOGGER = LoggerFactory.getLogger(ServerzkToxmlLoader.class); /** * 当前文件中的zkpath信息 * @字段说明 currZkPath */ private final String currZkPath; /** * 写入本地的文件路径 * @字段说明 WRITEPATH */ private static final String WRITEPATH = "server.xml"; /** * index_to_charset文件的路径信息 * @字段说明 SCHEMA_PATH */ private static final String INDEX_TOCHARSET_PATH = "index_to_charset.properties"; /** * server的xml的转换信息 * @字段说明 parseServerXMl */ private ParseXmlServiceInf parseServerXMl; /** * system信息 * @字段说明 parseJsonserver */ private ParseJsonServiceInf parseJsonSystem = new SystemJsonParse(); /** * system信息 * @字段说明 parseJsonserver */ private ParseJsonServiceInf> parseJsonUser = new UserJsonParse(); /** * zk监控路径 * @字段说明 zookeeperListen */ private ZookeeperProcessListen zookeeperListen; public ServerzkToxmlLoader(ZookeeperProcessListen zookeeperListen, CuratorFramework curator, XmlProcessBase xmlParseBase) { this.setCurator(curator); this.zookeeperListen = zookeeperListen; // 获得当前集群的名称 String serverPath = zookeeperListen.getBasePath(); serverPath = serverPath + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SERVER.getKey(); currZkPath = serverPath; // 将当前自己注册为事件接收对象 this.zookeeperListen.addListen(serverPath, this); // 生成xml与类的转换信息 parseServerXMl = new ServerParseXmlImpl(xmlParseBase); } @Override public boolean notiflyProcess() throws Exception { // 1,将集群server目录下的所有集群按层次结构加载出来 // 通过组合模式进行zk目录树的加载 DiretoryInf serverDirectory = new ZkDirectoryImpl(currZkPath, null); // 进行递归的数据获取 this.getTreeDirectory(currZkPath, ZookeeperPath.FLOW_ZK_PATH_SERVER.getKey(), serverDirectory); // 从当前的下一级开始进行遍历,获得到 ZkDirectoryImpl zkDirectory = (ZkDirectoryImpl) serverDirectory.getSubordinateInfo().get(0); Server server = this.zktoServerBean(zkDirectory); // 读取当前集群中当前节点的特殊的配制信息 Server currSer = this.zktoServerBeanByCurrNode(zkDirectory); // 为当前的参数赋新值 if (null != currSer) { server.getSystem().setNewValue(currSer.getSystem()); } LOGGER.info("ServerzkToxmlLoader notiflyProcess zk to object zk server Object :" + server); // 数配制信息写入文件 String path = ServerzkToxmlLoader.class.getClassLoader().getResource(ZookeeperPath.ZK_LOCAL_WRITE_PATH.getKey()) .getPath(); path = new File(path).getPath() + File.separator; path += WRITEPATH; LOGGER.info("ServerzkToxmlLoader notiflyProcess zk to object writePath :" + path); this.parseServerXMl.parseToXmlWrite(server, path, "server"); LOGGER.info("ServerzkToxmlLoader notiflyProcess zk to object zk server write :" + path + " is success"); // 得到server对象的目录信息 DataInf indexToCharSet = this.getZkData(zkDirectory, INDEX_TOCHARSET_PATH); if (null != indexToCharSet) { if (indexToCharSet instanceof ZkDataImpl) { ZkDataImpl dataImpl = (ZkDataImpl) indexToCharSet; this.writeProperties(dataImpl.getName(), dataImpl.getValue()); } LOGGER.info("ServerzkToxmlLoader notiflyProcess zk to write index_to_charset.properties is success"); } if (MycatServer.getInstance().getProcessors() != null) ReloadConfig.reload(); return true; } /** * 将zk上面的信息转换为javabean对象 * 方法描述 * @param zkDirectory * @return * @创建日期 2016年9月17日 */ private Server zktoServerBean(DiretoryInf zkDirectory) { Server server = new Server(); // 得到server对象的目录信息 DataInf serverZkDirectory = this.getZkData(zkDirectory, ZookeeperPath.FLOW_ZK_PATH_SERVER_DEFAULT.getKey()); System systemValue = parseJsonSystem.parseJsonToBean(serverZkDirectory.getDataValue()); server.setSystem(systemValue); this.zookeeperListen.watchPath(currZkPath, ZookeeperPath.FLOW_ZK_PATH_SERVER_DEFAULT.getKey()); // 得到user的信息 DataInf userZkDirectory = this.getZkData(zkDirectory, ZookeeperPath.FLOW_ZK_PATH_SERVER_USER.getKey()); List userList = parseJsonUser.parseJsonToBean(userZkDirectory.getDataValue()); server.setUser(userList); // 用户路径的监控 this.zookeeperListen.watchPath(currZkPath, ZookeeperPath.FLOW_ZK_PATH_SERVER_USER.getKey()); return server; } /** * 加载当前节点的特殊配制信息 * 方法描述 * @param zkDirectory * @return * @创建日期 2016年9月17日 */ private Server zktoServerBeanByCurrNode(DiretoryInf zkDirectory) { Server server = null; // 得到集群节点的配制信息 DiretoryInf directory = this.getZkDirectory(zkDirectory, ZookeeperPath.FLOW_ZK_PATH_SERVER_CLUSTER.getKey()); if (null != directory) { // 获得当前myid的名称 String myid = ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_MYID); // 获邓当前节点的信息 DataInf currDataCfg = this.getZkData(directory, myid); // 如果当前节点存在配制信息，则加载 if (null != currDataCfg) { server = new Server(); System systemValue = parseJsonSystem.parseJsonToBean(currDataCfg.getDataValue()); server.setSystem(systemValue); if (currDataCfg instanceof ZkDataImpl) { ZkDataImpl zkData = (ZkDataImpl) currDataCfg; // 监控的路径信息 String defaultWatchPath = ZookeeperPath.FLOW_ZK_PATH_SERVER_CLUSTER.getKey(); defaultWatchPath = defaultWatchPath + ZookeeperPath.ZK_SEPARATOR.getKey() + zkData.getName(); this.zookeeperListen.watchPath(currZkPath, defaultWatchPath); } } } return server; } /** * 写入本地文件配制信息 * 方法描述 * @param name 名称信息 * @return * @创建日期 2016年9月18日 */ private void writeProperties(String name, String value) { // 加载数据 String path = RuleszkToxmlLoader.class.getClassLoader().getResource(ZookeeperPath.ZK_LOCAL_WRITE_PATH.getKey()) .getPath(); checkNotNull(path, "write properties curr Path :" + path + " is null! must is not null"); path = new File(path).getPath() + File.separator; path += name; // 进行数据写入 try { Files.write(value.getBytes(), new File(path)); } catch (IOException e1) { e1.printStackTrace(); } } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/zookeeper/ClusterInfo.java ================================================ package io.mycat.config.loader.zkprocess.zookeeper; /** * Created by magicdoom on 2016/12/21. */ public class ClusterInfo { private int clusterSize; private String clusterNodes; public int getClusterSize() { return clusterSize; } public void setClusterSize(int clusterSize) { this.clusterSize = clusterSize; } public String getClusterNodes() { return clusterNodes; } public void setClusterNodes(String clusterNodes) { this.clusterNodes = clusterNodes; } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/zookeeper/DataInf.java ================================================ package io.mycat.config.loader.zkprocess.zookeeper; /** * 数据节点信息 * 源文件名：DataInf.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public interface DataInf { /** * 获取信息,以:分隔两个值 * @return */ String getDataInfo(); /** * 返回数据节点值信息 * 方法描述 * @return * @创建日期 2016年9月17日 */ String getDataValue(); } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/zookeeper/DiretoryInf.java ================================================ package io.mycat.config.loader.zkprocess.zookeeper; import java.util.List; /** * 目录接口信息 * 源文件名：DiretoryInf.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public interface DiretoryInf { /** * 获取当前的目录信息 * @return */ String getDiretoryInfo(); /** * 添加目录或者数据节点 * @param branch */ void add(DiretoryInf directory); /** * 添加数据节点信息 * 方法描述 * @param data * @创建日期 2016年9月15日 */ void add(DataInf data); /** * 获取子节点信息 * @return */ List getSubordinateInfo(); /** * 获取节点的名称 * @字段说明 getDataName */ String getDataName(); } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/zookeeper/process/ZkDataImpl.java ================================================ package io.mycat.config.loader.zkprocess.zookeeper.process; import io.mycat.config.loader.zkprocess.zookeeper.DataInf; /** * 数据节点信息 * 源文件名：DataImpl.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class ZkDataImpl implements DataInf { /** * 名称信息 * @字段说明 name */ private String name; /** * 当前值信息 * @字段说明 value */ private String value; public ZkDataImpl(String name, String value) { super(); this.name = name; this.value = value; } @Override public String getDataInfo() { return this.name + ":" + this.value; } public String getName() { return name; } public void setName(String name) { this.name = name; } public String getValue() { return value; } public void setValue(String value) { this.value = value; } @Override public String getDataValue() { return this.value; } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/zookeeper/process/ZkDirectoryImpl.java ================================================ package io.mycat.config.loader.zkprocess.zookeeper.process; import java.util.ArrayList; import java.util.List; import io.mycat.config.loader.zkprocess.zookeeper.DataInf; import io.mycat.config.loader.zkprocess.zookeeper.DiretoryInf; /** * zk的目录节点信息 * 源文件名：ZkDirectoryMsg.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class ZkDirectoryImpl implements DiretoryInf { /** * 整个节点信息 * @字段说明 subordinateInfo */ private List subordinateInfoList = new ArrayList(); /** * 节点的名称信息 * @字段说明 name */ private String name; /** * 当前节点的数据信息 * @字段说明 value */ private String value; public ZkDirectoryImpl(String name, String value) { this.name = name; this.value = value; } public String getName() { return name; } public void setName(String name) { this.name = name; } public String getValue() { return value; } public void setValue(String value) { this.value = value; } @Override public String getDiretoryInfo() { return name + ":" + value; } @Override public void add(DiretoryInf branch) { this.subordinateInfoList.add(branch); } @Override public List getSubordinateInfo() { return this.subordinateInfoList; } @Override public void add(DataInf data) { this.subordinateInfoList.add(data); } @Override public String getDataName() { return this.name; } } ================================================ FILE: src/main/java/io/mycat/config/loader/zkprocess/zookeeper/process/ZkMultLoader.java ================================================ package io.mycat.config.loader.zkprocess.zookeeper.process; import static com.google.common.base.Preconditions.checkNotNull; import java.nio.charset.StandardCharsets; import java.util.List; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.utils.ZKPaths; import org.apache.zookeeper.data.Stat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.gson.Gson; import io.mycat.config.loader.console.ZookeeperPath; import io.mycat.config.loader.zkprocess.zookeeper.DataInf; import io.mycat.config.loader.zkprocess.zookeeper.DiretoryInf; /** * 进行zk获取数据类信息 * 源文件名：AbstractLoader.java * 文件版本：1.0.0 * 创建作者：liujun * 创建日期：2016年9月15日 * 修改作者：liujun * 修改日期：2016年9月15日 * 文件描述：TODO * 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. */ public class ZkMultLoader { /** * 日志 * @字段说明 LOGGER */ private static final Logger LOGGER = LoggerFactory.getLogger(ZkMultLoader.class); /** * zk连接信息 * @字段说明 curator */ private CuratorFramework curator; /** * 进行数据转换操作 * @字段说明 gson */ private Gson gson = new Gson(); /** * 得到树形节点信息 * 方法描述 * @param path * @param zkDirectory * @throws Exception * @创建日期 2016年9月15日 */ public void getTreeDirectory(String path, String name, DiretoryInf zkDirectory) throws Exception { boolean check = this.checkPathExists(path); // 如果文件存在，则继续遍历 if (check) { // 首先获取当前节点的数据，然后再递归 String currDate = this.getDataToString(path); List childPathList = this.getChildNames(path); // 如果存在子目录信息，则进行 if (null != childPathList && !childPathList.isEmpty()) { DiretoryInf directory = new ZkDirectoryImpl(name, currDate); // 添加目录节点信息 zkDirectory.add(directory); for (String childPath : childPathList) { this.getTreeDirectory(path + ZookeeperPath.ZK_SEPARATOR.getKey() + childPath, childPath, directory); } } // 添加当前的数据节点信息 else { zkDirectory.add(new ZkDataImpl(name, currDate)); } } } /** * 检查文件是否存在 * 方法描述 * @param path * @return * @创建日期 2016年9月21日 */ protected boolean checkPathExists(String path) { try { Stat state = this.curator.checkExists().forPath(path); if (null != state) { return true; } } catch (Exception e) { e.printStackTrace(); } return false; } /** * get data from zookeeper and convert to string with check not null. */ protected String getDataToString(String path) throws Exception { byte[] raw = curator.getData().forPath(path); checkNotNull(raw, "data of " + path + " must be not null!"); return byteToString(raw); } /** * get child node name list based on path from zookeeper. * @throws Exception */ protected List getChildNames(String path) throws Exception { return curator.getChildren().forPath(path); } protected void checkAndwriteString(String parentPath, String currpath, String value) throws Exception { checkNotNull(parentPath, "data of path" + parentPath + " must be not null!"); checkNotNull(currpath, "data of path" + currpath + " must be not null!"); checkNotNull(value, "data of value:" + value + " must be not null!"); String nodePath = ZKPaths.makePath(parentPath, currpath); Stat stat = curator.checkExists().forPath(nodePath); if (null == stat) { this.createPath(nodePath); } LOGGER.debug("ZkMultLoader write file :" + nodePath + ", value :" + value); curator.setData().inBackground().forPath(nodePath, value.getBytes()); } /** * 创建配制信息 * 方法描述 * @param configKey 配制的当前路径名称信息 * @param filterInnerMap 最终的信息是否为map * @param configDirectory 配制的目录 * @param restDirectory 子目录信息 * @创建日期 2016年9月11日 */ public boolean createPath(String path) { // 得到当前的目录信息 LOGGER.trace("createPath child path is {}", path); boolean result = true; try { // 进行目录的创建操作 ZKPaths.mkdirs(curator.getZookeeperClient().getZooKeeper(), path); } catch (Exception e) { LOGGER.error(" createPath error", e); result = false; } return result; } protected void writeZkString(String path, String value) throws Exception { checkNotNull(path, "data of path" + path + " must be not null!"); checkNotNull(value, "data of value:" + value + " must be not null!"); curator.setData().forPath(path, value.getBytes()); } /** * raw byte data to string */ protected String byteToString(byte[] raw) { // return empty json {}. if (raw.length == 0) { return "{}"; } return new String(raw, StandardCharsets.UTF_8); } /** * 通过名称数据节点信息 * 方法描述 * @param zkDirectory * @param name * @return * @创建日期 2016年9月16日 */ protected DataInf getZkData(DiretoryInf zkDirectory, String name) { List list = zkDirectory.getSubordinateInfo(); if (null != list && !list.isEmpty()) { for (Object directObj : list) { if (directObj instanceof ZkDataImpl) { ZkDataImpl zkDirectoryValue = (ZkDataImpl) directObj; if (name.equals(zkDirectoryValue.getName())) { return zkDirectoryValue; } } } } return null; } /** * 通过名称获得目录节点信息 * 方法描述 * @param zkDirectory * @param name * @return * @创建日期 2016年9月16日 */ protected DiretoryInf getZkDirectory(DiretoryInf zkDirectory, String name) { List list = zkDirectory.getSubordinateInfo(); if (null != list && !list.isEmpty()) { for (Object directObj : list) { if (directObj instanceof DiretoryInf) { DiretoryInf zkDirectoryValue = (DiretoryInf) directObj; if (name.equals(zkDirectoryValue.getDataName())) { return zkDirectoryValue; } } } } return null; } public CuratorFramework getCurator() { return curator; } public void setCurator(CuratorFramework curator) { this.curator = curator; } public Gson getGson() { return gson; } public void setGson(Gson gson) { this.gson = gson; } } ================================================ FILE: src/main/java/io/mycat/config/model/ClusterConfig.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.model; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import io.mycat.config.util.ConfigException; import io.mycat.config.util.ConfigUtil; import io.mycat.util.SplitUtil; /** * @author mycat */ public class ClusterConfig { private final Map nodes; private final Map> groups; public ClusterConfig(Element root, int port) { nodes = Collections.unmodifiableMap(loadNode(root, port)); groups = Collections.unmodifiableMap(loadGroup(root, nodes)); } public Map getNodes() { return nodes; } public Map> getGroups() { return groups; } private static Map loadNode(Element root, int port) { Map nodes = new HashMap(); NodeList list = root.getElementsByTagName("node"); Set hostSet = new HashSet(); for (int i = 0, n = list.getLength(); i < n; i++) { Node node = list.item(i); if (node instanceof Element) { Element element = (Element) node; String name = element.getAttribute("name").trim(); if (nodes.containsKey(name)) { throw new ConfigException("node name duplicated :" + name); } Map props = ConfigUtil.loadElements(element); String host = (String) props.get("host"); if (null == host || "".equals(host)) { throw new ConfigException("host empty in node: " + name); } if (hostSet.contains(host)) { throw new ConfigException("node host duplicated :" + host); } String wei = (String) props.get("weight"); if (null == wei || "".equals(wei)) { throw new ConfigException("weight should not be null in host:" + host); } int weight = Integer.parseInt(wei); if (weight <= 0) { throw new ConfigException("weight should be > 0 in host:" + host + " weight:" + weight); } MycatNodeConfig conf = new MycatNodeConfig(name, host, port, weight); nodes.put(name, conf); hostSet.add(host); } } return nodes; } private static Map> loadGroup(Element root, Map nodes) { Map> groups = new HashMap>(); NodeList list = root.getElementsByTagName("group"); for (int i = 0, n = list.getLength(); i < n; i++) { Node node = list.item(i); if (node instanceof Element) { Element e = (Element) node; String groupName = e.getAttribute("name").trim(); if (groups.containsKey(groupName)) { throw new ConfigException("group duplicated : " + groupName); } Map props = ConfigUtil.loadElements(e); String value = (String) props.get("nodeList"); if (null == value || "".equals(value)) { throw new ConfigException("group should contain 'nodeList'"); } String[] sList = SplitUtil.split(value, ',', true); if (null == sList || sList.length == 0) { throw new ConfigException("group should contain 'nodeList'"); } for (String s : sList) { if (!nodes.containsKey(s)) { throw new ConfigException("[ node :" + s + "] in [ group:" + groupName + "] doesn't exist!"); } } List nodeList = Arrays.asList(sList); groups.put(groupName, nodeList); } } if (!groups.containsKey("default")) { List nodeList = new ArrayList(nodes.keySet()); groups.put("default", nodeList); } return groups; } } ================================================ FILE: src/main/java/io/mycat/config/model/DBHostConfig.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.model; public class DBHostConfig { private long idleTimeout = SystemConfig.DEFAULT_IDLE_TIMEOUT; // 连接池中连接空闲超时时间 private final String hostName; private final String ip; private final int port; private final String url; private final String user; private final String password; private final String encryptPassword; //密文 private final boolean checkAlive; private int maxCon ; private int minCon ; private String dbType; private String filters="mergeStat"; private long logTime = 300000; private int weight; public String getDbType() { return dbType; } public void setDbType(String dbType) { this.dbType = dbType; } public DBHostConfig(String hostName, String ip, int port, String url, String user, String password, String encryptPassword, boolean checkAlive) { super(); this.hostName = hostName; this.ip = ip; this.port = port; this.url = url; this.user = user; this.password = password; this.encryptPassword = encryptPassword; this.checkAlive = checkAlive; } public long getIdleTimeout() { return idleTimeout; } public void setIdleTimeout(long idleTimeout) { this.idleTimeout = idleTimeout; } public int getMaxCon() { return maxCon; } public void setMaxCon(int maxCon) { this.maxCon = maxCon; } public int getMinCon() { return minCon; } public void setMinCon(int minCon) { this.minCon = minCon; } public String getHostName() { return hostName; } public String getIp() { return ip; } public int getPort() { return port; } public String getUrl() { return url; } public String getUser() { return user; } public String getFilters() { return filters; } public void setFilters(String filters) { this.filters = filters; } public String getPassword() { return password; } public long getLogTime() { return logTime; } public void setLogTime(long logTime) { this.logTime = logTime; } public int getWeight() { return weight; } public void setWeight(int weight) { this.weight = weight; } public String getEncryptPassword() { return this.encryptPassword; } @Override public String toString() { return "DBHostConfig [hostName=" + hostName + ", url=" + url + "]"; } public boolean isCheckAlive() { return checkAlive; } } ================================================ FILE: src/main/java/io/mycat/config/model/DataHostConfig.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.model; import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import com.google.common.collect.Iterables; import io.mycat.backend.datasource.PhysicalDBPool; /** * Datahost is a group of DB servers which is synchronized with each other * * @author wuzhih * */ public class DataHostConfig { public static final int NOT_SWITCH_DS = -1; public static final int DEFAULT_SWITCH_DS = 1; public static final int SYN_STATUS_SWITCH_DS = 2; public static final int CLUSTER_STATUS_SWITCH_DS = 3; private static final Pattern pattern = Pattern.compile("\\s*show\\s+slave\\s+status\\s*",Pattern.CASE_INSENSITIVE); private static final Pattern patternCluster = Pattern.compile("\\s*show\\s+status\\s+like\\s+'wsrep%'",Pattern.CASE_INSENSITIVE); private String name; private int maxCon = SystemConfig.DEFAULT_POOL_SIZE; private int minCon = 10; private int balance = PhysicalDBPool.BALANCE_NONE; private int balanceType = PhysicalDBPool.RANDOM; private int writeType = PhysicalDBPool.WRITE_ONLYONE_NODE; private final String dbType; private final String dbDriver; private final DBHostConfig[] writeHosts; private final Map readHosts; private String hearbeatSQL; private boolean isShowSlaveSql=false; private boolean isShowClusterSql=false; private String connectionInitSql; private int slaveThreshold = -1; private final int switchType; private String filters="mergeStat"; private long logTime=300000; private boolean tempReadHostAvailable = false; //如果写服务挂掉, 临时读服务是否继续可用 private final Set dataNodes; //包含的所有dataNode名字 private String slaveIDs; private int maxRetryCount = 3; // 心跳失败时候重试的次数. @auth zwy public static final String FOVER_NOT_SWITCH_DS = "1"; public static final String CAN_SWITCH_DS = "0"; private String notSwitch = CAN_SWITCH_DS; public DataHostConfig(String name, String dbType, String dbDriver, DBHostConfig[] writeHosts, Map readHosts,int switchType,int slaveThreshold, boolean tempReadHostAvailable) { super(); this.name = name; this.dbType = dbType; this.dbDriver = dbDriver; this.writeHosts = writeHosts; this.readHosts = readHosts; this.switchType=switchType; this.slaveThreshold=slaveThreshold; this.tempReadHostAvailable = tempReadHostAvailable; this.dataNodes = new HashSet<>(); } public boolean isTempReadHostAvailable() { return this.tempReadHostAvailable; } public int getSlaveThreshold() { return slaveThreshold; } public void setSlaveThreshold(int slaveThreshold) { this.slaveThreshold = slaveThreshold; } public int getSwitchType() { return switchType; } public String getConnectionInitSql() { return connectionInitSql; } public void setConnectionInitSql(String connectionInitSql) { this.connectionInitSql = connectionInitSql; } public int getWriteType() { return writeType; } public void setWriteType(int writeType) { this.writeType = writeType; } public String getName() { return name; } public void setName(String name) { this.name = name; } public boolean isShowSlaveSql() { return isShowSlaveSql; } public int getMaxCon() { return maxCon; } public void setMaxCon(int maxCon) { this.maxCon = maxCon; } public int getMinCon() { return minCon; } public void setMinCon(int minCon) { this.minCon = minCon; } public String getSlaveIDs() { return slaveIDs; } public void setSlaveIDs(String slaveIDs) { this.slaveIDs = slaveIDs; } public int getBalance() { return balance; } public void setBalance(int balance) { this.balance = balance; } public int getBalanceType() { return balanceType; } public void setBalanceType(int balanceType) { this.balanceType = balanceType; } public String getDbType() { return dbType; } public String getDbDriver() { return dbDriver; } public DBHostConfig[] getWriteHosts() { return writeHosts; } public Map getReadHosts() { return readHosts; } public String getHearbeatSQL() { return hearbeatSQL; } public void setHearbeatSQL(String heartbeatSQL) { this.hearbeatSQL = heartbeatSQL; Matcher matcher = pattern.matcher(heartbeatSQL); if (matcher.find()) { isShowSlaveSql=true; } Matcher matcher2 = patternCluster.matcher(heartbeatSQL); if (matcher2.find()) { isShowClusterSql=true; } } public String getFilters() { return filters; } public void setFilters(String filters) { this.filters = filters; } public long getLogTime() { return logTime; } public boolean isShowClusterSql() { return this.isShowClusterSql; } public void setLogTime(long logTime) { this.logTime = logTime; } public void addDataNode(String name){ this.dataNodes.add(name); } public String getRandomDataNode() { int index = (int) (Math.random() * dataNodes.size()); return Iterables.get(dataNodes,index); } public boolean containDataNode(String randomDn) { return dataNodes.contains(randomDn); } public int getMaxRetryCount() { return maxRetryCount; } public void setMaxRetryCount(int maxRetryCount) { this.maxRetryCount = maxRetryCount; } public String getNotSwitch() { return notSwitch; } public void setNotSwitch(String notSwitch) { this.notSwitch = notSwitch; } public boolean isJDBCDriver() { return "jdbc".equalsIgnoreCase(dbDriver); } public boolean isNativeDriver() { return "native".equalsIgnoreCase(dbDriver); } } ================================================ FILE: src/main/java/io/mycat/config/model/DataNodeConfig.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.model; /** * 用于描述一个数据节点的配置 * * @author mycat */ public final class DataNodeConfig { private final String name; private final String database; private final String dataHost; public DataNodeConfig(String name, String database, String dataHost) { super(); this.name = name; this.database = database; this.dataHost = dataHost; } public String getName() { return name; } public String getDatabase() { return database; } public String getDataHost() { return dataHost; } } ================================================ FILE: src/main/java/io/mycat/config/model/FirewallConfig.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.model; import java.io.File; import java.io.InputStream; import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; import io.mycat.util.StringUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.xml.sax.EntityResolver; import org.xml.sax.InputSource; import org.xml.sax.SAXException; import com.alibaba.druid.wall.WallConfig; import com.alibaba.druid.wall.WallProvider; import com.alibaba.druid.wall.spi.MySqlWallProvider; import io.mycat.MycatServer; import io.mycat.config.MycatConfig; import io.mycat.config.loader.xml.XMLServerLoader; /** * 防火墙配置定义 * * @author songwie * @author zhuam */ public final class FirewallConfig { private static final Logger LOGGER = LoggerFactory.getLogger(FirewallConfig.class); private Map> whitehost;//具体host的白名单 private Map> whitehostMask;//网段的白名单 public static Pattern getMaskPattern(String host){ return Pattern.compile(host.replaceAll("\\.","\\\\.").replaceAll("[*]","[0-9]*").replaceAll("%","[0-9]*")); } public static String getHost(Pattern maskPattern){ return maskPattern.pattern().replaceAll("\\\\.","\\.").replaceAll("\\[0-9\\]",""); } private List blacklist; private boolean check = false; private WallConfig wallConfig = new WallConfig(); private static WallProvider provider ; public FirewallConfig() { } public void init(){ if(check){ provider = new MySqlWallProvider(wallConfig); provider.setBlackListEnable(true); } } public Map> getWhitehostMask() { return whitehostMask; } public void setWhitehostMask(Map> whitehostMask) { this.whitehostMask = whitehostMask; } public WallProvider getWallProvider(){ return provider; } public Map> getWhitehost() { return this.whitehost; } public void setWhitehost(Map> whitehost) { this.whitehost = whitehost; } /** * 通过manager端命令动态配置白名单，配置防火墙方法之一，一共有两处，另一处: * @see XMLServerLoader * * @modification 修改增加网段白名单 * @date 2016/12/8 * @modifiedBy Hash Zhang */ public boolean addWhitehost(String host, List Users) { if (existsHost(host)){ return false; } else { if(host.contains("*")||host.contains("%")){ this.whitehostMask.put(getMaskPattern(host),Users); }else { this.whitehost.put(host, Users); } return true; } } public List getBlacklist() { return this.blacklist; } public void setBlacklist(List blacklist) { this.blacklist = blacklist; } public WallProvider getProvider() { return provider; } public boolean existsHost(String host) { return this.whitehost!=null && whitehost.get(host)!=null ; } public boolean canConnect(String host,String user) { if(whitehost==null || whitehost.size()==0){ MycatConfig config = MycatServer.getInstance().getConfig(); Map users = config.getUsers(); return users.containsKey(user); }else{ List list = whitehost.get(host); if(list==null){ return false; } for(UserConfig userConfig : list){ if(userConfig.getName().equals(user)){ return true; } } } return false ; } public static void setProvider(WallProvider provider) { FirewallConfig.provider = provider; } public void setWallConfig(WallConfig wallConfig) { this.wallConfig = wallConfig; } public boolean isCheck() { return this.check; } public void setCheck(boolean check) { this.check = check; } public WallConfig getWallConfig() { return this.wallConfig; } public synchronized static void updateToFile(String host, List userConfigs) throws Exception{ LOGGER.debug("set white host:" + host + "user:" + userConfigs); String filename = SystemConfig.getHomePath()+ File.separator +"conf"+ File.separator +"server.xml"; //String filename = "E:\\MyProject\\Mycat-Server\\src\\main\\resources\\server.xml"; DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); factory.setNamespaceAware(false); factory.setValidating(false); DocumentBuilder builder = factory.newDocumentBuilder(); builder.setEntityResolver(new IgnoreDTDEntityResolver()); Document xmldoc = builder.parse(filename); Element whitehost = (Element) xmldoc.getElementsByTagName("whitehost").item(0); Element firewall = (Element) xmldoc.getElementsByTagName("firewall").item(0); if (firewall == null) { firewall = xmldoc.createElement("firewall"); Element root = xmldoc.getDocumentElement(); root.appendChild(firewall); if(whitehost==null){ whitehost = xmldoc.createElement("whitehost"); firewall.appendChild(whitehost); } } for(UserConfig userConfig : userConfigs){ String user = userConfig.getName(); Element hostEle = xmldoc.createElement("host"); hostEle.setAttribute("host", host); hostEle.setAttribute("user", user); whitehost.appendChild(hostEle); } TransformerFactory factory2 = TransformerFactory.newInstance(); Transformer former = factory2.newTransformer(); String systemId = xmldoc.getDoctype().getSystemId(); if(systemId!=null){ former.setOutputProperty(javax.xml.transform.OutputKeys.DOCTYPE_SYSTEM, systemId); } former.transform(new DOMSource(xmldoc), new StreamResult(new File(filename))); } static class IgnoreDTDEntityResolver implements EntityResolver{ public InputSource resolveEntity(java.lang.String publicId, java.lang.String systemId) throws SAXException, java.io.IOException{ if (systemId.contains("server.dtd")){ //InputSource is = new InputSource(new ByteArrayInputStream("".getBytes())); InputStream dtd = XMLServerLoader.class.getResourceAsStream("/server.dtd"); InputSource is = new InputSource(dtd); return is; } else { return null; } } } // public static void main(String[] args) throws Exception { // List userConfigs = new ArrayList(); // UserConfig user = new UserConfig(); // user.setName("mycat"); // userConfigs.add(user); // updateToFile("127.0.0.1",userConfigs); // } } ================================================ FILE: src/main/java/io/mycat/config/model/MycatNodeConfig.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.model; /** * @author mycat * @author mycat */ public final class MycatNodeConfig { private String name; private String host; private int port; private int weight; public MycatNodeConfig(String name, String host, int port, int weight) { this.name = name; this.host = host; this.port = port; this.weight = weight; } public String getName() { return name; } public void setName(String name) { this.name = name; } public String getHost() { return host; } public void setHost(String host) { this.host = host; } public int getPort() { return port; } public void setPort(int port) { this.port = port; } public int getWeight() { return weight; } public void setWeight(int weight) { this.weight = weight; } @Override public String toString() { return new StringBuilder().append("[name=").append(name).append(",host=").append(host).append(",port=") .append(port).append(",weight=").append(weight).append(']').toString(); } } ================================================ FILE: src/main/java/io/mycat/config/model/SchemaConfig.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.model; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Random; import java.util.Set; import java.util.concurrent.ThreadLocalRandom; /** * @author mycat */ public class SchemaConfig { // private final Random random = new Random(); private final String name; private final Map tables; private final boolean noSharding; private final String dataNode; private final Set metaDataNodes; private final Set allDataNodes; private String randomDataNode = null; /** * when a select sql has no limit condition ,and default max limit to * prevent memory problem when return a large result set */ private final int defaultMaxLimit; private final boolean checkSQLSchema; private boolean needSupportMultiDBType=false; private String defaultDataNodeDbType; /** * key is join relation ,A.ID=B.PARENT_ID value is Root Table ,if a->b*->c* * ,then A is root table */ private final Map joinRel2TableMap = new HashMap(); private final String[] allDataNodeStrArr; private Map dataNodeDbTypeMap=new HashMap<>(); public SchemaConfig(String name, String dataNode, Map tables, int defaultMaxLimit, boolean checkSQLschema,String randomDataNode) { this.name = name; this.dataNode = dataNode; this.checkSQLSchema = checkSQLschema; this.tables = tables; this.defaultMaxLimit = defaultMaxLimit; buildJoinMap(tables); this.noSharding = (tables == null || tables.isEmpty()); if (noSharding && dataNode == null) { throw new RuntimeException(name + " in noSharding mode schema must have default dataNode "); } this.metaDataNodes = buildMetaDataNodes(); this.allDataNodes = buildAllDataNodes(); // this.metaDataNodes = buildAllDataNodes(); if (this.allDataNodes != null && !this.allDataNodes.isEmpty()) { String[] dnArr = new String[this.allDataNodes.size()]; dnArr = this.allDataNodes.toArray(dnArr); this.allDataNodeStrArr = dnArr; } else { this.allDataNodeStrArr = null; } this.randomDataNode =randomDataNode; } public String getDefaultDataNodeDbType() { return defaultDataNodeDbType; } public void setDefaultDataNodeDbType(String defaultDataNodeDbType) { this.defaultDataNodeDbType = defaultDataNodeDbType; } public boolean isCheckSQLSchema() { return checkSQLSchema; } public int getDefaultMaxLimit() { return defaultMaxLimit; } private void buildJoinMap(Map tables2) { if (tables == null || tables.isEmpty()) { return; } for (TableConfig tc : tables.values()) { if (tc.isChildTable()) { TableConfig rootTc = tc.getRootParent(); String joinRel1 = tc.getName() + '.' + tc.getJoinKey() + '=' + tc.getParentTC().getName() + '.' + tc.getParentKey(); String joinRel2 = tc.getParentTC().getName() + '.' + tc.getParentKey() + '=' + tc.getName() + '.' + tc.getJoinKey(); joinRel2TableMap.put(joinRel1, rootTc); joinRel2TableMap.put(joinRel2, rootTc); } } } public boolean isNeedSupportMultiDBType() { return needSupportMultiDBType; } public void setNeedSupportMultiDBType(boolean needSupportMultiDBType) { this.needSupportMultiDBType = needSupportMultiDBType; } public Map getJoinRel2TableMap() { return joinRel2TableMap; } public String getName() { return name; } public String getDataNode() { return dataNode; } public Map getTables() { return tables; } public boolean isNoSharding() { return noSharding; } public Set getMetaDataNodes() { return metaDataNodes; } public Set getAllDataNodes() { return allDataNodes; } public Map getDataNodeDbTypeMap() { return dataNodeDbTypeMap; } public void setDataNodeDbTypeMap(Map dataNodeDbTypeMap) { this.dataNodeDbTypeMap = dataNodeDbTypeMap; } public String getRandomDataNode() { if (this.randomDataNode != null&&!"".equalsIgnoreCase(this.randomDataNode)){ return this.randomDataNode; } if (this.allDataNodeStrArr == null) { return null; } int index = Math.abs(ThreadLocalRandom.current().nextInt(Integer.MAX_VALUE)) % allDataNodeStrArr.length; return this.allDataNodeStrArr[index]; } /** * 取得含有不同Meta信息的数据节点,比如表和表结构。 */ private Set buildMetaDataNodes() { Set set = new HashSet(); if (!isEmpty(dataNode)) { set.add(dataNode); } if (!noSharding) { for (TableConfig tc : tables.values()) { set.add(tc.getDataNodes().get(0)); } } return set; } /** * 取得该schema的所有数据节点 */ private Set buildAllDataNodes() { Set set = new HashSet(); if (!isEmpty(dataNode)) { set.add(dataNode); } if (!noSharding) { for (TableConfig tc : tables.values()) { set.addAll(tc.getDataNodes()); } } return set; } private static boolean isEmpty(String str) { return ((str == null) || (str.length() == 0)); } } ================================================ FILE: src/main/java/io/mycat/config/model/SystemConfig.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.model; import java.io.File; import java.io.IOException; import io.mycat.config.Isolations; /** * 系统基础配置项 * * @author mycat */ public final class SystemConfig { public static final String SYS_HOME = "MYCAT_HOME"; private static final int DEFAULT_PORT = 8066; private static final int DEFAULT_MANAGER_PORT = 9066; private static final String DEFAULT_CHARSET = "utf8"; private static final String DEFAULT_SQL_PARSER = "druidparser";// fdbparser, druidparser private static final short DEFAULT_BUFFER_CHUNK_SIZE = 4096; private static final int DEFAULT_BUFFER_POOL_PAGE_SIZE = 512*1024*4; private static final short DEFAULT_BUFFER_POOL_PAGE_NUMBER = 64; private int removeGraveAccent; private int processorBufferLocalPercent; private static final int DEFAULT_PROCESSORS = Runtime.getRuntime().availableProcessors(); private int frontSocketSoRcvbuf = 1024 * 1024; private int frontSocketSoSndbuf = 4 * 1024 * 1024; private int backSocketSoRcvbuf = 4 * 1024 * 1024;// mysql 5.6 // net_buffer_length // defaut 4M private final static String RESERVED_SYSTEM_MEMORY_BYTES = "384m"; private final static String MEMORY_PAGE_SIZE = "1m"; private final static String SPILLS_FILE_BUFFER_SIZE = "2K"; private final static String DATANODE_SORTED_TEMP_DIR = "datanode"; private int backSocketSoSndbuf = 1024 * 1024; private int frontSocketNoDelay = 1; // 0=false private int backSocketNoDelay = 1; // 1=true public static final int DEFAULT_POOL_SIZE = 128;// 保持后端数据通道的默认最大值 public static final long DEFAULT_IDLE_TIMEOUT = 30 * 60 * 1000L; public static final long DEFAULT_AUTH_TIMEOUT = 15 * 1000L; private static final long DEFAULT_PROCESSOR_CHECK_PERIOD = 1 * 1000L; private static final long DEFAULT_DATANODE_IDLE_CHECK_PERIOD = 5 * 60 * 1000L; //连接空闲检查 private static final long DEFAULT_DATANODE_HEARTBEAT_PERIOD = 10 * 1000L; //心跳检查周期 private static final long DEFAULT_CLUSTER_HEARTBEAT_PERIOD = 5 * 1000L; private static final long DEFAULT_CLUSTER_HEARTBEAT_TIMEOUT = 10 * 1000L; private static final int DEFAULT_CLUSTER_HEARTBEAT_RETRY = 10; private static final int DEFAULT_MAX_LIMIT = 100; private static final String DEFAULT_CLUSTER_HEARTBEAT_USER = "_HEARTBEAT_USER_"; private static final String DEFAULT_CLUSTER_HEARTBEAT_PASS = "_HEARTBEAT_PASS_"; private static final int DEFAULT_PARSER_COMMENT_VERSION = 50148; private static final int DEFAULT_SQL_RECORD_COUNT = 10; private static final boolean DEFAULT_USE_ZK_SWITCH = false; private static final int DEFAULT_MAX_PREPAREDSTMT_COUNT = 16382; private int maxStringLiteralLength = 65535; private int frontWriteQueueSize = 2048; private String bindIp = "0.0.0.0"; private String fakeMySQLVersion = null; private int serverPort; private int managerPort; private String charset; private int processors; private int processorExecutor; private int timerExecutor; private int managerExecutor; private int serverBacklog = 2048; private long idleTimeout; private long authTimeout; private int catletClassCheckSeconds = 60; // sql execute timeout (second) private long sqlExecuteTimeout = 300; private long processorCheckPeriod; private long dataNodeIdleCheckPeriod; private long dataNodeHeartbeatPeriod; private String clusterHeartbeatUser; private String clusterHeartbeatPass; private long clusterHeartbeatPeriod; private long clusterHeartbeatTimeout; private int clusterHeartbeatRetry; private int txIsolation; private int parserCommentVersion; private int sqlRecordCount; private String sequnceHandlerPattern = SEQUENCEHANDLER_PATTERN; /** * 预处理占位符最大数量 */ private int maxPreparedStmtCount; // a page size private int bufferPoolPageSize; //minimum allocation unit private short bufferPoolChunkSize; // buffer pool page number private short bufferPoolPageNumber; //大结果集阈值，默认512kb private int maxResultSet=512*1024; //大结果集拒绝策略次数过滤限制,默认10次 private int bigResultSizeSqlCount=10; //大结果集拒绝策咯，bufferpool使用率阈值(0-100)，默认80% private int bufferUsagePercent=80; //大结果集保护策咯，0:不开启,1:级别1为在当前mucat bufferpool //使用率大于bufferUsagePercent阈值时，拒绝超过defaultBigResultSizeSqlCount //sql次数阈值并且符合超过大结果集阈值maxResultSet的所有sql //默认值0 private int flowControlRejectStrategy=0; //清理大结果集记录周期 private long clearBigSqLResultSetMapMs=10*60*1000; private int defaultMaxLimit = DEFAULT_MAX_LIMIT; public static final int SEQUENCEHANDLER_LOCALFILE = 0; public static final int SEQUENCEHANDLER_MYSQLDB = 1; public static final int SEQUENCEHANDLER_LOCAL_TIME = 2; public static final int SEQUENCEHANDLER_ZK_DISTRIBUTED = 3; public static final int SEQUENCEHANDLER_ZK_GLOBAL_INCREMENT = 4; public static final int SEQUENCEHANDLER_DEF_GLOBAL_INCREMENT = 5; public static String sequenceHanlderClass = null; public static final String SEQUENCEHANDLER_PATTERN = "(?:(\\s*next\\s+value\\s+for\\s*MYCATSEQ_(\\w+))(,|\\)|\\s)*)+"; private final int DEFAULT_SEQUNCE_MYSQL_RETRY_COUT=4; //mysql全局序列默认重试次数 private final long DEFAULT_SEQUNCE_MYSQL_WATI_TIME=10 * 1000;//mysql db方式默认等待时间 private int sequnceMySqlRetryCount = DEFAULT_SEQUNCE_MYSQL_RETRY_COUT; private long sequnceMySqlWaitTime = DEFAULT_SEQUNCE_MYSQL_WATI_TIME; private int ignoreUnknownCommand = 0;//io/mycat/net/handler/FrontendCommandHandler.java:忽略未知命令 /* * 注意！！！ 目前mycat支持的MySQL版本，如果后续有新的MySQL版本,请添加到此数组， 对于MySQL的其他分支， * 比如MariaDB目前版本号已经到10.1.x，但是其驱动程序仍然兼容官方的MySQL,因此这里版本号只需要MySQL官方的版本号即可。 */ public static final String[] MySQLVersions = { "5.5", "5.6", "5.7" }; private int sequenceHandlerType = SEQUENCEHANDLER_LOCALFILE; private String sqlInterceptor = "io.mycat.server.interceptor.impl.DefaultSqlInterceptor"; private String sqlInterceptorType = "select"; private String sqlInterceptorFile = System.getProperty("user.dir")+"/logs/sql.txt"; public static final int MUTINODELIMIT_SMALL_DATA = 0; public static final int MUTINODELIMIT_LAR_DATA = 1; private int mutiNodeLimitType = MUTINODELIMIT_SMALL_DATA; public static final int MUTINODELIMIT_PATCH_SIZE = 100; private int mutiNodePatchSize = MUTINODELIMIT_PATCH_SIZE; private String defaultSqlParser = DEFAULT_SQL_PARSER; private int usingAIO = 0; private int packetHeaderSize = 4; private int maxPacketSize = 16 * 1024 * 1024; private int mycatNodeId=1; private int useCompression =0; private int useSqlStat = 1; //子查询中存在关联查询的情况下,检查关联字段中是否有分片字段 .默认 false private boolean subqueryRelationshipCheck = false; // 是否使用HandshakeV10Packet来与client进行通讯, 1:是 , 0:否(使用HandshakePacket) // 使用HandshakeV10Packet为的是兼容高版本的jdbc驱动, 后期稳定下来考虑全部采用HandshakeV10Packet来通讯 private int useHandshakeV10 = 0; //处理分布式事务开关，默认为不过滤分布式事务 private int handleDistributedTransactions = 0; private int checkTableConsistency = 0; private long checkTableConsistencyPeriod = CHECKTABLECONSISTENCYPERIOD; private final static long CHECKTABLECONSISTENCYPERIOD = 1 * 60 * 1000; private int processorBufferPoolType = 0; // 全局表一致性检测任务，默认24小时调度一次 private static final long DEFAULT_GLOBAL_TABLE_CHECK_PERIOD = 24 * 60 * 60 * 1000L; private int useGlobleTableCheck = 1; // 全局表一致性检查开关 private long glableTableCheckPeriod; // 如果为true的话 严格遵守隔离级别,不会在仅仅只有select语句的时候在事务中切换连接 private boolean strictTxIsolation = false; /** * Mycat 使用 Off Heap For Merge/Order/Group/Limit计算相关参数 */ /** * 是否启用Off Heap for Merge 1-启用，0-不启用 */ private int useOffHeapForMerge; /** *页大小,对应MemoryBlock的大小，单位为M */ private String memoryPageSize; /** * DiskRowWriter写磁盘是临时写Buffer，单位为K */ private String spillsFileBufferSize; /** * 启用结果集流输出，不经过merge模块, */ private int useStreamOutput; /** * 该变量仅在Merge使用On Heap * 内存方式时起作用，如果使用Off Heap内存方式 * 那么可以认为-Xmx就是系统预留内存。 * 在On Heap上给系统预留的内存， * 主要供新小对象创建，JAVA简单数据结构使用 * 以保证在On Heap上大结果集计算时情况，能快速响应其他 * 连接操作。 */ private String systemReserveMemorySize; private String XARecoveryLogBaseDir; private String XARecoveryLogBaseName; /** * 排序时，内存不够时，将已经排序的结果集 * 写入到临时目录 */ private String dataNodeSortedTempDir; /** * 是否启用zk切换 */ private boolean useZKSwitch=DEFAULT_USE_ZK_SWITCH; /** * huangyiming add * 无密码登陆标示, 0:否,1:是,默认为0 */ private int nonePasswordLogin = DEFAULT_NONEPASSWORDLOGIN ; private final static int DEFAULT_NONEPASSWORDLOGIN = 0; private int parallExecute; private boolean enableWriteQueueFlowControl;// 写队列流量控制 private int writeQueueStopThreshold;// 写队列停止写入阈值 private int writeQueueRecoverThreshold;// 写队列恢复写入阈值 public String getDefaultSqlParser() { return defaultSqlParser; } public void setDefaultSqlParser(String defaultSqlParser) { this.defaultSqlParser = defaultSqlParser; } public SystemConfig() { this.serverPort = DEFAULT_PORT; this.managerPort = DEFAULT_MANAGER_PORT; this.charset = DEFAULT_CHARSET; this.processors = DEFAULT_PROCESSORS; this.bufferPoolPageSize = DEFAULT_BUFFER_POOL_PAGE_SIZE; this.bufferPoolChunkSize = DEFAULT_BUFFER_CHUNK_SIZE; /** * 大结果集时 需增大 network buffer pool pages. */ this.bufferPoolPageNumber = (short) (DEFAULT_PROCESSORS*20); this.processorExecutor = (DEFAULT_PROCESSORS != 1) ? DEFAULT_PROCESSORS * 2 : 4; this.managerExecutor = 2; this.processorBufferLocalPercent = 100; this.timerExecutor = 2; this.idleTimeout = DEFAULT_IDLE_TIMEOUT; this.authTimeout = DEFAULT_AUTH_TIMEOUT; this.processorCheckPeriod = DEFAULT_PROCESSOR_CHECK_PERIOD; this.dataNodeIdleCheckPeriod = DEFAULT_DATANODE_IDLE_CHECK_PERIOD; this.dataNodeHeartbeatPeriod = DEFAULT_DATANODE_HEARTBEAT_PERIOD; this.clusterHeartbeatUser = DEFAULT_CLUSTER_HEARTBEAT_USER; this.clusterHeartbeatPass = DEFAULT_CLUSTER_HEARTBEAT_PASS; this.clusterHeartbeatPeriod = DEFAULT_CLUSTER_HEARTBEAT_PERIOD; this.clusterHeartbeatTimeout = DEFAULT_CLUSTER_HEARTBEAT_TIMEOUT; this.clusterHeartbeatRetry = DEFAULT_CLUSTER_HEARTBEAT_RETRY; this.txIsolation = Isolations.REPEATED_READ; this.parserCommentVersion = DEFAULT_PARSER_COMMENT_VERSION; this.sqlRecordCount = DEFAULT_SQL_RECORD_COUNT; this.glableTableCheckPeriod = DEFAULT_GLOBAL_TABLE_CHECK_PERIOD; this.useOffHeapForMerge = 0; this.memoryPageSize = MEMORY_PAGE_SIZE; this.spillsFileBufferSize = SPILLS_FILE_BUFFER_SIZE; this.useStreamOutput = 0; this.systemReserveMemorySize = RESERVED_SYSTEM_MEMORY_BYTES; this.dataNodeSortedTempDir = System.getProperty("user.dir"); this.XARecoveryLogBaseDir = SystemConfig.getHomePath()+"/tmlogs/"; this.XARecoveryLogBaseName ="tmlog"; this.maxPreparedStmtCount = DEFAULT_MAX_PREPAREDSTMT_COUNT; this.ignoreUnknownCommand = 0; this.parallExecute = 0; this.removeGraveAccent = 1; // 流量控制相关 this.enableWriteQueueFlowControl = false; this.writeQueueStopThreshold = 10 * 1024; this.writeQueueRecoverThreshold = 512; } public void setMaxPreparedStmtCount(int maxPreparedStmtCount){ this.maxPreparedStmtCount = maxPreparedStmtCount; } public int getMaxPreparedStmtCount(){ return this.maxPreparedStmtCount; } public String getDataNodeSortedTempDir() { return dataNodeSortedTempDir; } public int getUseOffHeapForMerge() { return useOffHeapForMerge; } public void setUseOffHeapForMerge(int useOffHeapForMerge) { this.useOffHeapForMerge = useOffHeapForMerge; } public String getMemoryPageSize() { return memoryPageSize; } public void setMemoryPageSize(String memoryPageSize) { this.memoryPageSize = memoryPageSize; } public String getSpillsFileBufferSize() { return spillsFileBufferSize; } public void setSpillsFileBufferSize(String spillsFileBufferSize) { this.spillsFileBufferSize = spillsFileBufferSize; } public int getUseStreamOutput() { return useStreamOutput; } public void setUseStreamOutput(int useStreamOutput) { this.useStreamOutput = useStreamOutput; } public String getSystemReserveMemorySize() { return systemReserveMemorySize; } public void setSystemReserveMemorySize(String systemReserveMemorySize) { this.systemReserveMemorySize = systemReserveMemorySize; } public boolean isUseZKSwitch() { return useZKSwitch; } public void setUseZKSwitch(boolean useZKSwitch) { this.useZKSwitch = useZKSwitch; } public String getXARecoveryLogBaseDir() { return XARecoveryLogBaseDir; } public void setXARecoveryLogBaseDir(String XARecoveryLogBaseDir) { this.XARecoveryLogBaseDir = XARecoveryLogBaseDir; } public String getXARecoveryLogBaseName() { return XARecoveryLogBaseName; } public void setXARecoveryLogBaseName(String XARecoveryLogBaseName) { this.XARecoveryLogBaseName = XARecoveryLogBaseName; } public int getUseGlobleTableCheck() { return useGlobleTableCheck; } public void setUseGlobleTableCheck(int useGlobleTableCheck) { this.useGlobleTableCheck = useGlobleTableCheck; } public long getGlableTableCheckPeriod() { return glableTableCheckPeriod; } public void setGlableTableCheckPeriod(long glableTableCheckPeriod) { this.glableTableCheckPeriod = glableTableCheckPeriod; } public String getSqlInterceptor() { return sqlInterceptor; } public void setSqlInterceptor(String sqlInterceptor) { this.sqlInterceptor = sqlInterceptor; } public int getSequenceHandlerType() { return sequenceHandlerType; } public void setSequenceHandlerType(int sequenceHandlerType) { this.sequenceHandlerType = sequenceHandlerType; } public int getPacketHeaderSize() { return packetHeaderSize; } public void setPacketHeaderSize(int packetHeaderSize) { this.packetHeaderSize = packetHeaderSize; } public int getMaxPacketSize() { return maxPacketSize; } public int getCatletClassCheckSeconds() { return catletClassCheckSeconds; } public void setCatletClassCheckSeconds(int catletClassCheckSeconds) { this.catletClassCheckSeconds = catletClassCheckSeconds; } public void setMaxPacketSize(int maxPacketSize) { this.maxPacketSize = maxPacketSize; } public int getFrontWriteQueueSize() { return frontWriteQueueSize; } public void setFrontWriteQueueSize(int frontWriteQueueSize) { this.frontWriteQueueSize = frontWriteQueueSize; } public String getBindIp() { return bindIp; } public void setBindIp(String bindIp) { this.bindIp = bindIp; } public int getDefaultMaxLimit() { return defaultMaxLimit; } public void setDefaultMaxLimit(int defaultMaxLimit) { this.defaultMaxLimit = defaultMaxLimit; } public static String getHomePath() { String home = System.getProperty(SystemConfig.SYS_HOME); if (home != null && home.endsWith(File.pathSeparator)) { home = home.substring(0, home.length() - 1); System.setProperty(SystemConfig.SYS_HOME, home); } // MYCAT_HOME为空，默认尝试设置为当前目录或上级目录。BEN if(home == null) { try { String path = new File("..").getCanonicalPath().replaceAll("\\\\", "/"); File conf = new File(path+"/conf"); if(conf.exists() && conf.isDirectory()) { home = path; } else { path = new File(".").getCanonicalPath().replaceAll("\\\\", "/"); conf = new File(path+"/conf"); if(conf.exists() && conf.isDirectory()) { home = path; } } if (home != null) { System.setProperty(SystemConfig.SYS_HOME, home); } } catch (IOException e) { // 如出错，则忽略。 } } return home; } // 是否使用SQL统计 public int getUseSqlStat() { return useSqlStat; } public void setUseSqlStat(int useSqlStat) { this.useSqlStat = useSqlStat; } public int getUseCompression() { return useCompression; } public void setUseCompression(int useCompression) { this.useCompression = useCompression; } public String getCharset() { return charset; } public void setCharset(String charset) { this.charset = charset; } public String getFakeMySQLVersion() { return fakeMySQLVersion; } public void setFakeMySQLVersion(String mysqlVersion) { this.fakeMySQLVersion = mysqlVersion; } public int getServerPort() { return serverPort; } public void setServerPort(int serverPort) { this.serverPort = serverPort; } public int getManagerPort() { return managerPort; } public void setManagerPort(int managerPort) { this.managerPort = managerPort; } public int getProcessors() { return processors; } public void setProcessors(int processors) { this.processors = processors; } public int getProcessorExecutor() { return processorExecutor; } public void setProcessorExecutor(int processorExecutor) { this.processorExecutor = processorExecutor; } public int getManagerExecutor() { return managerExecutor; } public void setManagerExecutor(int managerExecutor) { this.managerExecutor = managerExecutor; } public int getTimerExecutor() { return timerExecutor; } public void setTimerExecutor(int timerExecutor) { this.timerExecutor = timerExecutor; } public int getServerBacklog() { return serverBacklog; } public void setServerBacklog(int serverBacklog) { this.serverBacklog = serverBacklog; } public long getIdleTimeout() { return idleTimeout; } public void setIdleTimeout(long idleTimeout) { this.idleTimeout = idleTimeout; } public long getAuthTimeout() { return authTimeout; } public void setAuthTimeout(long authTimeout) { this.authTimeout = authTimeout; } public long getProcessorCheckPeriod() { return processorCheckPeriod; } public void setProcessorCheckPeriod(long processorCheckPeriod) { this.processorCheckPeriod = processorCheckPeriod; } public long getDataNodeIdleCheckPeriod() { return dataNodeIdleCheckPeriod; } public void setDataNodeIdleCheckPeriod(long dataNodeIdleCheckPeriod) { this.dataNodeIdleCheckPeriod = dataNodeIdleCheckPeriod; } public long getDataNodeHeartbeatPeriod() { return dataNodeHeartbeatPeriod; } public void setDataNodeHeartbeatPeriod(long dataNodeHeartbeatPeriod) { this.dataNodeHeartbeatPeriod = dataNodeHeartbeatPeriod; } public String getClusterHeartbeatUser() { return clusterHeartbeatUser; } public void setClusterHeartbeatUser(String clusterHeartbeatUser) { this.clusterHeartbeatUser = clusterHeartbeatUser; } public long getSqlExecuteTimeout() { return sqlExecuteTimeout; } public void setSqlExecuteTimeout(long sqlExecuteTimeout) { this.sqlExecuteTimeout = sqlExecuteTimeout; } public String getClusterHeartbeatPass() { return clusterHeartbeatPass; } public void setClusterHeartbeatPass(String clusterHeartbeatPass) { this.clusterHeartbeatPass = clusterHeartbeatPass; } public long getClusterHeartbeatPeriod() { return clusterHeartbeatPeriod; } public void setClusterHeartbeatPeriod(long clusterHeartbeatPeriod) { this.clusterHeartbeatPeriod = clusterHeartbeatPeriod; } public long getClusterHeartbeatTimeout() { return clusterHeartbeatTimeout; } public void setClusterHeartbeatTimeout(long clusterHeartbeatTimeout) { this.clusterHeartbeatTimeout = clusterHeartbeatTimeout; } public int getFrontsocketsorcvbuf() { return frontSocketSoRcvbuf; } public int getFrontsocketsosndbuf() { return frontSocketSoSndbuf; } public int getBacksocketsorcvbuf() { return backSocketSoRcvbuf; } public int getBacksocketsosndbuf() { return backSocketSoSndbuf; } public int getClusterHeartbeatRetry() { return clusterHeartbeatRetry; } public void setClusterHeartbeatRetry(int clusterHeartbeatRetry) { this.clusterHeartbeatRetry = clusterHeartbeatRetry; } public int getTxIsolation() { return txIsolation; } public void setTxIsolation(int txIsolation) { this.txIsolation = txIsolation; } public int getParserCommentVersion() { return parserCommentVersion; } public void setParserCommentVersion(int parserCommentVersion) { this.parserCommentVersion = parserCommentVersion; } public int getSqlRecordCount() { return sqlRecordCount; } public void setSqlRecordCount(int sqlRecordCount) { this.sqlRecordCount = sqlRecordCount; } public short getBufferPoolChunkSize() { return bufferPoolChunkSize; } public void setBufferPoolChunkSize(short bufferPoolChunkSize) { this.bufferPoolChunkSize = bufferPoolChunkSize; } public int getMaxResultSet() { return maxResultSet; } public void setMaxResultSet(int maxResultSet) { this.maxResultSet = maxResultSet; } public int getBigResultSizeSqlCount() { return bigResultSizeSqlCount; } public void setBigResultSizeSqlCount(int bigResultSizeSqlCount) { this.bigResultSizeSqlCount = bigResultSizeSqlCount; } public int getBufferUsagePercent() { return bufferUsagePercent; } public void setBufferUsagePercent(int bufferUsagePercent) { this.bufferUsagePercent = bufferUsagePercent; } public int getFlowControlRejectStrategy() { return flowControlRejectStrategy; } public void setFlowControlRejectStrategy(int flowControlRejectStrategy) { this.flowControlRejectStrategy = flowControlRejectStrategy; } public long getClearBigSqLResultSetMapMs() { return clearBigSqLResultSetMapMs; } public void setClearBigSqLResultSetMapMs(long clearBigSqLResultSetMapMs) { this.clearBigSqLResultSetMapMs = clearBigSqLResultSetMapMs; } public int getBufferPoolPageSize() { return bufferPoolPageSize; } public void setBufferPoolPageSize(int bufferPoolPageSize) { this.bufferPoolPageSize = bufferPoolPageSize; } public short getBufferPoolPageNumber() { return bufferPoolPageNumber; } public void setBufferPoolPageNumber(short bufferPoolPageNumber) { this.bufferPoolPageNumber = bufferPoolPageNumber; } public int getFrontSocketSoRcvbuf() { return frontSocketSoRcvbuf; } public void setFrontSocketSoRcvbuf(int frontSocketSoRcvbuf) { this.frontSocketSoRcvbuf = frontSocketSoRcvbuf; } public int getFrontSocketSoSndbuf() { return frontSocketSoSndbuf; } public void setFrontSocketSoSndbuf(int frontSocketSoSndbuf) { this.frontSocketSoSndbuf = frontSocketSoSndbuf; } public int getBackSocketSoRcvbuf() { return backSocketSoRcvbuf; } public void setBackSocketSoRcvbuf(int backSocketSoRcvbuf) { this.backSocketSoRcvbuf = backSocketSoRcvbuf; } public int getBackSocketSoSndbuf() { return backSocketSoSndbuf; } public void setBackSocketSoSndbuf(int backSocketSoSndbuf) { this.backSocketSoSndbuf = backSocketSoSndbuf; } public int getFrontSocketNoDelay() { return frontSocketNoDelay; } public void setFrontSocketNoDelay(int frontSocketNoDelay) { this.frontSocketNoDelay = frontSocketNoDelay; } public int getBackSocketNoDelay() { return backSocketNoDelay; } public void setBackSocketNoDelay(int backSocketNoDelay) { this.backSocketNoDelay = backSocketNoDelay; } public int getMaxStringLiteralLength() { return maxStringLiteralLength; } public void setMaxStringLiteralLength(int maxStringLiteralLength) { this.maxStringLiteralLength = maxStringLiteralLength; } public int getMutiNodeLimitType() { return mutiNodeLimitType; } public void setMutiNodeLimitType(int mutiNodeLimitType) { this.mutiNodeLimitType = mutiNodeLimitType; } public int getMutiNodePatchSize() { return mutiNodePatchSize; } public void setMutiNodePatchSize(int mutiNodePatchSize) { this.mutiNodePatchSize = mutiNodePatchSize; } public int getProcessorBufferLocalPercent() { return processorBufferLocalPercent; } public void setProcessorBufferLocalPercent(int processorBufferLocalPercent) { this.processorBufferLocalPercent = processorBufferLocalPercent; } public String getSqlInterceptorType() { return sqlInterceptorType; } public void setSqlInterceptorType(String sqlInterceptorType) { this.sqlInterceptorType = sqlInterceptorType; } public String getSqlInterceptorFile() { return sqlInterceptorFile; } public void setSqlInterceptorFile(String sqlInterceptorFile) { this.sqlInterceptorFile = sqlInterceptorFile; } public int getUsingAIO() { return usingAIO; } public void setUsingAIO(int usingAIO) { this.usingAIO = usingAIO; } public int getMycatNodeId() { return mycatNodeId; } public void setMycatNodeId(int mycatNodeId) { this.mycatNodeId = mycatNodeId; } @Override public String toString() { return "SystemConfig [processorBufferLocalPercent=" + processorBufferLocalPercent + ", frontSocketSoRcvbuf=" + frontSocketSoRcvbuf + ", frontSocketSoSndbuf=" + frontSocketSoSndbuf + ", backSocketSoRcvbuf=" + backSocketSoRcvbuf + ", backSocketSoSndbuf=" + backSocketSoSndbuf + ", frontSocketNoDelay=" + frontSocketNoDelay + ", backSocketNoDelay=" + backSocketNoDelay + ", maxStringLiteralLength=" + maxStringLiteralLength + ", frontWriteQueueSize=" + frontWriteQueueSize + ", bindIp=" + bindIp + ", serverPort=" + serverPort + ", managerPort=" + managerPort + ", charset=" + charset + ", processors=" + processors + ", processorExecutor=" + processorExecutor + ", timerExecutor=" + timerExecutor + ", managerExecutor=" + managerExecutor + ", serverBacklog=" + serverBacklog + ", idleTimeout=" + idleTimeout + ", authTimeout=" + authTimeout + ", catletClassCheckSeconds=" + catletClassCheckSeconds + ", sqlExecuteTimeout=" + sqlExecuteTimeout + ", processorCheckPeriod=" + processorCheckPeriod + ", dataNodeIdleCheckPeriod=" + dataNodeIdleCheckPeriod + ", dataNodeHeartbeatPeriod=" + dataNodeHeartbeatPeriod + ", clusterHeartbeatUser=" + clusterHeartbeatUser + ", clusterHeartbeatPass=" + clusterHeartbeatPass + ", clusterHeartbeatPeriod=" + clusterHeartbeatPeriod + ", clusterHeartbeatTimeout=" + clusterHeartbeatTimeout + ", clusterHeartbeatRetry=" + clusterHeartbeatRetry + ", txIsolation=" + txIsolation + ", parserCommentVersion=" + parserCommentVersion + ", sqlRecordCount=" + sqlRecordCount + ", bufferPoolPageSize=" + bufferPoolPageSize + ", bufferPoolChunkSize=" + bufferPoolChunkSize + ", bufferPoolPageNumber=" + bufferPoolPageNumber + ", maxResultSet=" +maxResultSet + ", bigResultSizeSqlCount="+bigResultSizeSqlCount + ", bufferUsagePercent="+bufferUsagePercent + ", flowControlRejectStrategy="+flowControlRejectStrategy + ", clearBigSqLResultSetMapMs="+clearBigSqLResultSetMapMs + ", defaultMaxLimit=" + defaultMaxLimit + ", sequenceHandlerType=" + sequenceHandlerType + ", sqlInterceptor=" + sqlInterceptor + ", sqlInterceptorType=" + sqlInterceptorType + ", sqlInterceptorFile=" + sqlInterceptorFile + ", mutiNodeLimitType=" + mutiNodeLimitType + ", mutiNodePatchSize=" + mutiNodePatchSize + ", defaultSqlParser=" + defaultSqlParser + ", usingAIO=" + usingAIO + ", packetHeaderSize=" + packetHeaderSize + ", maxPacketSize=" + maxPacketSize + ", mycatNodeId=" + mycatNodeId + ",ignoreUnknownCommand="+ignoreUnknownCommand + ",parallExecute="+parallExecute + ",removeGraveAccent="+ removeGraveAccent + "]"; } public int getCheckTableConsistency() { return checkTableConsistency; } public void setCheckTableConsistency(int checkTableConsistency) { this.checkTableConsistency = checkTableConsistency; } public long getCheckTableConsistencyPeriod() { return checkTableConsistencyPeriod; } public void setCheckTableConsistencyPeriod(long checkTableConsistencyPeriod) { this.checkTableConsistencyPeriod = checkTableConsistencyPeriod; } public int getProcessorBufferPoolType() { return processorBufferPoolType; } public void setProcessorBufferPoolType(int processorBufferPoolType) { this.processorBufferPoolType = processorBufferPoolType; } public int getHandleDistributedTransactions() { return handleDistributedTransactions; } public void setHandleDistributedTransactions(int handleDistributedTransactions) { this.handleDistributedTransactions = handleDistributedTransactions; } public int getUseHandshakeV10() { return useHandshakeV10; } public void setUseHandshakeV10(int useHandshakeV10) { this.useHandshakeV10 = useHandshakeV10; } public int getNonePasswordLogin() { return nonePasswordLogin; } public void setNonePasswordLogin(int nonePasswordLogin) { this.nonePasswordLogin = nonePasswordLogin; } public boolean isSubqueryRelationshipCheck() { return subqueryRelationshipCheck; } public void setSubqueryRelationshipCheck(boolean subqueryRelationshipCheck) { this.subqueryRelationshipCheck = subqueryRelationshipCheck; } public boolean isStrictTxIsolation() { return strictTxIsolation; } public void setStrictTxIsolation(boolean strictTxIsolation) { this.strictTxIsolation = strictTxIsolation; } public int getSequnceMySqlRetryCount() { return sequnceMySqlRetryCount; } public void setSequnceMySqlRetryCount(int sequnceMySqlRetryCount) { this.sequnceMySqlRetryCount = sequnceMySqlRetryCount; } public long getSequnceMySqlWaitTime() { return sequnceMySqlWaitTime; } public void setSequnceMySqlWaitTime(long sequnceMySqlWaitTime) { this.sequnceMySqlWaitTime = sequnceMySqlWaitTime; } public String getSequnceHandlerPattern() { return sequnceHandlerPattern==null?SEQUENCEHANDLER_PATTERN:sequnceHandlerPattern; } public void setSequnceHandlerPattern(String sequnceHandlerPattern) { this.sequnceHandlerPattern = sequnceHandlerPattern; } public String getSequenceHanlderClass() { return sequenceHanlderClass; } public void setSequenceHanlderClass(String value) { sequenceHanlderClass = value; } public int getIgnoreUnknownCommand() { return ignoreUnknownCommand; } public void setIgnoreUnknownCommand(int ignoreUnknownCommand) { this.ignoreUnknownCommand = ignoreUnknownCommand; } public int getParallExecute() { return parallExecute; } public void setParallExecute(int parallExecute) { this.parallExecute = parallExecute; } public int getRemoveGraveAccent() { return removeGraveAccent; } public void setRemoveGraveAccent(int removeGraveAccent) { this.removeGraveAccent = removeGraveAccent; } public boolean isEnableWriteQueueFlowControl() { return enableWriteQueueFlowControl; } public void setEnableWriteQueueFlowControl(boolean enableWriteQueueFlowControl) { this.enableWriteQueueFlowControl = enableWriteQueueFlowControl; } public int getWriteQueueStopThreshold() { return writeQueueStopThreshold; } public void setWriteQueueStopThreshold(int writeQueueStopThreshold) { this.writeQueueStopThreshold = writeQueueStopThreshold; } public int getWriteQueueRecoverThreshold() { return writeQueueRecoverThreshold; } public void setWriteQueueRecoverThreshold(int writeQueueRecoverThreshold) { this.writeQueueRecoverThreshold = writeQueueRecoverThreshold; } } ================================================ FILE: src/main/java/io/mycat/config/model/TableConfig.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.model; import com.alibaba.druid.sql.ast.statement.SQLTableElement; import io.mycat.config.model.rule.RuleConfig; import io.mycat.util.SplitUtil; import java.util.*; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.locks.ReentrantReadWriteLock; /** * @author mycat */ public class TableConfig { public static final int TYPE_GLOBAL_TABLE = 1; public static final int TYPE_GLOBAL_DEFAULT = 0; private final String name; private final String primaryKey; private final boolean autoIncrement; private final boolean fetchStoreNodeByJdbc; private final boolean needAddLimit; private final Set dbTypes; private final int tableType; private final ArrayList dataNodes; private final ArrayList distTables; private final RuleConfig rule; private final String partitionColumn; private final boolean ruleRequired; private final TableConfig parentTC; private final boolean childTable; private final String joinKey; private final String parentKey; private final String locateRTableKeySql; // only has one level of parent private final boolean secondLevel; private final boolean partionKeyIsPrimaryKey; private volatile List tableElementList; private volatile String tableStructureSQL; private volatile Map> dataNodeTableStructureSQLMap; private ReentrantReadWriteLock reentrantReadWriteLock = new ReentrantReadWriteLock(false); public TableConfig(String tableName, String primaryKey, boolean autoIncrement, boolean needAddLimit, int tableType, String dataNode, Set dbType, RuleConfig rule, boolean ruleRequired, TableConfig parentTC, boolean isChildTable, String joinKey, String parentKey, String subTables, boolean fetchStoreNodeByJdbc) { if (tableName == null) { throw new IllegalArgumentException("table name is null"); } else if (dataNode == null) { throw new IllegalArgumentException("dataNode name is null"); } this.primaryKey = primaryKey; this.autoIncrement = autoIncrement; this.needAddLimit = needAddLimit; this.fetchStoreNodeByJdbc = fetchStoreNodeByJdbc; this.tableType = tableType; this.dbTypes = dbType; if (ruleRequired && rule == null) { throw new IllegalArgumentException("ruleRequired but rule is null"); } this.name = tableName.toUpperCase(); String theDataNodes[] = SplitUtil.split(dataNode, ',', '$', '-'); if (theDataNodes == null || theDataNodes.length <= 0) { throw new IllegalArgumentException("invalid table dataNodes: " + dataNode); } dataNodes = new ArrayList(theDataNodes.length); for (String dn : theDataNodes) { dataNodes.add(dn); } if (subTables != null && !subTables.equals("")) { String sTables[] = SplitUtil.split(subTables, ',', '$', '-'); if (sTables == null || sTables.length <= 0) { throw new IllegalArgumentException("invalid table subTables"); } this.distTables = new ArrayList(sTables.length); for (String table : sTables) { distTables.add(table); } } else { this.distTables = new ArrayList(); } this.rule = rule; this.partitionColumn = (rule == null) ? null : rule.getColumn(); partionKeyIsPrimaryKey = (partitionColumn == null) ? primaryKey == null : partitionColumn.equals(primaryKey); this.ruleRequired = ruleRequired; this.childTable = isChildTable; this.parentTC = parentTC; this.joinKey = joinKey; this.parentKey = parentKey; if (parentTC != null) { locateRTableKeySql = genLocateRootParentSQL(); secondLevel = (parentTC.parentTC == null); } else { locateRTableKeySql = null; secondLevel = false; } } public String getPrimaryKey() { return primaryKey; } public Set getDbTypes() { return dbTypes; } public boolean isAutoIncrement() { return autoIncrement; } public boolean isNeedAddLimit() { return needAddLimit; } public boolean isSecondLevel() { return secondLevel; } public String getLocateRTableKeySql() { return locateRTableKeySql; } public boolean isGlobalTable() { return this.tableType == TableConfig.TYPE_GLOBAL_TABLE; } public String genLocateRootParentSQL() { TableConfig tb = this; StringBuilder tableSb = new StringBuilder(); StringBuilder condition = new StringBuilder(); TableConfig prevTC = null; String ancestorTableName = null; int level = 0; String latestCond = null; while (tb.parentTC != null) { ancestorTableName = tb.parentTC.name; String currentTableName = tb.name; if (!ancestorTableName.contains("`")){ ancestorTableName = "`"+ancestorTableName+"`"; } if (!currentTableName.contains("`")){ currentTableName = "`"+currentTableName+"`"; } tableSb.append(ancestorTableName).append(','); String relation = null; if (level == 0) { latestCond = " " + ancestorTableName + '.' + tb.parentKey + "="; } else { relation = ancestorTableName + '.' + tb.parentKey + '=' + currentTableName + '.' + tb.joinKey; condition.append(relation).append(" AND "); } level++; prevTC = tb; tb = tb.parentTC; } String sql = "SELECT " + ancestorTableName + '.' + prevTC.parentKey + " FROM " + tableSb.substring(0, tableSb.length() - 1) + " WHERE " + ((level < 2) ? latestCond : condition.toString() + latestCond); // System.out.println(this.name+" sql " + sql); return sql; } public String getPartitionColumn() { return partitionColumn; } public int getTableType() { return tableType; } /** * get root parent * * @return */ public TableConfig getRootParent() { if (parentTC == null) { return null; } TableConfig preParent = parentTC; TableConfig parent = preParent.getParentTC(); while (parent != null) { preParent = parent; parent = parent.getParentTC(); } return preParent; } public TableConfig getParentTC() { return parentTC; } public boolean isChildTable() { return childTable; } public String getJoinKey() { return joinKey; } public String getParentKey() { return parentKey; } /** * @return upper-case */ public String getName() { return name; } public ArrayList getDataNodes() { return dataNodes; } public String getRandomDataNode() { int index = Math.abs(ThreadLocalRandom.current().nextInt(Integer.MAX_VALUE)) % dataNodes.size(); return dataNodes.get(index); } public boolean isRuleRequired() { return ruleRequired; } public RuleConfig getRule() { return rule; } public boolean primaryKeyIsPartionKey() { return partionKeyIsPrimaryKey; } public ArrayList getDistTables() { return this.distTables; } public boolean isDistTable() { if (this.distTables != null && !this.distTables.isEmpty()) { return true; } return false; } public List getTableElementList() { return tableElementList; } public void setTableElementList(List tableElementList) { this.tableElementList = tableElementList; } public ReentrantReadWriteLock getReentrantReadWriteLock() { return reentrantReadWriteLock; } public void setReentrantReadWriteLock(ReentrantReadWriteLock reentrantReadWriteLock) { this.reentrantReadWriteLock = reentrantReadWriteLock; } public String getTableStructureSQL() { return tableStructureSQL; } public void setTableStructureSQL(String tableStructureSQL) { this.tableStructureSQL = tableStructureSQL; } public Map> getDataNodeTableStructureSQLMap() { return dataNodeTableStructureSQLMap; } public void setDataNodeTableStructureSQLMap(Map> dataNodeTableStructureSQLMap) { this.dataNodeTableStructureSQLMap = dataNodeTableStructureSQLMap; } public boolean getFetchStoreNodeByJdbc() { return this.fetchStoreNodeByJdbc; } } ================================================ FILE: src/main/java/io/mycat/config/model/TableConfigMap.java ================================================ package io.mycat.config.model; import java.util.HashMap; /** * 支持表名中包含引号[`] * * @author BEN GONG */ public class TableConfigMap extends HashMap { private static final long serialVersionUID = -6605226933829917213L; @Override public TableConfig get(Object key) { String tableName = key.toString(); // 忽略表名中的引号。 if(tableName.contains("`")) { tableName = tableName.replaceAll("`", ""); } return super.get(tableName); } @Override public boolean containsKey(Object key) { String tableName = key.toString(); // 忽略表名中的引号。 if(tableName.contains("`")) { tableName = tableName.replaceAll("`", ""); } return super.containsKey(tableName); } } ================================================ FILE: src/main/java/io/mycat/config/model/TableRuleConfig.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.model; import java.beans.Expression; /** * @author mycat */ public final class TableRuleConfig { private final String name; private final RuleConfig[] rules; public TableRuleConfig(String name, RuleConfig[] rules) { this.name = name; this.rules = rules; if (rules != null) { for (RuleConfig r : rules) { r.tableRuleName = name; } } } public String getName() { return name; } public RuleConfig[] getRules() { return rules; } public static final class RuleConfig { private String tableRuleName; /** upper-case */ private final String[] columns; private final Expression algorithm; public RuleConfig(String[] columns, Expression algorithm) { this.columns = columns == null ? new String[0] : columns; this.algorithm = algorithm; } public String[] getColumns() { return columns; } public Expression getAlgorithm() { return algorithm; } @Override public String toString() { StringBuilder s = new StringBuilder(); s.append("{tableRule:").append(tableRuleName).append(", columns:["); for (int i = 0; i < columns.length; ++i) { if (i > 0) { s.append(", "); } s.append(columns[i]); } s.append("]}"); return s.toString(); } } } ================================================ FILE: src/main/java/io/mycat/config/model/UserConfig.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.model; import java.util.Set; /** * @author mycat */ public class UserConfig { private String name; private String password; //明文 private String encryptPassword; //密文 private int benchmark = 0; // 负载限制, 默认0表示不限制 private UserPrivilegesConfig privilegesConfig; //SQL表级的增删改查权限控制 private String defaultSchema; /** * 是否无密码登陆的默认账户 */ private boolean defaultAccount = false; private boolean readOnly = false; public boolean isReadOnly() { return readOnly; } public void setReadOnly(boolean readOnly) { this.readOnly = readOnly; } private Set schemas; public String getName() { return name; } public void setName(String name) { this.name = name; } public String getPassword() { return password; } public void setPassword(String password) { this.password = password; } public int getBenchmark() { return benchmark; } public void setBenchmark(int benchmark) { this.benchmark = benchmark; } public Set getSchemas() { return schemas; } public String getEncryptPassword() { return this.encryptPassword; } public void setEncryptPassword(String encryptPassword) { this.encryptPassword = encryptPassword; } public void setSchemas(Set schemas) { this.schemas = schemas; } public UserPrivilegesConfig getPrivilegesConfig() { return privilegesConfig; } public void setPrivilegesConfig(UserPrivilegesConfig privilegesConfig) { this.privilegesConfig = privilegesConfig; } public boolean isDefaultAccount() { return defaultAccount; } public void setDefaultAccount(boolean defaultAccount) { this.defaultAccount = defaultAccount; } public String getDefaultSchema() { return defaultSchema; } public void setDefaultSchema(String defaultSchema) { this.defaultSchema = defaultSchema; } @Override public String toString() { return "UserConfig [name=" + name + ", password=" + password + ", encryptPassword=" + encryptPassword + ", benchmark=" + benchmark + ", privilegesConfig=" + privilegesConfig + ", defaultAccount=" + defaultAccount + ", readOnly=" + readOnly + ", schemas=" + schemas +", defaultSchema=" + defaultSchema + "]"; } } ================================================ FILE: src/main/java/io/mycat/config/model/UserPrivilegesConfig.java ================================================ package io.mycat.config.model; import java.util.HashMap; import java.util.Map; /** * 用户 SQL 权限配置 * * @author zhuam * */ public class UserPrivilegesConfig { private boolean check = false; /** 库级权限 */ private Map schemaPrivileges = new HashMap(); /** dataNode权限 */ private Map dataNodePrivileges = new HashMap(); public boolean isCheck() { return check; } public void setCheck(boolean check) { this.check = check; } public void addSchemaPrivilege(String schemaName, SchemaPrivilege privilege) { this.schemaPrivileges.put(schemaName, privilege); } public SchemaPrivilege getSchemaPrivilege(String schemaName) { return schemaPrivileges.get( schemaName ); } public void addDataNodePrivileges(String dataNodeName, DataNodePrivilege privilege) { this.dataNodePrivileges.put(dataNodeName, privilege); } public DataNodePrivilege getDataNodePrivilege(String dataNodeName) { return dataNodePrivileges.get(dataNodeName); } /** * 库级权限 */ public static class SchemaPrivilege { private String name; private int[] dml = new int[]{0, 0, 0, 0}; private Map tablePrivileges = new HashMap(); public String getName() { return name; } public void setName(String name) { this.name = name; } public int[] getDml() { return dml; } public void setDml(int[] dml) { this.dml = dml; } public void addTablePrivilege(String tableName, TablePrivilege privilege) { this.tablePrivileges.put(tableName, privilege); } public TablePrivilege getTablePrivilege(String tableName) { TablePrivilege tablePrivilege = tablePrivileges.get( tableName ); if ( tablePrivilege == null ) { tablePrivilege = new TablePrivilege(); tablePrivilege.setName(tableName); tablePrivilege.setDml(dml); } return tablePrivilege; } } /** * 表级权限 */ public static class TablePrivilege { private String name; private int[] dml = new int[] { 0, 0, 0, 0 }; public String getName() { return name; } public void setName(String name) { this.name = name; } public int[] getDml() { return dml; } public void setDml(int[] dml) { this.dml = dml; } } /** * dataNode权限 */ public static class DataNodePrivilege { private String name; private int[] dml = new int[] { 0, 0, 0, 0 }; public String getName() { return name; } public void setName(String name) { this.name = name; } public int[] getDml() { return dml; } public void setDml(int[] dml) { this.dml = dml; } } } ================================================ FILE: src/main/java/io/mycat/config/model/rule/RuleAlgorithm.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.model.rule; /** * @author mycat */ public interface RuleAlgorithm { /** * init * * @param */ void init(); /** * * return sharding nodes's id * columnValue is column's value * @return never null */ Integer calculate(String columnValue) ; Integer[] calculateRange(String beginValue,String endValue) ; } ================================================ FILE: src/main/java/io/mycat/config/model/rule/RuleConfig.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.model.rule; import io.mycat.route.function.AbstractPartitionAlgorithm; import java.io.Serializable; /** * 分片规则，column是用于分片的数据库物理字段 * @author mycat */ public class RuleConfig implements Serializable { private final String column; private final String functionName; private AbstractPartitionAlgorithm ruleAlgorithm; public RuleConfig(String column, String functionName) { if (functionName == null) { throw new IllegalArgumentException("functionName is null"); } this.functionName = functionName; if (column == null || column.length() <= 0) { throw new IllegalArgumentException("no rule column is found"); } this.column = column; } public AbstractPartitionAlgorithm getRuleAlgorithm() { return ruleAlgorithm; } public void setRuleAlgorithm(AbstractPartitionAlgorithm ruleAlgorithm) { this.ruleAlgorithm = ruleAlgorithm; } /** * @return unmodifiable, upper-case */ public String getColumn() { return column; } public String getFunctionName() { return functionName; } @Override public int hashCode() { final int prime = 31; int result = 1; result = prime * result + ((column == null) ? 0 : column.hashCode()); result = prime * result + ((functionName == null) ? 0 : functionName.hashCode()); result = prime * result + ((ruleAlgorithm == null) ? 0 : ruleAlgorithm.hashCode()); return result; } //huangyiming add 判断分片规则是否相同,暂时根据这个去判断 @Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (getClass() != obj.getClass()) return false; RuleConfig other = (RuleConfig) obj; if (column == null) { if (other.column != null) return false; } else if (!column.equals(other.column)) return false; if (functionName == null) { if (other.functionName != null) return false; } else if (!functionName.equals(other.functionName)) return false; if (ruleAlgorithm == null) { if (other.ruleAlgorithm != null) return false; } else if (!ruleAlgorithm.equals(other.ruleAlgorithm)) return false; return true; } } ================================================ FILE: src/main/java/io/mycat/config/model/rule/TableRuleConfig.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.model.rule; import java.io.Serializable; /** * @author mycat */ public class TableRuleConfig implements Serializable { private String name; private final RuleConfig rule; public TableRuleConfig(String name, RuleConfig rule) { if (name == null) { throw new IllegalArgumentException("name is null"); } this.name = name; if (rule == null) { throw new IllegalArgumentException("no rule is found"); } this.rule =rule; } public String getName() { return name; } public void setName(String name) { this.name = name; } /** * @return unmodifiable */ public RuleConfig getRule() { return rule; } } ================================================ FILE: src/main/java/io/mycat/config/table/structure/MySQLTableStructureDetector.java ================================================ package io.mycat.config.table.structure; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.TableConfig; import io.mycat.sqlengine.OneRawSQLQueryResultHandler; import io.mycat.sqlengine.SQLJob; import io.mycat.sqlengine.SQLQueryResult; import io.mycat.sqlengine.SQLQueryResultListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; /** * 表结构结果处理 * * @author Hash Zhang * @version 1.0 * @time 00:09:03 2016/5/11 */ public class MySQLTableStructureDetector implements Runnable { private static final Logger LOGGER = LoggerFactory.getLogger(MySQLTableStructureDetector.class); private static final String[] MYSQL_SHOW_CREATE_TABLE_COLMS = new String[]{ "Table", "Create Table"}; private static final String sqlPrefix = "show create table "; @Override public void run() { for (SchemaConfig schema : MycatServer.getInstance().getConfig().getSchemas().values()) { for (TableConfig table : schema.getTables().values()) { for (String dataNode : table.getDataNodes()) { try { table.getReentrantReadWriteLock().writeLock().lock(); ConcurrentHashMap> map = new ConcurrentHashMap<>(); table.setDataNodeTableStructureSQLMap(map); } finally { table.getReentrantReadWriteLock().writeLock().unlock(); } OneRawSQLQueryResultHandler resultHandler = new OneRawSQLQueryResultHandler(MYSQL_SHOW_CREATE_TABLE_COLMS, new MySQLTableStructureListener(dataNode, table)); resultHandler.setMark("Table Structure"); PhysicalDBNode dn = MycatServer.getInstance().getConfig().getDataNodes().get(dataNode); SQLJob sqlJob = new SQLJob(sqlPrefix + table.getName(), dn.getDatabase(), resultHandler, dn.getDbPool().getSource()); sqlJob.run(); } } } } private static class MySQLTableStructureListener implements SQLQueryResultListener>> { private String dataNode; private TableConfig table; public MySQLTableStructureListener(String dataNode, TableConfig table) { this.dataNode = dataNode; this.table = table; } /** * @param result * @// TODO: 2016/5/11 检查表元素，来确定是哪个元素不一致，未来还有其他用 */ @Override public void onResult(SQLQueryResult> result) { try { table.getReentrantReadWriteLock().writeLock().lock(); if (!result.isSuccess()) { LOGGER.warn("Can't get table " + table.getName() + "'s config from DataNode:" + dataNode + "! Maybe the table is not initialized!"); return; } String currentSql = result.getResult().get(MYSQL_SHOW_CREATE_TABLE_COLMS[1]); Map> dataNodeTableStructureSQLMap = table.getDataNodeTableStructureSQLMap(); if (dataNodeTableStructureSQLMap.containsKey(currentSql)) { List dataNodeList = dataNodeTableStructureSQLMap.get(currentSql); dataNodeList.add(dataNode); } else { List dataNodeList = new LinkedList<>(); dataNodeList.add(dataNode); dataNodeTableStructureSQLMap.put(currentSql,dataNodeList); } if (dataNodeTableStructureSQLMap.size() > 1) { LOGGER.warn("Table [" + table.getName() + "] structure are not consistent!"); LOGGER.warn("Currently detected: "); for(String sql : dataNodeTableStructureSQLMap.keySet()){ StringBuilder stringBuilder = new StringBuilder(); for(String dn : dataNodeTableStructureSQLMap.get(sql)){ stringBuilder.append("DataNode:[").append(dn).append("]"); } stringBuilder.append(":").append(sql); LOGGER.warn(stringBuilder.toString()); } } } finally { table.getReentrantReadWriteLock().writeLock().unlock(); } } } // public static void main(String[] args) { // System.out.println(UUID.randomUUID()); // } } ================================================ FILE: src/main/java/io/mycat/config/table/structure/TableStructureProcessor.java ================================================ package io.mycat.config.table.structure; /** * 将表结构持久化 * * @author Hash Zhang * @version 1.0 * @time 00:09:03 2016/5/11 */ public abstract class TableStructureProcessor { public abstract void saveTableStructure(); public abstract void loadTableStructure(); } ================================================ FILE: src/main/java/io/mycat/config/util/BeanConfig.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.util; import java.lang.reflect.InvocationTargetException; import java.util.HashMap; import java.util.Map; import io.mycat.util.ObjectUtil; /** * @author mycat */ public class BeanConfig implements Cloneable { private static final ReflectionProvider refProvider = new ReflectionProvider(); private String name; private String className; private Map params = new HashMap(); public String getName() { return name; } public void setName(String name) { this.name = name; } public String getClassName() { return className; } public void setClassName(String beanObject) { this.className = beanObject; } public Map getParams() { return params; } public void setParams(Map params) { this.params = params; } public Object create(boolean initEarly) throws IllegalAccessException, InvocationTargetException { Object obj = null; try { obj = refProvider.newInstance(Class.forName(className)); } catch (ClassNotFoundException e) { throw new ConfigException(e); } ParameterMapping.mapping(obj, params); if (initEarly && (obj instanceof Initializable)) { ((Initializable) obj).init(); } return obj; } @Override public Object clone() { try { super.clone(); } catch (CloneNotSupportedException e) { throw new ConfigException(e); } BeanConfig bc = null; try { bc = getClass().newInstance(); } catch (InstantiationException e) { throw new ConfigException(e); } catch (IllegalAccessException e) { throw new ConfigException(e); } // if (bc == null) { // return null; // } bc.className = className; bc.name = name; // Map params = new HashMap(); // params.putAll(params); return bc; } @Override public int hashCode() { int hashcode = 37; hashcode += (name == null ? 0 : name.hashCode()); hashcode += (className == null ? 0 : className.hashCode()); hashcode += (params == null ? 0 : params.hashCode()); return hashcode; } @Override public boolean equals(Object object) { if (object instanceof BeanConfig) { BeanConfig entity = (BeanConfig) object; boolean isEquals = equals(name, entity.name); isEquals = isEquals && equals(className, entity.getClassName()); isEquals = isEquals && (ObjectUtil.equals(params, entity.params)); return isEquals; } return false; } private static boolean equals(String str1, String str2) { if (str1 == null) { return str2 == null; } return str1.equals(str2); } } ================================================ FILE: src/main/java/io/mycat/config/util/ConfigException.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.util; /** * @author mycat */ public class ConfigException extends RuntimeException { private static final long serialVersionUID = -180146385688342818L; public ConfigException() { super(); } public ConfigException(String message, Throwable cause) { super(message, cause); } public ConfigException(String message) { super(message); } public ConfigException(Throwable cause) { super(cause); } } ================================================ FILE: src/main/java/io/mycat/config/util/ConfigUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.util; import java.io.IOException; import java.io.InputStream; import java.util.HashMap; import java.util.Map; import java.util.Properties; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import org.w3c.dom.Attr; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NamedNodeMap; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.EntityResolver; import org.xml.sax.ErrorHandler; import org.xml.sax.InputSource; import org.xml.sax.SAXException; import org.xml.sax.SAXParseException; import io.mycat.util.StringUtil; /** * @author mycat */ public class ConfigUtil { public static String filter(String text) { return filter(text, System.getProperties()); } public static String filter(String text, Properties properties) { StringBuilder s = new StringBuilder(); int cur = 0; int textLen = text.length(); int propStart = -1; int propStop = -1; String propName = null; String propValue = null; for (; cur < textLen; cur = propStop + 1) { propStart = text.indexOf("${", cur); if (propStart < 0) { break; } s.append(text.substring(cur, propStart)); propStop = text.indexOf("}", propStart); if (propStop < 0) { throw new ConfigException("Unterminated property: " + text.substring(propStart)); } propName = text.substring(propStart + 2, propStop); propValue = properties.getProperty(propName); if (propValue == null) { s.append("${").append(propName).append('}'); } else { s.append(propValue); } } return s.append(text.substring(cur)).toString(); } public static Document getDocument(final InputStream dtd, InputStream xml) throws ParserConfigurationException, SAXException, IOException { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); factory.setValidating(true); factory.setNamespaceAware(false); DocumentBuilder builder = factory.newDocumentBuilder(); builder.setEntityResolver(new EntityResolver() { @Override public InputSource resolveEntity(String publicId, String systemId) { return new InputSource(dtd); } }); builder.setErrorHandler(new ErrorHandler() { @Override public void warning(SAXParseException e) { } @Override public void error(SAXParseException e) throws SAXException { throw e; } @Override public void fatalError(SAXParseException e) throws SAXException { throw e; } }); return builder.parse(xml); } public static Map loadAttributes(Element e) { Map map = new HashMap(); NamedNodeMap nm = e.getAttributes(); for (int j = 0; j < nm.getLength(); j++) { Node n = nm.item(j); if (n instanceof Attr) { Attr attr = (Attr) n; map.put(attr.getName(), attr.getNodeValue()); } } return map; } public static Element loadElement(Element parent, String tagName) { NodeList nodeList = parent.getElementsByTagName(tagName); if (nodeList.getLength() > 1) { throw new ConfigException(tagName + " elements length over one!"); } if (nodeList.getLength() == 1) { return (Element) nodeList.item(0); } else { return null; } } /** * 获取节点下所有property * @param parent * @return key-value property键值对 */ public static Map loadElements(Element parent) { Map map = new HashMap(); NodeList children = parent.getChildNodes(); for (int i = 0; i < children.getLength(); i++) { Node node = children.item(i); if (node instanceof Element) { Element e = (Element) node; String name = e.getNodeName(); //获取property if ("property".equals(name)) { String key = e.getAttribute("name"); NodeList nl = e.getElementsByTagName("bean"); if (nl.getLength() == 0) { String value = e.getTextContent(); map.put(key, StringUtil.isEmpty(value) ? null : value.trim()); } else { map.put(key, loadBean((Element) nl.item(0))); } } } } return map; } public static BeanConfig loadBean(Element parent, String tagName) { NodeList nodeList = parent.getElementsByTagName(tagName); if (nodeList.getLength() > 1) { throw new ConfigException(tagName + " elements length over one!"); } return loadBean((Element) nodeList.item(0)); } public static BeanConfig loadBean(Element e) { if (e == null) { return null; } BeanConfig bean = new BeanConfig(); bean.setName(e.getAttribute("name")); Element element = loadElement(e, "className"); if (element != null) { bean.setClassName(element.getTextContent()); } else { bean.setClassName(e.getAttribute("class")); } bean.setParams(loadElements(e)); return bean; } } ================================================ FILE: src/main/java/io/mycat/config/util/DnPropertyUtil.java ================================================ package io.mycat.config.util; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.config.model.SystemConfig; /** * * @author yanglixue * */ public class DnPropertyUtil { private static final Logger LOGGER = LoggerFactory.getLogger("DnPropertyUtil"); /** * 加载dnindex.properties属性文件 * @return 属性文件 */ public static Properties loadDnIndexProps() { Properties prop = new Properties(); File file = new File(SystemConfig.getHomePath(), "conf" + File.separator + "dnindex.properties"); if (!file.exists()) { return prop; } FileInputStream filein = null; try { filein = new FileInputStream(file); prop.load(filein); } catch (Exception e) { LOGGER.warn("load DataNodeIndex err:" + e); } finally { if (filein != null) { try { filein.close(); } catch (IOException e) { } } } return prop; } } ================================================ FILE: src/main/java/io/mycat/config/util/FieldDictionary.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.util; import java.lang.reflect.Field; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.Map; /** * @author mycat */ public class FieldDictionary { private final Map> nameCache = Collections .synchronizedMap(new HashMap>()); private final Map> keyCache = Collections .synchronizedMap(new HashMap>()); /** * Returns an iterator for all serializable fields for some class * * @param cls * the class you are interested on * @return an iterator for its serializable fields */ public Iterator serializableFieldsFor(Class cls) { return buildMap(cls, true).values().iterator(); } /** * Returns an specific field of some class. If definedIn is null, it searchs * for the field named 'name' inside the class cls. If definedIn is * different than null, tries to find the specified field name in the * specified class cls which should be defined in class definedIn (either * equals cls or a one of it's superclasses) * * @param cls * the class where the field is to be searched * @param name * the field name * @param definedIn * the superclass (or the class itself) of cls where the field * was defined * @return the field itself */ public Field field(Class cls, String name, Class definedIn) { Map fields = buildMap(cls, definedIn != null); Field field = fields.get(definedIn != null ? new FieldKey(name, definedIn, 0) : name); if (field == null) { throw new ObjectAccessException("No such field " + cls.getName() + "." + name); } else { return field; } } private Map buildMap(Class cls, boolean tupleKeyed) { final String clsName = cls.getName(); if (!nameCache.containsKey(clsName)) { synchronized (keyCache) { if (!nameCache.containsKey(clsName)) { // double check final Map keyedByFieldName = new HashMap(); final Map keyedByFieldKey = new OrderRetainingMap(); while (!Object.class.equals(cls)) { Field[] fields = cls.getDeclaredFields(); if (JVMInfo.reverseFieldDefinition()) { for (int i = fields.length >> 1; i-- > 0;) { final int idx = fields.length - i - 1; final Field field = fields[i]; fields[i] = fields[idx]; fields[idx] = field; } } for (int i = 0; i < fields.length; i++) { Field field = fields[i]; field.setAccessible(true); if (!keyedByFieldName.containsKey(field.getName())) { keyedByFieldName.put(field.getName(), field); } keyedByFieldKey.put(new FieldKey(field.getName(), field.getDeclaringClass(), i), field); } cls = cls.getSuperclass(); } nameCache.put(clsName, keyedByFieldName); keyCache.put(clsName, keyedByFieldKey); } } } return tupleKeyed ? keyCache.get(clsName) : nameCache.get(clsName); } /** * @author mycat */ private static class FieldKey { private String fieldName; private Class declaringClass; private Integer depth; private int order; public FieldKey(String fieldName, Class declaringClass, int order) { this.fieldName = fieldName; this.declaringClass = declaringClass; this.order = order; Class c = declaringClass; int i = 0; while (c.getSuperclass() != null) { i++; c = c.getSuperclass(); } //不用构造器创建Integer，用静态方法节省时间和空间，因为depth是不可变变量 depth = Integer.valueOf(i); } @Override public boolean equals(Object o) { if (this == o) { return true; } if (!(o instanceof FieldKey)) { return false; } final FieldKey fieldKey = (FieldKey) o; if (declaringClass != null ? !declaringClass.equals(fieldKey.declaringClass) : fieldKey.declaringClass != null) { return false; } if (fieldName != null ? !fieldName.equals(fieldKey.fieldName) : fieldKey.fieldName != null) { return false; } return true; } @Override public int hashCode() { int result; result = (fieldName != null ? fieldName.hashCode() : 0); result = 29 * result + (declaringClass != null ? declaringClass.hashCode() : 0); return result; } @Override public String toString() { return "FieldKey{" + "order=" + order + ", writer=" + depth + ", declaringClass=" + declaringClass + ", fieldName='" + fieldName + "'" + "}"; } } } ================================================ FILE: src/main/java/io/mycat/config/util/Initializable.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.util; /** * @author mycat */ public interface Initializable { void init(); } ================================================ FILE: src/main/java/io/mycat/config/util/JVMInfo.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.util; import java.lang.reflect.Field; import java.text.AttributedString; import java.util.HashMap; import java.util.Map; /** * @author mycat */ public class JVMInfo { private static final float DEFAULT_JAVA_VERSION = 1.3f; private static final boolean reverseFieldOrder; private static final float majorJavaVersion = getMajorJavaVersion(System.getProperty("java.specification.version")); private ReflectionProvider reflectionProvider; private Map> loaderCache = new HashMap>(); static { boolean reverse = false; final Field[] fields = AttributedString.class.getDeclaredFields(); for (int i = 0; i < fields.length; i++) { if (fields[i].getName().equals("text")) { reverse = i > 3; } } reverseFieldOrder = reverse; } /** * Parses the java version system property to determine the major java * version, ie 1.x * * @param javaVersion * the system property 'java.specification.version' * @return A float of the form 1.x */ public static final float getMajorJavaVersion(String javaVersion) { try { return Float.parseFloat(javaVersion.substring(0, 3)); } catch (NumberFormatException e) { // Some JVMs may not conform to the x.y.z java.version format return DEFAULT_JAVA_VERSION; } } public static boolean is14() { return majorJavaVersion >= 1.4f; } public static boolean is15() { return majorJavaVersion >= 1.5f; } public static boolean is16() { return majorJavaVersion >= 1.6f; } private static boolean isSun() { return System.getProperty("java.vm.vendor").indexOf("Sun") != -1; } private static boolean isApple() { return System.getProperty("java.vm.vendor").indexOf("Apple") != -1; } private static boolean isHPUX() { return System.getProperty("java.vm.vendor").indexOf("Hewlett-Packard Company") != -1; } private static boolean isIBM() { return System.getProperty("java.vm.vendor").indexOf("IBM") != -1; } private static boolean isBlackdown() { return System.getProperty("java.vm.vendor").indexOf("Blackdown") != -1; } /* * Support for sun.misc.Unsafe and sun.reflect.ReflectionFactory is present * in JRockit versions R25.1.0 and later, both 1.4.2 and 5.0 (and in future * 6.0 builds). */ private static boolean isBEAWithUnsafeSupport() { // This property should be "BEA Systems, Inc." if (System.getProperty("java.vm.vendor").indexOf("BEA") != -1) { /* * Recent 1.4.2 and 5.0 versions of JRockit have a java.vm.version * string starting with the "R" JVM version number, i.e. * "R26.2.0-38-57237-1.5.0_06-20060209..." */ String vmVersion = System.getProperty("java.vm.version"); if (vmVersion.startsWith("R")) { /* * Wecould also check that it's R26 or later, but that is * implicitly true */ return true; } /* * For older JRockit versions we can check java.vm.info. JRockit * 1.4.2 R24 -> "Native Threads, GC strategy: parallel" and JRockit * 5.0 R25 -> "R25.2.0-28". */ String vmInfo = System.getProperty("java.vm.info"); if (vmInfo != null) { // R25.1 or R25.2 supports Unsafe, other versions do not return (vmInfo.startsWith("R25.1") || vmInfo.startsWith("R25.2")); } } // If non-BEA, or possibly some very old JRockit version return false; } public Class loadClass(String name) { try { Class clazz = loaderCache.get(name); if (clazz == null) { clazz = Class.forName(name, false, getClass().getClassLoader()); loaderCache.put(name, clazz); } return clazz; } catch (ClassNotFoundException e) { return null; } } public synchronized ReflectionProvider getReflectionProvider() { if (reflectionProvider == null) { reflectionProvider = new ReflectionProvider(); } return reflectionProvider; } protected boolean canUseSun14ReflectionProvider() { return (isSun() || isApple() || isHPUX() || isIBM() || isBlackdown() || isBEAWithUnsafeSupport()) && is14() && loadClass("sun.misc.Unsafe") != null; } public static boolean reverseFieldDefinition() { return reverseFieldOrder; } public static void main(String[] args) { System.out.println(majorJavaVersion); } } ================================================ FILE: src/main/java/io/mycat/config/util/ObjectAccessException.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.util; /** * @author mycat */ public class ObjectAccessException extends RuntimeException { private static final long serialVersionUID = 1L; public ObjectAccessException(String message) { super(message); } public ObjectAccessException(String message, Throwable cause) { super(message, cause); } } ================================================ FILE: src/main/java/io/mycat/config/util/OrderRetainingMap.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.util; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map.Entry; import java.util.Set; /** * @author mycat */ public class OrderRetainingMap extends HashMap { private static final long serialVersionUID = 1L; private Set keyOrder = new ArraySet(); private List valueOrder = new ArrayList(); @Override public V put(K key, V value) { keyOrder.add(key); valueOrder.add(value); return super.put(key, value); } @Override public Collection values() { return Collections.unmodifiableList(valueOrder); } @Override public Set keySet() { return Collections.unmodifiableSet(keyOrder); } @Override public Set> entrySet() { throw new UnsupportedOperationException(); } /** * @author mycat */ private static class ArraySet extends ArrayList implements Set { private static final long serialVersionUID = 1L; } } ================================================ FILE: src/main/java/io/mycat/config/util/ParameterMapping.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.util; import java.beans.BeanInfo; import java.beans.IntrospectionException; import java.beans.Introspector; import java.beans.PropertyDescriptor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.util.StringUtil; /** * @author mycat */ public class ParameterMapping { private static final Logger LOGGER = LoggerFactory .getLogger(ParameterMapping.class); private static final Map, PropertyDescriptor[]> descriptors = new HashMap, PropertyDescriptor[]>(); /** * 将property键值对赋值组装到object中 * @param object 目标反射对象 * @param parameter property的键值对 * @throws IllegalAccessException * @throws InvocationTargetException */ public static void mapping(Object object, Map parameter) throws IllegalAccessException, InvocationTargetException { //获取用于导出clazz这个JavaBean的所有属性的PropertyDescriptor PropertyDescriptor[] pds = getDescriptors(object.getClass()); for (int i = 0; i < pds.length; i++) { PropertyDescriptor pd = pds[i]; Object obj = parameter.get(pd.getName()); Object value = obj; Class cls = pd.getPropertyType(); //类型转换 if (obj instanceof String) { String string = (String) obj; if (!StringUtil.isEmpty(string)) { string = ConfigUtil.filter(string); } if (isPrimitiveType(cls)) { value = convert(cls, string); } } else if (obj instanceof BeanConfig) { value = createBean((BeanConfig) obj); } else if (obj instanceof BeanConfig[]) { List list = new ArrayList(); for (BeanConfig beanconfig : (BeanConfig[]) obj) { list.add(createBean(beanconfig)); } value = list.toArray(); } //赋值 if (cls != null && value != null) { Method method = pd.getWriteMethod(); if (method != null) { method.invoke(object, new Object[] { value }); } } } } @SuppressWarnings("unchecked") public static Object createBean(BeanConfig config) throws IllegalAccessException, InvocationTargetException { Object bean = config.create(true); if (bean instanceof Map) { Map map = (Map) bean; for (Map.Entry entry : config.getParams().entrySet()) { String key = entry.getKey(); Object value = entry.getValue(); if (value instanceof BeanConfig) { BeanConfig mapBeanConfig = (BeanConfig) entry.getValue(); value = mapBeanConfig.create(true); mapping(value, mapBeanConfig.getParams()); } map.put(key, value); } } else if (bean instanceof List) { } else { mapping(bean, config.getParams()); } return bean; } /** * 用于导出clazz这个JavaBean的所有属性的PropertyDescriptor * @param clazz * @return */ private static PropertyDescriptor[] getDescriptors(Class clazz) { //PropertyDescriptor类表示JavaBean类通过存储器导出一个属性 PropertyDescriptor[] pds; List list; PropertyDescriptor[] pds2 = descriptors.get(clazz); //该clazz是否第一次加载 if (null == pds2) { try { BeanInfo beanInfo = Introspector.getBeanInfo(clazz); pds = beanInfo.getPropertyDescriptors(); list = new ArrayList(); //加载每一个类型不为空的property for (int i = 0; i < pds.length; i++) { if (null != pds[i].getPropertyType()) { list.add(pds[i]); } } pds2 = new PropertyDescriptor[list.size()]; list.toArray(pds2); } catch (IntrospectionException ie) { LOGGER.error("ParameterMappingError", ie); pds2 = new PropertyDescriptor[0]; } } descriptors.put(clazz, pds2); return (pds2); } private static Object convert(Class cls, String string) { Method method = null; Object value = null; if (cls.equals(String.class)) { value = string; } else if (cls.equals(Boolean.TYPE)) { value = Boolean.valueOf(string); } else if (cls.equals(Byte.TYPE)) { value = Byte.valueOf(string); } else if (cls.equals(Short.TYPE)) { value = Short.valueOf(string); } else if (cls.equals(Integer.TYPE)) { value = Integer.valueOf(string); } else if (cls.equals(Long.TYPE)) { value = Long.valueOf(string); } else if (cls.equals(Double.TYPE)) { value = Double.valueOf(string); } else if (cls.equals(Float.TYPE)) { value = Float.valueOf(string); } else if ((cls.equals(Boolean.class)) || (cls.equals(Byte.class)) || (cls.equals(Short.class)) || (cls.equals(Integer.class)) || (cls.equals(Long.class)) || (cls.equals(Float.class)) || (cls.equals(Double.class))) { try { method = cls.getMethod("valueOf", new Class[] { String.class }); value = method.invoke(null, new Object[] { string }); } catch (Exception t) { LOGGER.error("valueofError", t); value = null; } } else if (cls.equals(Class.class)) { try { value = Class.forName(string); } catch (ClassNotFoundException e) { throw new ConfigException(e); } } else { value = null; } return (value); } private static boolean isPrimitiveType(Class cls) { if (cls.equals(String.class) || cls.equals(Boolean.TYPE) || cls.equals(Byte.TYPE) || cls.equals(Short.TYPE) || cls.equals(Integer.TYPE) || cls.equals(Long.TYPE) || cls.equals(Double.TYPE) || cls.equals(Float.TYPE) || cls.equals(Boolean.class) || cls.equals(Byte.class) || cls.equals(Short.class) || cls.equals(Integer.class) || cls.equals(Long.class) || cls.equals(Float.class) || cls.equals(Double.class) || cls.equals(Class.class)) { return true; } else { return false; } } } ================================================ FILE: src/main/java/io/mycat/config/util/ReflectionProvider.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.util; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectStreamClass; import java.io.ObjectStreamConstants; import java.io.Serializable; import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.Map; /** * @author mycat */ public class ReflectionProvider { private transient Map, byte[]> serializedDataCache = Collections .synchronizedMap(new HashMap, byte[]>()); private transient FieldDictionary fieldDictionary = new FieldDictionary(); public Object newInstance(Class type) { try { Constructor[] c = type.getDeclaredConstructors(); for (int i = 0; i < c.length; i++) { if (c[i].getParameterTypes().length == 0) { if (!Modifier.isPublic(c[i].getModifiers())) { c[i].setAccessible(true); } return c[i].newInstance(new Object[0]); } } if (Serializable.class.isAssignableFrom(type)) { return instantiateUsingSerialization(type); } else { throw new ObjectAccessException("Cannot construct " + type.getName() + " as it does not have a no-args constructor"); } } catch (InstantiationException e) { throw new ObjectAccessException("Cannot construct " + type.getName(), e); } catch (IllegalAccessException e) { throw new ObjectAccessException("Cannot construct " + type.getName(), e); } catch (InvocationTargetException e) { if (e.getTargetException() instanceof RuntimeException) { throw (RuntimeException) e.getTargetException(); } else if (e.getTargetException() instanceof Error) { throw (Error) e.getTargetException(); } else { throw new ObjectAccessException("Constructor for " + type.getName() + " threw an exception", e.getTargetException()); } } } public void visitSerializableFields(Object object, Visitor visitor) { for (Iterator iterator = fieldDictionary.serializableFieldsFor(object.getClass()); iterator.hasNext();) { Field field = iterator.next(); if (!fieldModifiersSupported(field)) { continue; } validateFieldAccess(field); try { Object value = field.get(object); visitor.visit(field.getName(), field.getType(), field.getDeclaringClass(), value); } catch (IllegalArgumentException e) { throw new ObjectAccessException("Could not get field " + field.getClass() + "." + field.getName(), e); } catch (IllegalAccessException e) { throw new ObjectAccessException("Could not get field " + field.getClass() + "." + field.getName(), e); } } } public void writeField(Object object, String fieldName, Object value, Class definedIn) { Field field = fieldDictionary.field(object.getClass(), fieldName, definedIn); validateFieldAccess(field); try { field.set(object, value); } catch (IllegalArgumentException e) { throw new ObjectAccessException("Could not set field " + field.getName() + "@" + object.getClass(), e); } catch (IllegalAccessException e) { throw new ObjectAccessException("Could not set field " + field.getName() + "@" + object.getClass(), e); } } public void invokeMethod(Object object, String methodName, Object value, Class definedIn) { try { Method method = object.getClass().getMethod(methodName, new Class[] { value.getClass() }); method.invoke(object, new Object[] { value }); } catch (Exception e) { throw new ObjectAccessException("Could not invoke " + object.getClass() + "." + methodName, e); } } public Class getFieldType(Object object, String fieldName, Class definedIn) { return fieldDictionary.field(object.getClass(), fieldName, definedIn).getType(); } public boolean fieldDefinedInClass(String fieldName, Class type) { try { Field field = fieldDictionary.field(type, fieldName, null); return fieldModifiersSupported(field); } catch (ObjectAccessException e) { return false; } } public Field getField(Class definedIn, String fieldName) { return fieldDictionary.field(definedIn, fieldName, null); } private Object instantiateUsingSerialization(Class type) { try { byte[] data; if (serializedDataCache.containsKey(type)) { data = serializedDataCache.get(type); } else { ByteArrayOutputStream bytes = new ByteArrayOutputStream(); DataOutputStream stream = new DataOutputStream(bytes); stream.writeShort(ObjectStreamConstants.STREAM_MAGIC); stream.writeShort(ObjectStreamConstants.STREAM_VERSION); stream.writeByte(ObjectStreamConstants.TC_OBJECT); stream.writeByte(ObjectStreamConstants.TC_CLASSDESC); stream.writeUTF(type.getName()); stream.writeLong(ObjectStreamClass.lookup(type).getSerialVersionUID()); stream.writeByte(2); // classDescFlags (2 = Serializable) stream.writeShort(0); // field count stream.writeByte(ObjectStreamConstants.TC_ENDBLOCKDATA); stream.writeByte(ObjectStreamConstants.TC_NULL); data = bytes.toByteArray(); serializedDataCache.put(type, data); } ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(data)); return in.readObject(); } catch (IOException e) { throw new ObjectAccessException("Cannot create " + type.getName() + " by JDK serialization", e); } catch (ClassNotFoundException e) { throw new ObjectAccessException("Cannot find class " + e.getMessage()); } } private boolean fieldModifiersSupported(Field field) { return !(Modifier.isStatic(field.getModifiers()) || Modifier.isTransient(field.getModifiers())); } private void validateFieldAccess(Field field) { if (Modifier.isFinal(field.getModifiers())) { if (JVMInfo.is15()) { field.setAccessible(true); } else { throw new ObjectAccessException("Invalid final field " + field.getDeclaringClass().getName() + "." + field.getName()); } } } private Object readResolve() { serializedDataCache = Collections.synchronizedMap(new HashMap, byte[]>()); fieldDictionary = new FieldDictionary(); return this; } } ================================================ FILE: src/main/java/io/mycat/config/util/Visitor.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.config.util; /** * @author mycat */ public interface Visitor { void visit(String name, Class type, Class definedIn, Object value); } ================================================ FILE: src/main/java/io/mycat/manager/ManagerConnection.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager; import java.io.IOException; import java.nio.channels.NetworkChannel; import io.mycat.config.model.SystemConfig; import io.mycat.net.FrontendConnection; import io.mycat.util.TimeUtil; /** * @author mycat */ public class ManagerConnection extends FrontendConnection { private long authTimeout = SystemConfig.DEFAULT_AUTH_TIMEOUT; public ManagerConnection(NetworkChannel channel) throws IOException { super(channel); } @Override public boolean isIdleTimeout() { if (isAuthenticated) { return super.isIdleTimeout(); } else { return TimeUtil.currentTimeMillis() > Math.max(lastWriteTime, lastReadTime) + this.authTimeout; } } public long getAuthTimeout() { return authTimeout; } public void setAuthTimeout(long authTimeout) { this.authTimeout = authTimeout; } @Override public void handle(final byte[] data) { this.executeSqlId ++; handler.handle(data); } @Override public void checkQueueFlow() { // TODO Auto-generated method stub } } ================================================ FILE: src/main/java/io/mycat/manager/ManagerConnectionFactory.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager; import java.io.IOException; import java.nio.channels.NetworkChannel; import io.mycat.MycatServer; import io.mycat.config.MycatPrivileges; import io.mycat.config.model.SystemConfig; import io.mycat.net.FrontendConnection; import io.mycat.net.factory.FrontendConnectionFactory; /** * @author mycat */ public class ManagerConnectionFactory extends FrontendConnectionFactory { @Override protected FrontendConnection getConnection(NetworkChannel channel) throws IOException { SystemConfig sys = MycatServer.getInstance().getConfig().getSystem(); ManagerConnection c = new ManagerConnection(channel); MycatServer.getInstance().getConfig().setSocketParams(c, true); c.setAuthTimeout(sys.getAuthTimeout()); c.setPrivileges(MycatPrivileges.instance()); c.setQueryHandler(new ManagerQueryHandler(c)); return c; } } ================================================ FILE: src/main/java/io/mycat/manager/ManagerQueryHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager; import io.mycat.config.ErrorCode; import io.mycat.manager.handler.*; import io.mycat.manager.response.KillConnection; import io.mycat.manager.response.Offline; import io.mycat.manager.response.Online; import io.mycat.net.handler.FrontendQueryHandler; import io.mycat.net.mysql.OkPacket; import io.mycat.route.parser.ManagerParse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author mycat */ public class ManagerQueryHandler implements FrontendQueryHandler { private static final Logger LOGGER = LoggerFactory.getLogger(ManagerQueryHandler.class); private static final int SHIFT = 8; private final ManagerConnection source; protected Boolean readOnly; public ManagerQueryHandler(ManagerConnection source) { this.source = source; } public void setReadOnly(Boolean readOnly) { this.readOnly = readOnly; } @Override public void query(String sql) { ManagerConnection c = this.source; if (LOGGER.isDebugEnabled()) { LOGGER.debug(new StringBuilder().append(c).append(sql).toString()); } int rs = ManagerParse.parse(sql); switch (rs & 0xff) { case ManagerParse.SELECT: SelectHandler.handle(sql, c, rs >>> SHIFT); break; case ManagerParse.SET: c.write(c.writeToBuffer(OkPacket.OK, c.allocate())); break; case ManagerParse.SHOW: ShowHandler.handle(sql, c, rs >>> SHIFT); break; case ManagerParse.SWITCH: SwitchHandler.handler(sql, c, rs >>> SHIFT); break; case ManagerParse.KILL_CONN: KillConnection.response(sql, rs >>> SHIFT, c); break; case ManagerParse.OFFLINE: Offline.execute(sql, c); break; case ManagerParse.ONLINE: Online.execute(sql, c); break; case ManagerParse.STOP: StopHandler.handle(sql, c, rs >>> SHIFT); break; case ManagerParse.RELOAD: ReloadHandler.handle(sql, c, rs >>> SHIFT); break; case ManagerParse.ROLLBACK: RollbackHandler.handle(sql, c, rs >>> SHIFT); break; case ManagerParse.CLEAR: ClearHandler.handle(sql, c, rs >>> SHIFT); break; case ManagerParse.CONFIGFILE: ConfFileHandler.handle(sql, c); break; case ManagerParse.LOGFILE: ShowServerLog.handle(sql, c); break; case ManagerParse.ZK: ZKHandler.handle(sql, c, rs >>> SHIFT); break; default: c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement"); } } } ================================================ FILE: src/main/java/io/mycat/manager/handler/ClearHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.handler; import io.mycat.config.ErrorCode; import io.mycat.manager.ManagerConnection; import io.mycat.manager.response.ClearSlow; import io.mycat.route.parser.ManagerParseClear; import io.mycat.util.StringUtil; /** * @author mycat */ public class ClearHandler { public static void handle(String stmt, ManagerConnection c, int offset) { int rs = ManagerParseClear.parse(stmt, offset); switch (rs & 0xff) { case ManagerParseClear.SLOW_DATANODE: { String name = stmt.substring(rs >>> 8).trim(); if (StringUtil.isEmpty(name)) { c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement"); } else { ClearSlow.dataNode(c, name); } break; } case ManagerParseClear.SLOW_SCHEMA: { String name = stmt.substring(rs >>> 8).trim(); if (StringUtil.isEmpty(name)) { c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement"); } else { ClearSlow.schema(c, name); } break; } default: c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement"); } } } ================================================ FILE: src/main/java/io/mycat/manager/handler/ConfFileHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.handler; import java.io.BufferedOutputStream; import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; import java.text.SimpleDateFormat; import java.util.Date; import javax.xml.parsers.ParserConfigurationException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.xml.sax.SAXException; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.config.model.SystemConfig; import io.mycat.config.util.ConfigUtil; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.StringUtil; /** * Mycat conf file related Handler * * @author wuzh */ public final class ConfFileHandler { private static final Logger LOGGER = LoggerFactory .getLogger(ConfFileHandler.class); private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil .getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); private static final String UPLOAD_CMD = "FILE @@UPLOAD"; static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("DATA", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void handle( String stmt,ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; String theStmt = stmt.toUpperCase().trim(); PackageBufINf bufInf = null; if (theStmt.equals("FILE @@LIST")) { bufInf = listConfigFiles(c, buffer, packetId); } else if (theStmt.startsWith("FILE @@SHOW")) { int index = stmt.lastIndexOf(' '); String fileName = stmt.substring(index + 1); bufInf = showConfigFile(c, buffer, packetId, fileName); } else if (theStmt.startsWith(UPLOAD_CMD)) { int index = stmt.indexOf(' ', UPLOAD_CMD.length()); int index2 = stmt.indexOf(' ', index + 1); if (index <= 0 || index2 <= 0 || index + 1 > stmt.length() || index2 + 1 > stmt.length()) { bufInf = showInfo(c, buffer, packetId, "Invald param ,usage "); } String fileName = stmt.substring(index + 1, index2); String content = stmt.substring(index2 + 1).trim(); bufInf = upLoadConfigFile(c, buffer, packetId, fileName, content); } else { bufInf = showInfo(c, buffer, packetId, "Invald command "); } packetId = bufInf.packetId; buffer = bufInf.buffer; // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } private static void checkXMLFile(String xmlFileName, byte[] data) throws ParserConfigurationException, SAXException, IOException { InputStream dtdStream = new ByteArrayInputStream(new byte[0]); File confDir = new File(SystemConfig.getHomePath(), "conf"); if (xmlFileName.equals("schema.xml")) { dtdStream = MycatServer.class.getResourceAsStream("/schema.dtd"); if (dtdStream == null) { dtdStream = new ByteArrayInputStream(readFileByBytes(new File( confDir, "schema.dtd"))); } } else if (xmlFileName.equals("server.xml")) { dtdStream = MycatServer.class.getResourceAsStream("/server.dtd"); if (dtdStream == null) { dtdStream = new ByteArrayInputStream(readFileByBytes(new File( confDir, "server.dtd"))); } } else if (xmlFileName.equals("rule.xml")) { dtdStream = MycatServer.class.getResourceAsStream("/rule.dtd"); if (dtdStream == null) { dtdStream = new ByteArrayInputStream(readFileByBytes(new File( confDir, "rule.dtd"))); } } ConfigUtil.getDocument(dtdStream, new ByteArrayInputStream(data)); } /** * 以字节为单位读取文件，常用于读二进制文件，如图片、声音、影像等文件。 */ private static byte[] readFileByBytes(File fileName) { InputStream in = null; ByteArrayOutputStream outStream = new ByteArrayOutputStream(); try { // 一次读多个字节 byte[] tempbytes = new byte[100]; int byteread = 0; in = new FileInputStream(fileName); // 读入多个字节到字节数组中，byteread为一次读入的字节数 while ((byteread = in.read(tempbytes)) != -1) { outStream.write(tempbytes, 0, byteread); } } catch (Exception e1) { LOGGER.error("readFileByBytesError",e1); } finally { if (in != null) { try { in.close(); } catch (IOException e1) { LOGGER.error("readFileByBytesError",e1); } } } return outStream.toByteArray(); } private static PackageBufINf upLoadConfigFile(ManagerConnection c, ByteBuffer buffer, byte packetId, String fileName, String content) { LOGGER.info("Upload Daas Config file " + fileName + " ,content:" + content); String tempFileName = System.currentTimeMillis() + "_" + fileName; File tempFile = new File(SystemConfig.getHomePath(), "conf" + File.separator + tempFileName); BufferedOutputStream buff = null; boolean suc = false; try { byte[] fileData = content.getBytes("UTF-8"); if (fileName.endsWith(".xml")) { checkXMLFile(fileName, fileData); } buff = new BufferedOutputStream(new FileOutputStream(tempFile)); buff.write(fileData); buff.flush(); } catch (Exception e) { LOGGER.warn("write file err " + e); return showInfo(c, buffer, packetId, "write file err " + e); } finally { if (buff != null) { try { buff.close(); suc = true; } catch (IOException e) { LOGGER.warn("save config file err " + e); } } } if (suc) { // if succcess File oldFile = new File(SystemConfig.getHomePath(), "conf" + File.separator + fileName); if (oldFile.exists()) { File backUP = new File(SystemConfig.getHomePath(), "conf" + File.separator + fileName + "_" + System.currentTimeMillis() + "_auto"); if (!oldFile.renameTo(backUP)) { String msg = "rename old file failed"; LOGGER.warn(msg + " for upload file " + oldFile.getAbsolutePath()); return showInfo(c, buffer, packetId, msg); } } File dest = new File(SystemConfig.getHomePath(), "conf" + File.separator + fileName); if (!tempFile.renameTo(dest)) { String msg = "rename file failed"; LOGGER.warn(msg + " for upload file " + tempFile.getAbsolutePath()); return showInfo(c, buffer, packetId, msg); } return showInfo(c, buffer, packetId, "SUCCESS SAVED FILE:" + fileName); } else { return showInfo(c, buffer, packetId, "UPLOAD ERROR OCCURD:" + fileName); } } private static PackageBufINf showInfo(ManagerConnection c, ByteBuffer buffer, byte packetId, String string) { PackageBufINf bufINf = new PackageBufINf(); RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(string, c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); bufINf.packetId = packetId; bufINf.buffer = buffer; return bufINf; } private static PackageBufINf showConfigFile(ManagerConnection c, ByteBuffer buffer, byte packetId, String fileName) { File file = new File(SystemConfig.getHomePath(), "conf" + File.separator + fileName); BufferedReader br = null; PackageBufINf bufINf = new PackageBufINf(); try { br = new BufferedReader(new FileReader(file)); String line = null; while ((line = br.readLine()) != null) { if (line.isEmpty()) { continue; } RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(line, c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } bufINf.buffer = buffer; bufINf.packetId = packetId; return bufINf; } catch (Exception e) { LOGGER.error("showConfigFileError",e); RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(e.toString(), c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); bufINf.buffer = buffer; } finally { if (br != null) { try { br.close(); } catch (IOException e) { LOGGER.error("showConfigFileError",e); } } } bufINf.packetId = packetId; return bufINf; } private static PackageBufINf listConfigFiles(ManagerConnection c, ByteBuffer buffer, byte packetId) { PackageBufINf bufINf = new PackageBufINf(); SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm"); try { int i = 1; File[] file = new File(SystemConfig.getHomePath(), "conf") .listFiles(); for (File f : file) { if (f.isFile()) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode( (i++) + " : " + f.getName() + " time:" + df.format(new Date(f.lastModified())), c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } } bufINf.buffer = buffer; bufINf.packetId = packetId; return bufINf; } catch (Exception e) { LOGGER.error("listConfigFilesError",e); RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(e.toString(), c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); bufINf.buffer = buffer; } bufINf.packetId = packetId; return bufINf; } public static void main(String[] args) { String stmt = "FILE @@UPLOAD test.xml 1234567890"; int index = stmt.indexOf(' ', UPLOAD_CMD.length()); int index2 = stmt.indexOf(' ', index + 1); if (index <= 0 || index2 <= 0 || index + 1 > stmt.length() || index2 + 1 > stmt.length()) { System.out.println("valid ...."); } else { String fileName = stmt.substring(index + 1, index2); String content = stmt.substring(index2 + 1).trim(); System.out.println(fileName + " content:" + content); } } } ================================================ FILE: src/main/java/io/mycat/manager/handler/ReloadHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.handler; import io.mycat.config.ErrorCode; import io.mycat.manager.ManagerConnection; import io.mycat.manager.response.ReloadConfig; import io.mycat.manager.response.ReloadQueryCf; import io.mycat.manager.response.ReloadSqlSlowTime; import io.mycat.manager.response.ReloadSqlStat; import io.mycat.manager.response.ReloadUser; import io.mycat.manager.response.ReloadUserStat; import io.mycat.route.parser.ManagerParseReload; import io.mycat.route.parser.util.ParseUtil; /** * @author mycat */ public final class ReloadHandler { public static void handle(String stmt, ManagerConnection c, int offset) { int rs = ManagerParseReload.parse(stmt, offset); switch (rs) { case ManagerParseReload.CONFIG: ReloadConfig.execute(c,false); break; case ManagerParseReload.CONFIG_ALL: ReloadConfig.execute(c,true); break; case ManagerParseReload.ROUTE: c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement"); break; case ManagerParseReload.USER: ReloadUser.execute(c); break; case ManagerParseReload.USER_STAT: ReloadUserStat.execute(c); break; case ManagerParseReload.SQL_SLOW: ReloadSqlSlowTime.execute(c, ParseUtil.getSQLId(stmt)); break; case ManagerParseReload.QUERY_CF: String filted = ParseUtil.parseString(stmt) ; ReloadQueryCf.execute(c, filted); break; case ManagerParseReload.SQL_STAT: String openCloseFlag = ParseUtil.parseString(stmt) ; ReloadSqlStat.execute(c, openCloseFlag); break; default: c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement"); } } } ================================================ FILE: src/main/java/io/mycat/manager/handler/RollbackHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.handler; import io.mycat.config.ErrorCode; import io.mycat.manager.ManagerConnection; import io.mycat.manager.response.RollbackConfig; import io.mycat.manager.response.RollbackUser; import io.mycat.route.parser.ManagerParseRollback; /** * @author mycat */ public final class RollbackHandler { public static void handle(String stmt, ManagerConnection c, int offset) { switch (ManagerParseRollback.parse(stmt, offset)) { case ManagerParseRollback.CONFIG: RollbackConfig.execute(c); break; case ManagerParseRollback.ROUTE: c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement"); break; case ManagerParseRollback.USER: RollbackUser.execute(c); break; default: c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement"); } } } ================================================ FILE: src/main/java/io/mycat/manager/handler/SelectHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.handler; import static io.mycat.route.parser.ManagerParseSelect.SESSION_AUTO_INCREMENT; import static io.mycat.route.parser.ManagerParseSelect.VERSION_COMMENT; import static io.mycat.route.parser.ManagerParseSelect.SESSION_TX_READ_ONLY; import io.mycat.config.ErrorCode; import io.mycat.manager.ManagerConnection; import io.mycat.manager.response.SelectSessionAutoIncrement; import io.mycat.manager.response.SelectSessionTxReadOnly; import io.mycat.manager.response.SelectVersionComment; import io.mycat.route.parser.ManagerParseSelect; /** * @author mycat */ public final class SelectHandler { public static void handle(String stmt, ManagerConnection c, int offset) { switch (ManagerParseSelect.parse(stmt, offset)) { case VERSION_COMMENT: SelectVersionComment.execute(c); break; case SESSION_AUTO_INCREMENT: SelectSessionAutoIncrement.execute(c); break; case SESSION_TX_READ_ONLY: SelectSessionTxReadOnly.execute(c); break; default: c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement"); } } } ================================================ FILE: src/main/java/io/mycat/manager/handler/ShowHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.handler; import io.mycat.config.ErrorCode; import io.mycat.manager.ManagerConnection; import io.mycat.manager.response.CheckGlobalTable; import io.mycat.manager.response.ShowBackend; import io.mycat.manager.response.ShowBackendOld; import io.mycat.manager.response.ShowCollation; import io.mycat.manager.response.ShowCommand; import io.mycat.manager.response.ShowConnection; import io.mycat.manager.response.ShowConnectionSQL; import io.mycat.manager.response.ShowDataNode; import io.mycat.manager.response.ShowDataSource; import io.mycat.manager.response.ShowDatabase; import io.mycat.manager.response.ShowDatasourceCluster; import io.mycat.manager.response.ShowDatasourceSyn; import io.mycat.manager.response.ShowDatasourceSynDetail; import io.mycat.manager.response.ShowHeartbeat; import io.mycat.manager.response.ShowHeartbeatDetail; import io.mycat.manager.response.ShowHelp; import io.mycat.manager.response.ShowParser; import io.mycat.manager.response.ShowProcessor; import io.mycat.manager.response.ShowRouter; import io.mycat.manager.response.ShowSQL; import io.mycat.manager.response.ShowSQLCondition; import io.mycat.manager.response.ShowSQLDetail; import io.mycat.manager.response.ShowSQLExecute; import io.mycat.manager.response.ShowSQLHigh; import io.mycat.manager.response.ShowSQLLarge; import io.mycat.manager.response.ShowSQLSlow; import io.mycat.manager.response.ShowSQLSumTable; import io.mycat.manager.response.ShowSQLSumUser; import io.mycat.manager.response.ShowServer; import io.mycat.manager.response.ShowSession; import io.mycat.manager.response.ShowSqlResultSet; import io.mycat.manager.response.ShowSysLog; import io.mycat.manager.response.ShowSysParam; import io.mycat.manager.response.ShowThreadPool; import io.mycat.manager.response.ShowTime; import io.mycat.manager.response.ShowVariables; import io.mycat.manager.response.ShowVersion; import io.mycat.manager.response.ShowWhiteHost; import io.mycat.manager.response.ShowDirectMemory; import io.mycat.route.parser.ManagerParseShow; import io.mycat.route.parser.util.ParseUtil; import io.mycat.server.handler.ShowCache; import io.mycat.util.StringUtil; /** * @author mycat */ public final class ShowHandler { public static void handle(String stmt, ManagerConnection c, int offset) { int rs = ManagerParseShow.parse(stmt, offset); switch (rs & 0xff) { case ManagerParseShow.SYSPARAM://add rainbow ShowSysParam.execute(c); break; case ManagerParseShow.SYSLOG: //add by zhuam String lines = stmt.substring(rs >>> 8).trim(); ShowSysLog.execute(c, Integer.parseInt( lines ) ); break; case ManagerParseShow.COMMAND: ShowCommand.execute(c); break; case ManagerParseShow.COLLATION: ShowCollation.execute(c); break; case ManagerParseShow.CONNECTION: ShowConnection.execute(c); break; case ManagerParseShow.BACKEND: ShowBackend.execute(c); break; case ManagerParseShow.BACKEND_OLD: ShowBackendOld.execute(c); break; case ManagerParseShow.CONNECTION_SQL: ShowConnectionSQL.execute(c); break; case ManagerParseShow.DATABASE: ShowDatabase.execute(c); break; case ManagerParseShow.DATANODE: ShowDataNode.execute(c, null); break; case ManagerParseShow.DATANODE_WHERE: { String name = stmt.substring(rs >>> 8).trim(); if (StringUtil.isEmpty(name)) { c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement"); } else { ShowDataNode.execute(c, name); } break; } case ManagerParseShow.DATASOURCE: ShowDataSource.execute(c, null); break; case ManagerParseShow.DATASOURCE_WHERE: { String name = stmt.substring(rs >>> 8).trim(); if (StringUtil.isEmpty(name)) { c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement"); } else { ShowDataSource.execute(c, name); } break; } case ManagerParseShow.HELP: ShowHelp.execute(c); break; case ManagerParseShow.HEARTBEAT: ShowHeartbeat.response(c); break; case ManagerParseShow.PARSER: ShowParser.execute(c); break; case ManagerParseShow.PROCESSOR: ShowProcessor.execute(c); break; case ManagerParseShow.ROUTER: ShowRouter.execute(c); break; case ManagerParseShow.SERVER: ShowServer.execute(c); break; case ManagerParseShow.WHITE_HOST: ShowWhiteHost.execute(c); break; case ManagerParseShow.WHITE_HOST_SET: ShowWhiteHost.setHost(c,ParseUtil.parseString(stmt)); break; case ManagerParseShow.SQL: boolean isClearSql = Boolean.valueOf( stmt.substring(rs >>> 8).trim() ); ShowSQL.execute(c, isClearSql); break; case ManagerParseShow.SQL_DETAIL: ShowSQLDetail.execute(c, ParseUtil.getSQLId(stmt)); break; case ManagerParseShow.SQL_EXECUTE: ShowSQLExecute.execute(c); break; case ManagerParseShow.SQL_SLOW: boolean isClearSlow = Boolean.valueOf( stmt.substring(rs >>> 8).trim() ); ShowSQLSlow.execute(c, isClearSlow, null); break; case ManagerParseShow.SQL_HIGH: boolean isClearHigh = Boolean.valueOf( stmt.substring(rs >>> 8).trim() ); ShowSQLHigh.execute(c, isClearHigh); break; case ManagerParseShow.SQL_LARGE: boolean isClearLarge = Boolean.valueOf( stmt.substring(rs >>> 8).trim() ); ShowSQLLarge.execute(c, isClearLarge); break; case ManagerParseShow.SQL_CONDITION: ShowSQLCondition.execute(c); break; case ManagerParseShow.SQL_RESULTSET: ShowSqlResultSet.execute(c); break; case ManagerParseShow.SQL_SUM_USER: boolean isClearSum = Boolean.valueOf( stmt.substring(rs >>> 8).trim() ); ShowSQLSumUser.execute(c,isClearSum); break; case ManagerParseShow.SQL_SUM_TABLE: boolean isClearTable = Boolean.valueOf( stmt.substring(rs >>> 8).trim() ); ShowSQLSumTable.execute(c, isClearTable); break; case ManagerParseShow.SLOW_DATANODE: { String name = stmt.substring(rs >>> 8).trim(); if (StringUtil.isEmpty(name)) { c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement"); } else { // ShowSlow.dataNode(c, name); c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement"); } break; } case ManagerParseShow.SLOW_SCHEMA: { String name = stmt.substring(rs >>> 8).trim(); if (StringUtil.isEmpty(name)) { c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement,schema value is null"); } else { // c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement"); ShowSQLSlow.execute(c, false, StringUtil.removeBackquote(name)); } break; } case ManagerParseShow.THREADPOOL: ShowThreadPool.execute(c); break; case ManagerParseShow.CACHE: ShowCache.execute(c); break; case ManagerParseShow.SESSION: ShowSession.execute(c); break; case ManagerParseShow.TIME_CURRENT: ShowTime.execute(c, ManagerParseShow.TIME_CURRENT); break; case ManagerParseShow.TIME_STARTUP: ShowTime.execute(c, ManagerParseShow.TIME_STARTUP); break; case ManagerParseShow.VARIABLES: ShowVariables.execute(c); break; case ManagerParseShow.VERSION: ShowVersion.execute(c); break; case ManagerParseShow.HEARTBEAT_DETAIL://by songwie ShowHeartbeatDetail.response(c,stmt); break; case ManagerParseShow.DATASOURCE_SYNC://by songwie ShowDatasourceSyn.response(c,stmt); break; case ManagerParseShow.DATASOURCE_SYNC_DETAIL://by songwie ShowDatasourceSynDetail.response(c,stmt); break; case ManagerParseShow.DATASOURCE_CLUSTER://by songwie ShowDatasourceCluster.response(c,stmt); break; case ManagerParseShow.DIRECTMEMORY_DETAILl: ShowDirectMemory.execute(c,2); break; case ManagerParseShow.DIRECTMEMORY_TOTAL: ShowDirectMemory.execute(c,1); break; case ManagerParseShow.CHECK_GLOBAL: CheckGlobalTable.execute(c, stmt); break; default: c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement"); } } } ================================================ FILE: src/main/java/io/mycat/manager/handler/ShowServerLog.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.handler; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.config.model.SystemConfig; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.CircularArrayList; import io.mycat.util.StringUtil; public final class ShowServerLog { private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil .getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); private static final String DEFAULT_LOGFILE = "mycat.log"; private static final Logger LOGGER = LoggerFactory .getLogger(ShowServerLog.class); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("LOG", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } private static File getLogFile(String logFile) { String daasHome = SystemConfig.getHomePath(); File file = new File(daasHome, "logs" + File.separator + logFile); return file; } public static void handle(String stmt,ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; PackageBufINf bufInf = null; // show log key=warn limit=0,30 Map condPairMap = getCondPair(stmt); if (condPairMap.isEmpty()) { bufInf = showLogSum(c, buffer, packetId); } else { String logFile = condPairMap.get("file"); if (logFile == null) { logFile = DEFAULT_LOGFILE; } String limitStr = condPairMap.get("limit"); limitStr = (limitStr != null) ? limitStr : "0," + 100000; String[] limtArry = limitStr.split("\\s|,"); int start = Integer.parseInt(limtArry[0]); int page = Integer.parseInt(limtArry[1]); int end = Integer.valueOf(start + page); String key = condPairMap.get("key"); String regex = condPairMap.get("regex"); bufInf = showLogRange(c, buffer, packetId, key, regex, start, end, logFile); } packetId = bufInf.packetId; buffer = bufInf.buffer; EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } public static PackageBufINf showLogRange(ManagerConnection c, ByteBuffer buffer, byte packetId, String key, String regex, int start, int end, String logFile) { PackageBufINf bufINf = new PackageBufINf(); Pattern pattern = null; if (regex != null) { pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE); } if (key != null) { key = key.toLowerCase(); } File file = getLogFile(logFile); BufferedReader br = null; int curLine = 0; try { br = new BufferedReader(new FileReader(file)); String line = null; while ((line = br.readLine()) != null) { curLine++; if (curLine >= start && curLine <= end && ( (pattern != null && pattern.matcher(line).find()) || (pattern == null && key == null) || (key != null && line.toLowerCase().contains(key)) )) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(curLine + "->" + line, c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } } bufINf.buffer = buffer; bufINf.packetId = packetId; return bufINf; } catch (Exception e) { LOGGER.error("showLogRangeError", e); RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(e.toString(), c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); bufINf.buffer = buffer; } finally { if (br != null) { try { br.close(); } catch (IOException e) { LOGGER.error("showLogRangeError", e); } } } bufINf.packetId = packetId; return bufINf; } private static PackageBufINf showLogSum(ManagerConnection c, ByteBuffer buffer, byte packetId) { PackageBufINf bufINf = new PackageBufINf(); File[] logFiles = new File(SystemConfig.getHomePath(), "logs") .listFiles(); String fileNames = ""; for (File f : logFiles) { if (f.isFile()) { fileNames += " " + f.getName(); } } File file = getLogFile(DEFAULT_LOGFILE); BufferedReader br = null; int totalLines = 0; CircularArrayList queue = new CircularArrayList(50); try { br = new BufferedReader(new FileReader(file)); String line = null; while ((line = br.readLine()) != null) { totalLines++; if (queue.size() == queue.capacity()) { queue.remove(0); } queue.add(line); } RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode("files in log dir:" + totalLines + fileNames, c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode("Total lines " + totalLines + " ,tail " + queue.size() + " line is following:", c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); int size = queue.size() - 1; for (int i = size; i >= 0; i--) { String data = queue.get(i); row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(data, c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } bufINf.buffer = buffer; bufINf.packetId = packetId; return bufINf; } catch (Exception e) { LOGGER.error("showLogSumError", e); RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(e.toString(), c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); bufINf.buffer = buffer; } finally { if (br != null) { try { br.close(); } catch (IOException e) { LOGGER.error("showLogSumError", e); } } } bufINf.packetId = packetId; return bufINf; } public static Map getCondPair(String sql) { HashMap map = new HashMap(); Pattern p = Pattern.compile("(\\S+\\s*=\\s*\\S+)"); Matcher m = p.matcher(sql); while (m.find()) { String item = m.group(); Pattern p2 = Pattern.compile("(\\S+)\\s*=\\s*(\\S+)"); Matcher m2 = p2.matcher(item); if (m2.find()) { map.put(m2.group(1), m2.group(2)); } } return map; } public static void main(String[] args) { String sql = "show log limit =1,2 key=warn file= \"2\" "; Map condPairMap = getCondPair(sql); for (Map.Entry entry : condPairMap.entrySet()) { System.out.println("key:" + entry.getKey() + ",value:" + entry.getValue()); } String limt = "1,2"; System.out.println(Arrays.toString(limt.split("\\s|,"))); } } class PackageBufINf { public byte packetId; public ByteBuffer buffer; } ================================================ FILE: src/main/java/io/mycat/manager/handler/StopHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.handler; import io.mycat.config.ErrorCode; import io.mycat.manager.ManagerConnection; import io.mycat.manager.response.StopHeartbeat; import io.mycat.route.parser.ManagerParseStop; /** * @author mycat */ public final class StopHandler { public static void handle(String stmt, ManagerConnection c, int offset) { switch (ManagerParseStop.parse(stmt, offset)) { case ManagerParseStop.HEARTBEAT: StopHeartbeat.execute(stmt, c); break; default: c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement"); } } } ================================================ FILE: src/main/java/io/mycat/manager/handler/SwitchHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.handler; import static io.mycat.route.parser.ManagerParseSwitch.DATASOURCE; import io.mycat.config.ErrorCode; import io.mycat.manager.ManagerConnection; import io.mycat.manager.response.SwitchDataSource; import io.mycat.route.parser.ManagerParseSwitch; /** * @author mycat */ public final class SwitchHandler { public static void handler(String stmt, ManagerConnection c, int offset) { switch (ManagerParseSwitch.parse(stmt, offset)) { case DATASOURCE: SwitchDataSource.response(stmt, c); break; default: c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement"); } } } ================================================ FILE: src/main/java/io/mycat/manager/handler/ZKHandler.java ================================================ package io.mycat.manager.handler; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.utils.ZKPaths; import org.apache.zookeeper.data.Stat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.config.ErrorCode; import io.mycat.config.loader.zkprocess.comm.ZkConfig; import io.mycat.config.loader.zkprocess.comm.ZkParamCfg; import io.mycat.config.loader.zkprocess.console.ZkNofiflyCfg; import io.mycat.config.loader.zkprocess.zktoxml.ZktoXmlMain; import io.mycat.manager.ManagerConnection; import io.mycat.manager.response.ReloadZktoXml; import io.mycat.util.ZKUtils; /** * zookeeper 实现动态配置 * * @author Hash Zhang * @version 1.0 * @time 23:35 2016/5/7 */ public class ZKHandler { private static final Logger LOGGER = LoggerFactory.getLogger(ZKHandler.class); /** * 直接从zk拉所有配置，然后本地执行reload_all */ public static final String RELOAD_FROM_ZK = "zk reload_from_zk"; /** * 强制所有节点操作 */ private static final String RELOAD_ALL = "all"; /** * 命令节点信息 */ public static final String ZK_NODE_PATH = "command"; public static void handle(String stmt, ManagerConnection c, int offset) { String command = stmt.toLowerCase(); // 检查当前的命令是否为zk reload_from_zk if (RELOAD_FROM_ZK.equals(command)) { // 调用zktoxml操作 try { // 通知所有节点进行数据更新 ZktoXmlMain.ZKLISTENER.notifly(ZkNofiflyCfg.ZK_NOTIFLY_LOAD_ALL.getKey()); // 执行重新加载本地配制信息 ReloadHandler.handle("RELOAD @@config_all", c, 6); offset += RELOAD_FROM_ZK.length(); ReloadZktoXml.execute(c, "zk reload success "); } catch (Exception e) { LOGGER.error("ZKHandler loadZktoFile exception", e); c.writeErrMessage(ErrorCode.ER_YES, "zk command send error,command is :" + command); } } else { String[] matchKeys = stmt.split("\\s+"); if (null != matchKeys && matchKeys.length > 2) { // 取得所有配制的节点信息 String key = ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_CLUSTER_NODES); String[] myidArray = key.split(","); String idkeys = matchKeys[1].toLowerCase(); // 发送的命令信息 StringBuilder commandMsg = new StringBuilder(); for (int i = 2; i < matchKeys.length; i++) { if (i == matchKeys.length - 1) { commandMsg.append(matchKeys[i]); } else { commandMsg.append(matchKeys[i]).append(" "); } } // 命令的形式为zk all reload_from_zk // 进行第二个匹配，检查是否为所有节点更新 if (RELOAD_ALL.equals(idkeys)) { // 按所有id，将把所有的节点都更新 try { // 将所有指令发送至服务器 for (String myid : myidArray) { sendZkCommand(myid, commandMsg.toString()); } ReloadZktoXml.execute(c, "zk reload " + matchKeys[1] + " success "); } catch (Exception e) { c.writeErrMessage(ErrorCode.ER_YES, "zk command send error"); } } // 如果不是所有节点，则检查是否能匹配上单独的节点 else { for (String myid : myidArray) { if (myid.equals(idkeys)) { try { sendZkCommand(myid, commandMsg.toString()); ReloadZktoXml.execute(c, "zk reload " + matchKeys[1] + " success "); } catch (Exception e) { c.writeErrMessage(ErrorCode.ER_YES, "zk command send error,myid :" + myid); } break; } } } } else { c.writeErrMessage(ErrorCode.ER_YES, "zk command is error"); } } } /** * 向节点发送命令 * @param myId 节点的id信息 * @param command 命令内容 * @throws Exception 异常信息 */ private static void sendZkCommand(String myId, String command) throws Exception { CuratorFramework zkConn = ZKUtils.getConnection(); String basePath = ZKUtils.getZKBasePath(); String nodePath = ZKPaths.makePath(basePath, ZK_NODE_PATH + "/" + myId); Stat stat; try { stat = zkConn.checkExists().forPath(nodePath); if (null == stat) { // 进行目录的创建操作 ZKPaths.mkdirs(zkConn.getZookeeperClient().getZooKeeper(), nodePath); } // 设置节点信息 zkConn.setData().inBackground().forPath(nodePath, command.getBytes()); } catch (Exception e) { LOGGER.error("ZKHandler sendZkCommand exception", e); throw e; } } } ================================================ FILE: src/main/java/io/mycat/manager/response/CheckGlobalTable.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.List; import java.util.Map; import com.google.common.base.CharMatcher; import com.google.common.base.Splitter; import io.mycat.MycatServer; import io.mycat.backend.heartbeat.ConsistenCollectHandler; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.ErrorCode; import io.mycat.config.Fields; import io.mycat.config.MycatConfig; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.TableConfig; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; /** * 全局表一致性检测 * * @author mycat */ public final class CheckGlobalTable { private static final int FIELD_COUNT = 3; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("ID", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("TABLENAME", Fields.FIELD_TYPE_VARCHAR); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("RESTULT", Fields.FIELD_TYPE_VARCHAR); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } private static Map parse(String sql) { Map map = new HashMap<>(); List rtn = Splitter.on(CharMatcher.whitespace()).omitEmptyStrings().splitToList(sql); for (String s : rtn) { if (s.contains("=")) { int dindex = s.indexOf("="); if (s.startsWith("--")) { String key = s.substring(2, dindex).toLowerCase().trim(); String value = s.substring(dindex + 1).trim(); map.put(key, value); } else if (s.startsWith("-")) { String key = s.substring(1, dindex).toLowerCase().trim(); String value = s.substring(dindex + 1).trim(); map.put(key, value); } } } return map; } public static void execute(ManagerConnection c ,String stmt) { Map paramster = parse(stmt); int retryTime = 1; long intervalTime = 200 ; //"show @@CKECK_GLOBAL -SCHEMA=TESTDB -TABLE=E_ACCOUNT_SUBJECT -retrytime=2" // + " -intervaltime=20" String tableName = paramster.get("table"); String schemaName = paramster.get("schema"); String retryTimeStr = paramster.get("retry"); String intervalTimeStr = paramster.get("interval"); MycatConfig config = MycatServer.getInstance().getConfig(); TableConfig table; SchemaConfig schemaConfig = null; if(StringUtil.isEmpty(schemaName)) { c.writeErrMessage(ErrorCode.ER_UNKNOWN_ERROR, "schemaName is null, please add paramster -schema=schemaname "); return; } else { schemaConfig = config.getSchemas().get(schemaName); if(schemaConfig == null){ c.writeErrMessage(ErrorCode.ER_UNKNOWN_ERROR, "schemaName is null, please add paramster -schema=schemaname "); } } if(StringUtil.isEmpty(tableName)) { c.writeErrMessage(ErrorCode.ER_UNKNOWN_ERROR, "tableName is null, please add paramster -table=tablename "); return; } else { table = schemaConfig.getTables().get(tableName.toUpperCase()); } if(StringUtil.isEmpty(retryTimeStr)) { c.writeErrMessage(ErrorCode.ER_UNKNOWN_ERROR, "retryTime is null, please add paramster -retry= "); return; }else { retryTime = Integer.valueOf(retryTimeStr); } if(StringUtil.isEmpty(intervalTimeStr)) { c.writeErrMessage(ErrorCode.ER_BAD_TABLE_ERROR, "intervalTime is null, please add paramster -interval= "); return; } else { intervalTime = Long.valueOf(intervalTimeStr); } // tableName = "e_account_subject"; // schemaName = "TESTDB"; List dataNodeList = table.getDataNodes(); ConsistenCollectHandler cHandler = new ConsistenCollectHandler( c, tableName, schemaName, dataNodeList.size(), retryTime, intervalTime); cHandler.startDetector(); //c.writeErrMessage(ErrorCode.ER_BAD_TABLE_ERROR, "XXX"); } private static RowDataPacket getRow(int i, String tableName, String result,String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(LongUtil.toBytes(i)); row.add(StringUtil.encode(tableName, charset)); row.add(StringUtil.encode(result, charset)); return row; } public static void response(ManagerConnection c,String tableName, String result) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; RowDataPacket row = getRow(1, tableName, result, c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c, true); // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } public static void main(String[] args) { // show @@check_global --schema=TESTDB -table=e_account_subject -retry=40 -interval=20; Map params = CheckGlobalTable.parse("show @@CKECK_GLOBAL -SCHEMA=TESTDB -TABLE=E_ACCOUNT_SUBJECT -retrytime=2" + " -intervaltime=20"); System.out.println(params); } } ================================================ FILE: src/main/java/io/mycat/manager/response/ClearSlow.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.config.ErrorCode; import io.mycat.config.MycatConfig; import io.mycat.config.model.SchemaConfig; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.OkPacket; /** * @author mycat */ public class ClearSlow { public static void dataNode(ManagerConnection c, String name) { PhysicalDBNode dn = MycatServer.getInstance().getConfig().getDataNodes().get(name); PhysicalDBPool ds = null; if (dn != null && ((ds = dn.getDbPool())!= null)) { // ds.getSqlRecorder().clear(); c.write(c.writeToBuffer(OkPacket.OK, c.allocate())); } else { c.writeErrMessage(ErrorCode.ER_YES, "Invalid DataNode:" + name); } } public static void schema(ManagerConnection c, String name) { MycatConfig conf = MycatServer.getInstance().getConfig(); SchemaConfig schema = conf.getSchemas().get(name); if (schema != null) { // Map dataNodes = conf.getDataNodes(); // for (String n : schema.getAllDataNodes()) { // MySQLDataNode dn = dataNodes.get(n); // MySQLDataSource ds = null; // if (dn != null && (ds = dn.getSource()) != null) { // ds.getSqlRecorder().clear(); // } // } c.write(c.writeToBuffer(OkPacket.OK, c.allocate())); } else { c.writeErrMessage(ErrorCode.ER_YES, "Invalid Schema:" + name); } } } ================================================ FILE: src/main/java/io/mycat/manager/response/KillConnection.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.util.ArrayList; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.manager.ManagerConnection; import io.mycat.net.FrontendConnection; import io.mycat.net.NIOConnection; import io.mycat.net.NIOProcessor; import io.mycat.net.mysql.OkPacket; import io.mycat.util.SplitUtil; /** * @author mycat */ public final class KillConnection { private static final Logger logger = LoggerFactory.getLogger(KillConnection.class); public static void response(String stmt, int offset, ManagerConnection mc) { int count = 0; List list = getList(stmt, offset, mc); if (list != null) { for (NIOConnection c : list) { StringBuilder s = new StringBuilder(); logger.warn(s.append(c).append("killed by manager").toString()); c.close("kill by manager"); count++; } } OkPacket packet = new OkPacket(); packet.packetId = 1; packet.affectedRows = count; packet.serverStatus = 2; packet.write(mc); } private static List getList(String stmt, int offset, ManagerConnection mc) { String ids = stmt.substring(offset).trim(); if (ids.length() > 0) { String[] idList = SplitUtil.split(ids, ',', true); List fcList = new ArrayList(idList.length); NIOProcessor[] processors = MycatServer.getInstance().getProcessors(); for (String id : idList) { long value = 0; try { value = Long.parseLong(id); } catch (NumberFormatException e) { continue; } FrontendConnection fc = null; for (NIOProcessor p : processors) { if ((fc = p.getFrontends().get(value)) != null) { fcList.add(fc); break; } } } return fcList; } return null; } } ================================================ FILE: src/main/java/io/mycat/manager/response/Offline.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import io.mycat.MycatServer; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.OkPacket; /** * @author mycat */ public class Offline { private static final OkPacket ok = new OkPacket(); static { ok.packetId = 1; ok.affectedRows = 1; ok.serverStatus = 2; } public static void execute(String stmt, ManagerConnection c) { MycatServer.getInstance().offline(); ok.write(c); } } ================================================ FILE: src/main/java/io/mycat/manager/response/Online.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import io.mycat.MycatServer; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.OkPacket; /** * @author mycat */ public class Online { private static final OkPacket ok = new OkPacket(); static { ok.packetId = 1; ok.affectedRows = 1; ok.serverStatus = 2; } public static void execute(String stmt, ManagerConnection mc) { MycatServer.getInstance().online(); ok.write(mc); } } ================================================ FILE: src/main/java/io/mycat/manager/response/ReloadConfig.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.util.ArrayList; import java.util.Iterator; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import io.mycat.MycatServer; import io.mycat.backend.BackendConnection; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.jdbc.JDBCConnection; import io.mycat.backend.mysql.nio.MySQLConnection; import io.mycat.config.ConfigInitializer; import io.mycat.config.ErrorCode; import io.mycat.config.MycatCluster; import io.mycat.config.MycatConfig; import io.mycat.config.model.FirewallConfig; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.UserConfig; import io.mycat.config.util.DnPropertyUtil; import io.mycat.manager.ManagerConnection; import io.mycat.net.NIOProcessor; import io.mycat.net.mysql.OkPacket; /** * @author mycat * @author zhuam */ public final class ReloadConfig { private static final Logger LOGGER = LoggerFactory.getLogger(ReloadConfig.class); public static void execute(ManagerConnection c, final boolean loadAll) { // reload @@config_all 校验前一次的事务完成情况 if ( loadAll && !NIOProcessor.backends_old.isEmpty() ) { c.writeErrMessage(ErrorCode.ER_YES, "The are several unfinished db transactions before executing \"reload @@config_all\", therefore the execution is terminated for logical integrity and please try again later."); return; } final ReentrantLock lock = MycatServer.getInstance().getConfig().getLock(); lock.lock(); try { ListenableFuture listenableFuture = MycatServer.getInstance().getListeningExecutorService().submit( new Callable() { @Override public Boolean call() throws Exception { return loadAll ? reload_all() : reload(); } } ); Futures.addCallback(listenableFuture, new ReloadCallBack(c), MycatServer.getInstance().getListeningExecutorService()); } finally { lock.unlock(); } } public static boolean reload_all() { /** * 1、载入新的配置 * 1.1、ConfigInitializer 初始化，基本自检 * 1.2、DataNode/DataHost 实际链路检测 */ ConfigInitializer loader = new ConfigInitializer(true); Map newUsers = loader.getUsers(); Map newSchemas = loader.getSchemas(); Map newDataNodes = loader.getDataNodes(); Map newDataHosts = loader.getDataHosts(); MycatCluster newCluster = loader.getCluster(); FirewallConfig newFirewall = loader.getFirewall(); /** * 1.2、实际链路检测 */ loader.testConnection(); /** * 2、承接 * 2.1、老的 dataSource 继续承接新建请求 * 2.2、新的 dataSource 开始初始化， 完毕后交由 2.3 * 2.3、新的 dataSource 开始承接新建请求 * 2.4、老的 dataSource 内部的事务执行完毕， 相继关闭 * 2.5、老的 dataSource 超过阀值的，强制关闭 */ MycatConfig config = MycatServer.getInstance().getConfig(); /** * 2.1 、老的 dataSource 继续承接新建请求， 此处什么也不需要做 */ boolean isReloadStatusOK = true; /** * 2.2、新的 dataHosts 初始化 */ for (PhysicalDBPool dbPool : newDataHosts.values()) { String hostName = dbPool.getHostName(); // 设置 schemas ArrayList dnSchemas = new ArrayList(30); for (PhysicalDBNode dn : newDataNodes.values()) { if (dn.getDbPool().getHostName().equals(hostName)) { dnSchemas.add(dn.getDatabase()); } } dbPool.setSchemas( dnSchemas.toArray(new String[dnSchemas.size()]) ); // 获取 data host String dnIndex = DnPropertyUtil.loadDnIndexProps().getProperty(dbPool.getHostName(), "0"); if ( !"0".equals(dnIndex) ) { LOGGER.info("init datahost: " + dbPool.getHostName() + " to use datasource index:" + dnIndex); } dbPool.init( Integer.valueOf(dnIndex) ); if ( !dbPool.isInitSuccess() ) { isReloadStatusOK = false; break; } } /** * TODO： 确认初始化情况 * * 新的 dataHosts 是否初始化成功 */ if ( isReloadStatusOK ) { config.getSystem().setUseSqlStat(loader.getSystem().getUseSqlStat()); MycatServer.getInstance().ensureSqlstatRecycleFuture(); /** * 2.3、 在老的配置上，应用新的配置，开始准备承接任务 */ config.reload(newUsers, newSchemas, newDataNodes, newDataHosts, newCluster, newFirewall, true); /** * 2.4、 处理旧的资源 */ LOGGER.warn("1、clear old backend connection(size): " + NIOProcessor.backends_old.size()); // 清除前一次 reload 转移出去的 old Cons Iterator iter = NIOProcessor.backends_old.iterator(); while( iter.hasNext() ) { BackendConnection con = iter.next(); con.close("clear old datasources"); iter.remove(); } Map oldDataHosts = config.getBackupDataHosts(); for (PhysicalDBPool dbPool : oldDataHosts.values()) { dbPool.stopHeartbeat(); // 提取数据源下的所有连接 for (PhysicalDatasource ds : dbPool.getAllDataSources()) { // for (NIOProcessor processor : MycatServer.getInstance().getProcessors()) { for (BackendConnection con : processor.getBackends().values()) { if (con instanceof MySQLConnection) { MySQLConnection mysqlCon = (MySQLConnection) con; if ( mysqlCon.getPool() == ds) { NIOProcessor.backends_old.add( con ); } } else if (con instanceof JDBCConnection) { JDBCConnection jdbcCon = (JDBCConnection) con; if (jdbcCon.getPool() == ds) { NIOProcessor.backends_old.add( con ); } } } } } } LOGGER.warn("2、to be recycled old backend connection(size): " + NIOProcessor.backends_old.size()); //清理缓存 MycatServer.getInstance().getCacheService().clearCache(); MycatServer.getInstance().initRuleData(); return true; } else { // 如果重载不成功，则清理已初始化的资源。 LOGGER.warn("reload failed, clear previously created datasources "); for (PhysicalDBPool dbPool : newDataHosts.values()) { dbPool.clearDataSources("reload config"); dbPool.stopHeartbeat(); } return false; } } public static boolean reload() { /** * 1、载入新的配置， ConfigInitializer 内部完成自检工作, 由于不更新数据源信息,此处不自检 dataHost dataNode */ ConfigInitializer loader = new ConfigInitializer(false); Map users = loader.getUsers(); Map schemas = loader.getSchemas(); Map dataNodes = loader.getDataNodes(); Map dataHosts = loader.getDataHosts(); MycatCluster cluster = loader.getCluster(); FirewallConfig firewall = loader.getFirewall(); MycatServer.getInstance().getConfig().getSystem().setUseSqlStat(loader.getSystem().getUseSqlStat()); MycatServer.getInstance().ensureSqlstatRecycleFuture(); /** * 2、在老的配置上，应用新的配置 */ MycatServer.getInstance().getConfig().reload(users, schemas, dataNodes, dataHosts, cluster, firewall, false); /** * 3、清理缓存 */ MycatServer.getInstance().getCacheService().clearCache(); MycatServer.getInstance().initRuleData(); return true; } /** * 异步执行回调类，用于回写数据给用户等。 */ private static class ReloadCallBack implements FutureCallback { private ManagerConnection mc; private ReloadCallBack(ManagerConnection c) { this.mc = c; } @Override public void onSuccess(Boolean result) { if (result) { LOGGER.warn("send ok package to client " + String.valueOf(mc)); OkPacket ok = new OkPacket(); ok.packetId = 1; ok.affectedRows = 1; ok.serverStatus = 2; ok.message = "Reload config success".getBytes(); ok.write(mc); } else { mc.writeErrMessage(ErrorCode.ER_YES, "Reload config failure"); } } @Override public void onFailure(Throwable t) { mc.writeErrMessage(ErrorCode.ER_YES, "Reload config failure"); } } } ================================================ FILE: src/main/java/io/mycat/manager/response/ReloadQueryCf.java ================================================ package io.mycat.manager.response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.OkPacket; import io.mycat.statistic.stat.QueryConditionAnalyzer; public class ReloadQueryCf { private static final Logger logger = LoggerFactory.getLogger(ReloadSqlSlowTime.class); public static void execute(ManagerConnection c, String cf) { if ( cf == null ) { cf = "NULL"; } QueryConditionAnalyzer.getInstance().setCf(cf); StringBuilder s = new StringBuilder(); s.append(c).append("Reset show @@sql.condition="+ cf +" success by manager"); logger.warn(s.toString()); OkPacket ok = new OkPacket(); ok.packetId = 1; ok.affectedRows = 1; ok.serverStatus = 2; ok.message = "Reset show @@sql.condition success".getBytes(); ok.write(c); System.out.println(s.toString()); } } ================================================ FILE: src/main/java/io/mycat/manager/response/ReloadSqlSlowTime.java ================================================ package io.mycat.manager.response; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.OkPacket; import io.mycat.statistic.stat.UserStat; import io.mycat.statistic.stat.UserStatAnalyzer; public class ReloadSqlSlowTime { private static final Logger logger = LoggerFactory.getLogger(ReloadSqlSlowTime.class); public static void execute(ManagerConnection c,long time) { Map statMap = UserStatAnalyzer.getInstance().getUserStatMap(); for (UserStat userStat : statMap.values()) { userStat.setSlowTime(time); } StringBuilder s = new StringBuilder(); s.append(c).append("Reset show @@sql.slow="+time+" time success by manager"); logger.warn(s.toString()); OkPacket ok = new OkPacket(); ok.packetId = 1; ok.affectedRows = 1; ok.serverStatus = 2; ok.message = "Reset show @@sql.slow time success".getBytes(); ok.write(c); System.out.println(s.toString()); } } ================================================ FILE: src/main/java/io/mycat/manager/response/ReloadSqlStat.java ================================================ package io.mycat.manager.response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.config.ErrorCode; import io.mycat.config.model.SystemConfig; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.OkPacket; public class ReloadSqlStat { private static final Logger logger = LoggerFactory.getLogger(ReloadSqlStat.class); public static void execute(ManagerConnection c, String openCloseFlag) { SystemConfig system = MycatServer.getInstance().getConfig().getSystem(); int oldStat = system.getUseSqlStat(); int newStat = oldStat; if ("open".equalsIgnoreCase(openCloseFlag)) { newStat = 1; } else if ("close".equalsIgnoreCase(openCloseFlag)) { newStat = 0; } else { c.writeErrMessage(ErrorCode.ER_YES, "reload @@sqlstat=open|close"); return; } system.setUseSqlStat(newStat); MycatServer.getInstance().ensureSqlstatRecycleFuture(); StringBuilder s = new StringBuilder(); s.append(c).append("Reset @@sqlstat=" + openCloseFlag + " success by manager"); logger.warn(s.toString()); OkPacket ok = new OkPacket(); ok.packetId = 1; ok.affectedRows = oldStat != newStat ? 1 : 0; ok.serverStatus = 2; ok.message = "Reset @@sqlstat success".getBytes(); ok.write(c); System.out.println(s.toString()); } } ================================================ FILE: src/main/java/io/mycat/manager/response/ReloadUser.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.config.ErrorCode; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.OkPacket; /** * @author mycat */ public final class ReloadUser { private static final Logger logger = LoggerFactory.getLogger(ReloadUser.class); public static void execute(ManagerConnection c) { boolean status = false; if (status) { StringBuilder s = new StringBuilder(); s.append(c).append("Reload userConfig success by manager"); logger.warn(s.toString()); OkPacket ok = new OkPacket(); ok.packetId = 1; ok.affectedRows = 1; ok.serverStatus = 2; ok.message = "Reload userConfig success".getBytes(); ok.write(c); } else { c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement"); } } } ================================================ FILE: src/main/java/io/mycat/manager/response/ReloadUserStat.java ================================================ package io.mycat.manager.response; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.OkPacket; import io.mycat.statistic.stat.UserStat; import io.mycat.statistic.stat.UserStatAnalyzer; public final class ReloadUserStat { private static final Logger logger = LoggerFactory.getLogger(ReloadUserStat.class); public static void execute(ManagerConnection c) { Map statMap = UserStatAnalyzer.getInstance().getUserStatMap(); for (UserStat userStat : statMap.values()) { userStat.reset(); } StringBuilder s = new StringBuilder(); s.append(c).append("Reset show @@sql @@sql.sum @@sql.slow success by manager"); logger.warn(s.toString()); OkPacket ok = new OkPacket(); ok.packetId = 1; ok.affectedRows = 1; ok.serverStatus = 2; ok.message = "Reset show @@sql @@sql.sum @@sql.slow success".getBytes(); ok.write(c); } } ================================================ FILE: src/main/java/io/mycat/manager/response/ReloadZktoXml.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.StringUtil; /** * 进行reload_zk操作的响应 * * @author mycat * @author mycat */ public final class ReloadZktoXml { private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("STATEMENT", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c, String rsp) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c, true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c, true); } // write eof buffer = eof.write(buffer, c, true); // write rows byte packetId = eof.packetId; RowDataPacket row = getRow(rsp, c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c, true); // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c, true); // post write c.write(buffer); } private static RowDataPacket getRow(String stmt, String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(stmt, charset)); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/RollbackConfig.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.util.Map; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.config.ErrorCode; import io.mycat.config.MycatCluster; import io.mycat.config.MycatConfig; import io.mycat.config.model.FirewallConfig; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.UserConfig; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.OkPacket; /** * @author mycat */ public final class RollbackConfig { private static final Logger LOGGER = LoggerFactory.getLogger(RollbackConfig.class); public static void execute(ManagerConnection c) { final ReentrantLock lock = MycatServer.getInstance().getConfig() .getLock(); lock.lock(); try { if (rollback()) { StringBuilder s = new StringBuilder(); s.append(c).append("Rollback config success by manager"); LOGGER.warn(s.toString()); OkPacket ok = new OkPacket(); ok.packetId = 1; ok.affectedRows = 1; ok.serverStatus = 2; ok.message = "Rollback config success".getBytes(); ok.write(c); } else { c.writeErrMessage(ErrorCode.ER_YES, "Rollback config failure"); } } finally { lock.unlock(); } } private static boolean rollback() { MycatConfig conf = MycatServer.getInstance().getConfig(); Map users = conf.getBackupUsers(); Map schemas = conf.getBackupSchemas(); Map dataNodes = conf.getBackupDataNodes(); Map dataHosts = conf.getBackupDataHosts(); MycatCluster cluster = conf.getBackupCluster(); FirewallConfig firewall = conf.getBackupFirewall(); // 检查可回滚状态 if (!conf.canRollback()) { return false; } // 如果回滚已经存在的pool boolean rollbackStatus = true; Map cNodes = conf.getDataHosts(); for (PhysicalDBPool dn : dataHosts.values()) { dn.init(dn.getActivedIndex()); if (!dn.isInitSuccess()) { rollbackStatus = false; break; } } // 如果回滚不成功，则清理已初始化的资源。 if (!rollbackStatus) { for (PhysicalDBPool dn : dataHosts.values()) { dn.clearDataSources("rollbackup config"); dn.stopHeartbeat(); } return false; } // 应用回滚 conf.rollback(users, schemas, dataNodes, dataHosts, cluster, firewall); // 处理旧的资源 for (PhysicalDBPool dn : cNodes.values()) { dn.clearDataSources("clear old config "); dn.stopHeartbeat(); } //清理缓存 MycatServer.getInstance().getCacheService().clearCache(); return true; } } ================================================ FILE: src/main/java/io/mycat/manager/response/RollbackUser.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.config.ErrorCode; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.OkPacket; /** * @author mycat */ public final class RollbackUser { private static final Logger logger = LoggerFactory.getLogger(RollbackUser.class); public static void execute(ManagerConnection c) { boolean status = false; if (status) { StringBuilder s = new StringBuilder(); s.append(c).append("Rollback user success by manager"); logger.warn(s.toString()); OkPacket ok = new OkPacket(); ok.packetId = 1; ok.affectedRows = 1; ok.serverStatus = 2; ok.message = "Rollback user success".getBytes(); ok.write(c); } else { c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement"); } } } ================================================ FILE: src/main/java/io/mycat/manager/response/SelectSessionAutoIncrement.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.LongUtil; /** * @author mycat */ public final class SelectSessionAutoIncrement { private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("SESSION.AUTOINCREMENT", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.packetId = ++packetId; row.add(LongUtil.toBytes(1)); buffer = row.write(buffer, c,true); // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } } ================================================ FILE: src/main/java/io/mycat/manager/response/SelectSessionTxReadOnly.java ================================================ package io.mycat.manager.response; import java.nio.ByteBuffer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.LongUtil; public final class SelectSessionTxReadOnly { private static final String SESSION_TX_READ_ONLY = "@@SESSION.TX_READ_ONLY"; private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField(SESSION_TX_READ_ONLY, Fields.FIELD_TYPE_INT24); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.packetId = ++packetId; row.add(LongUtil.toBytes(0)); buffer = row.write(buffer, c,true); // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } } ================================================ FILE: src/main/java/io/mycat/manager/response/SelectVersionComment.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; /** * @author mycat */ public final class SelectVersionComment { private static final byte[] VERSION_COMMENT = "MyCat Server (monitor)".getBytes(); private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("@@VERSION_COMMENT", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(VERSION_COMMENT); row.packetId = ++packetId; buffer = row.write(buffer, c,true); // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowBackend.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import io.mycat.MycatServer; import io.mycat.backend.BackendConnection; import io.mycat.backend.jdbc.JDBCConnection; import io.mycat.backend.mysql.PacketUtil; import io.mycat.backend.mysql.nio.MySQLConnection; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.BackendAIOConnection; import io.mycat.net.NIOProcessor; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.IntegerUtil; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; import io.mycat.util.TimeUtil; /** * 查询后端连接 * * @author mycat */ public class ShowBackend { private static final int FIELD_COUNT = 17; private static final ResultSetHeaderPacket header = PacketUtil .getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("processor", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("id", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("mysqlId", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("host", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("port", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("l_port", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("net_in", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("net_out", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("life", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("closed", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; // fields[i] = PacketUtil.getField("run", Fields.FIELD_TYPE_VAR_STRING); // fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("borrowed", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("SEND_QUEUE", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("schema", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil .getField("charset", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil .getField("txlevel", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("autocommit", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("tx_readonly", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); buffer = header.write(buffer, c, true); for (FieldPacket field : fields) { buffer = field.write(buffer, c, true); } buffer = eof.write(buffer, c, true); byte packetId = eof.packetId; String charset = c.getCharset(); for (NIOProcessor p : MycatServer.getInstance().getProcessors()) { for (BackendConnection bc : p.getBackends().values()) { if (bc != null) { RowDataPacket row = getRow(bc, charset); row.packetId = ++packetId; buffer = row.write(buffer, c, true); } } } EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c, true); c.write(buffer); } private static RowDataPacket getRow(BackendConnection c, String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); if (c instanceof BackendAIOConnection) { row.add(((BackendAIOConnection) c).getProcessor().getName() .getBytes()); } else if(c instanceof JDBCConnection){ row.add(((JDBCConnection)c).getProcessor().getName().getBytes()); }else{ row.add("N/A".getBytes()); } row.add(LongUtil.toBytes(c.getId())); long threadId = 0; if (c instanceof MySQLConnection) { threadId = ((MySQLConnection) c).getThreadId(); } row.add(LongUtil.toBytes(threadId)); row.add(StringUtil.encode(c.getHost(), charset)); row.add(IntegerUtil.toBytes(c.getPort())); row.add(IntegerUtil.toBytes(c.getLocalPort())); row.add(LongUtil.toBytes(c.getNetInBytes())); row.add(LongUtil.toBytes(c.getNetOutBytes())); row.add(LongUtil.toBytes((TimeUtil.currentTimeMillis() - c .getStartupTime()) / 1000L)); row.add(c.isClosed() ? "true".getBytes() : "false".getBytes()); // boolean isRunning = c.isRunning(); // row.add(isRunning ? "true".getBytes() : "false".getBytes()); boolean isBorrowed = c.isBorrowed(); row.add(isBorrowed ? "true".getBytes() : "false".getBytes()); int writeQueueSize = 0; String schema = ""; String charsetInf = ""; String txLevel = ""; String txAutommit = ""; String txReadonly = ""; if (c instanceof MySQLConnection) { MySQLConnection mysqlC = (MySQLConnection) c; writeQueueSize = mysqlC.getWriteQueue().size(); schema = mysqlC.getSchema(); charsetInf = mysqlC.getCharset() + ":" + mysqlC.getCharsetIndex(); txLevel = mysqlC.getTxIsolation() + ""; txAutommit = mysqlC.isAutocommit() + ""; txReadonly = mysqlC.isTxReadonly() + ""; } row.add(IntegerUtil.toBytes(writeQueueSize)); row.add(schema.getBytes()); row.add(charsetInf.getBytes()); row.add(txLevel.getBytes()); row.add(txAutommit.getBytes()); row.add(txReadonly.getBytes()); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowBackendOld.java ================================================ package io.mycat.manager.response; import java.nio.ByteBuffer; import io.mycat.backend.BackendConnection; import io.mycat.backend.mysql.PacketUtil; import io.mycat.backend.mysql.nio.MySQLConnection; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.NIOProcessor; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.IntegerUtil; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; import io.mycat.util.TimeUtil; /** * 查询 reload @@config_all 后产生的后端连接（待回收） * * @author zhuam */ public class ShowBackendOld { private static final int FIELD_COUNT = 10; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("id", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("mysqlId", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("host", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("port", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("l_port", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("net_in", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("net_out", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("life", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("lasttime", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("borrowed",Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); buffer = header.write(buffer, c, true); for (FieldPacket field : fields) { buffer = field.write(buffer, c, true); } buffer = eof.write(buffer, c, true); byte packetId = eof.packetId; String charset = c.getCharset(); for (BackendConnection bc : NIOProcessor.backends_old) { if ( bc != null) { RowDataPacket row = getRow(bc, charset); row.packetId = ++packetId; buffer = row.write(buffer, c, true); } } EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c, true); c.write(buffer); } private static RowDataPacket getRow(BackendConnection c, String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(LongUtil.toBytes(c.getId())); long threadId = 0; if (c instanceof MySQLConnection) { threadId = ((MySQLConnection) c).getThreadId(); } row.add(LongUtil.toBytes(threadId)); row.add(StringUtil.encode(c.getHost(), charset)); row.add(IntegerUtil.toBytes(c.getPort())); row.add(IntegerUtil.toBytes(c.getLocalPort())); row.add(LongUtil.toBytes(c.getNetInBytes())); row.add(LongUtil.toBytes(c.getNetOutBytes())); row.add(LongUtil.toBytes((TimeUtil.currentTimeMillis() - c.getStartupTime()) / 1000L)); row.add(LongUtil.toBytes( c.getLastTime() )); boolean isBorrowed = c.isBorrowed(); row.add(isBorrowed ? "true".getBytes() : "false".getBytes()); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowCollation.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.IntegerUtil; import io.mycat.util.LongUtil; /** * @author mycat * @author mycat */ public final class ShowCollation { private static final int FIELD_COUNT = 6; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("COLLATION", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("CHARSET", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("ID", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("DEFAULT", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("COMPILED", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("SORTLEN", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; RowDataPacket row = getRow(c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c,true); // write lastEof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } private static RowDataPacket getRow(String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add("utf8_general_ci".getBytes()); row.add("utf8".getBytes()); row.add(IntegerUtil.toBytes(33)); row.add("Yes".getBytes()); row.add("Yes".getBytes()); row.add(LongUtil.toBytes(1)); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowCommand.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.NIOProcessor; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.statistic.CommandCount; import io.mycat.util.LongUtil; /** * 统计各类数据包的执行次数 * * @author mycat * @author mycat */ public final class ShowCommand { private static final int FIELD_COUNT = 10; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("PROCESSOR", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("INIT_DB", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("QUERY", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("STMT_PREPARE", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("STMT_EXECUTE", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("STMT_CLOSE", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("PING", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("KILL", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("QUIT", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("OTHER", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; for (NIOProcessor p : MycatServer.getInstance().getProcessors()) { RowDataPacket row = getRow(p, c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } private static RowDataPacket getRow(NIOProcessor processor, String charset) { CommandCount cc = processor.getCommands(); RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(processor.getName().getBytes()); row.add(LongUtil.toBytes(cc.initDBCount())); row.add(LongUtil.toBytes(cc.queryCount())); row.add(LongUtil.toBytes(cc.stmtPrepareCount())); row.add(LongUtil.toBytes(cc.stmtExecuteCount())); row.add(LongUtil.toBytes(cc.stmtCloseCount())); row.add(LongUtil.toBytes(cc.pingCount())); row.add(LongUtil.toBytes(cc.killCount())); row.add(LongUtil.toBytes(cc.quitCount())); row.add(LongUtil.toBytes(cc.otherCount())); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowConnection.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.FrontendConnection; import io.mycat.net.NIOProcessor; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.ServerConnection; import io.mycat.util.IntegerUtil; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; import io.mycat.util.TimeUtil; /** * 查看当前有效连接信息 * * @author mycat * @author mycat */ public final class ShowConnection { private static final int FIELD_COUNT = 15; private static final ResultSetHeaderPacket header = PacketUtil .getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("PROCESSOR", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("ID", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("HOST", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("PORT", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("LOCAL_PORT", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("USER", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("SCHEMA", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil .getField("CHARSET", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("NET_IN", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("NET_OUT", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("ALIVE_TIME(S)", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("RECV_BUFFER", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("SEND_QUEUE", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil .getField("txlevel", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("autocommit", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c, true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c, true); } // write eof buffer = eof.write(buffer, c, true); // write rows byte packetId = eof.packetId; String charset = c.getCharset(); NIOProcessor[] processors = MycatServer.getInstance().getProcessors(); for (NIOProcessor p : processors) { for (FrontendConnection fc : p.getFrontends().values()) { if (fc != null) { RowDataPacket row = getRow(fc, charset); row.packetId = ++packetId; buffer = row.write(buffer, c, true); } } } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c, true); // write buffer c.write(buffer); } private static RowDataPacket getRow(FrontendConnection c, String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(c.getProcessor().getName().getBytes()); row.add(LongUtil.toBytes(c.getId())); row.add(StringUtil.encode(c.getHost(), charset)); row.add(IntegerUtil.toBytes(c.getPort())); row.add(IntegerUtil.toBytes(c.getLocalPort())); row.add(StringUtil.encode(c.getUser(), charset)); row.add(StringUtil.encode(c.getSchema(), charset)); row.add(StringUtil.encode(c.getCharset()+":"+c.getCharsetIndex(), charset)); row.add(LongUtil.toBytes(c.getNetInBytes())); row.add(LongUtil.toBytes(c.getNetOutBytes())); row.add(LongUtil.toBytes((TimeUtil.currentTimeMillis() - c.getStartupTime()) / 1000L)); ByteBuffer bb = c.getReadBuffer(); row.add(IntegerUtil.toBytes(bb == null ? 0 : bb.capacity())); row.add(IntegerUtil.toBytes(c.getWriteQueue().size())); String txLevel = ""; String txAutommit = ""; if (c instanceof ServerConnection) { ServerConnection mysqlC = (ServerConnection) c; txLevel = mysqlC.getTxIsolation() + ""; txAutommit = mysqlC.isAutocommit() + ""; } row.add(txLevel.getBytes()); row.add(txAutommit.getBytes()); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowConnectionSQL.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.FrontendConnection; import io.mycat.net.NIOProcessor; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.ServerConnection; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; import io.mycat.util.TimeUtil; /** * @author mycat */ public final class ShowConnectionSQL { private static final int FIELD_COUNT = 7; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("ID", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("HOST", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("USER", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("SCHEMA", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("START_TIME", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("EXECUTE_TIME", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("SQL", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; String charset = c.getCharset(); for (NIOProcessor p : MycatServer.getInstance().getProcessors()) { for (FrontendConnection fc : p.getFrontends().values()) { if (!fc.isClosed()) { if(fc.getExecuteSql()==null){ continue; } if(fc instanceof ServerConnection){ RowDataPacket row = getRow(fc, charset); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } } } } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } private static RowDataPacket getRow(FrontendConnection c, String charset) { String executeSql = c.getExecuteSql(); RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(LongUtil.toBytes(c.getId())); row.add(StringUtil.encode(c.getHost(), charset)); row.add(StringUtil.encode(c.getUser(), charset)); row.add(StringUtil.encode(c.getSchema(), charset)); row.add(LongUtil.toBytes(c.getLastReadTime())); long rt = c.getLastReadTime(); long wt = c.getLastWriteTime(); row.add(LongUtil.toBytes(executeSql==null?0:((wt > rt) ? (wt - rt) : (TimeUtil.currentTimeMillis() - rt)))); row.add(StringUtil.encode(executeSql==null?"":executeSql, charset) ); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowDataNode.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.text.DecimalFormat; import java.text.NumberFormat; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.config.MycatConfig; import io.mycat.config.MycatPrivileges; import io.mycat.config.model.SchemaConfig; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.route.parser.util.Pair; import io.mycat.route.parser.util.PairUtil; import io.mycat.util.IntegerUtil; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; import io.mycat.util.TimeUtil; /** * 查看数据节点信息 * * @author mycat * @author mycat */ public final class ShowDataNode { private static final NumberFormat nf = DecimalFormat.getInstance(); private static final int FIELD_COUNT = 12; private static final ResultSetHeaderPacket header = PacketUtil .getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { nf.setMaximumFractionDigits(3); int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("NAME", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil .getField("DATHOST", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("INDEX", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("TYPE", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("ACTIVE", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("IDLE", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("SIZE", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("EXECUTE", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("TOTAL_TIME", Fields.FIELD_TYPE_DOUBLE); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("MAX_TIME", Fields.FIELD_TYPE_DOUBLE); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("MAX_SQL", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("RECOVERY_TIME", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c, String name) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c, true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c, true); } // write eof buffer = eof.write(buffer, c, true); // write rows byte packetId = eof.packetId; MycatConfig conf = MycatServer.getInstance().getConfig(); Map dataNodes = conf.getDataNodes(); List keys = new ArrayList(); if (StringUtil.isEmpty(name)) { for(String key : dataNodes.keySet()){ MycatPrivileges myCatprivileges = (MycatPrivileges)( c.getPrivileges()); if(myCatprivileges.checkDataNodeDmlPrivilege(c.getUser(), key, c.getExecuteSql())) { keys.add(key); } } } else { SchemaConfig sc = conf.getSchemas().get(name); if (null != sc) { keys.addAll(sc.getAllDataNodes()); } } Collections.sort(keys, new Comparators()); for (String key : keys) { RowDataPacket row = getRow(dataNodes.get(key), c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c, true); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c, true); // post write c.write(buffer); } private static RowDataPacket getRow(PhysicalDBNode node, String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(node.getName(), charset)); row.add(StringUtil.encode( node.getDbPool().getHostName() + '/' + node.getDatabase(), charset)); PhysicalDBPool pool = node.getDbPool(); PhysicalDatasource ds = pool.getSource(); if (ds != null) { int active = ds.getActiveCountForSchema(node.getDatabase()); int idle = ds.getIdleCountForSchema(node.getDatabase()); row.add(IntegerUtil.toBytes(pool.getActivedIndex())); row.add(StringUtil.encode(ds.getConfig().getDbType(), charset)); row.add(IntegerUtil.toBytes(active)); row.add(IntegerUtil.toBytes(idle)); row.add(IntegerUtil.toBytes(ds.getSize())); } else { row.add(null); row.add(null); row.add(null); row.add(null); row.add(null); } row.add(LongUtil.toBytes(ds.getExecuteCountForSchema(node.getDatabase()))); row.add(StringUtil.encode(nf.format(0), charset)); row.add(StringUtil.encode(nf.format(0), charset)); row.add(LongUtil.toBytes(0)); long recoveryTime = pool.getSource().getHeartbeatRecoveryTime() - TimeUtil.currentTimeMillis(); row.add(LongUtil.toBytes(recoveryTime > 0 ? recoveryTime / 1000L : -1L)); return row; } private static final class Comparators implements Comparator { @Override public int compare(String s1, String s2) { Pair p1 = PairUtil.splitIndex(s1, '[', ']'); Pair p2 = PairUtil.splitIndex(s2, '[', ']'); if (p1.getKey().compareTo(p2.getKey()) == 0) { return p1.getValue() - p2.getValue(); } else { return p1.getKey().compareTo(p2.getKey()); } } } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowDataSource.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.config.MycatConfig; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.IntegerUtil; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; /** * 查看数据源信息 * * @author mycat * @author mycat */ public final class ShowDataSource { private static final int FIELD_COUNT = 12; private static final ResultSetHeaderPacket header = PacketUtil .getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("DATANODE", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("NAME", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("TYPE", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("HOST", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("PORT", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("W/R", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("ACTIVE", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("IDLE", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("SIZE", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("EXECUTE", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("READ_LOAD", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("WRITE_LOAD", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c, String name) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; MycatConfig conf = MycatServer.getInstance().getConfig(); Map> dataSources = new HashMap>(); if (null != name) { PhysicalDBNode dn = conf.getDataNodes().get(name); if (dn != null) { List dslst = new LinkedList(); dslst.addAll(dn.getDbPool().getAllDataSources()); dataSources.put(dn.getName(), dslst); } } else { // add all for (PhysicalDBNode dn : conf.getDataNodes().values()) { List dslst = new LinkedList(); dslst.addAll(dn.getDbPool().getAllDataSources()); dataSources.put(dn.getName(), dslst); } } for (Map.Entry> dsEntry : dataSources .entrySet()) { String dnName = dsEntry.getKey(); for (PhysicalDatasource ds : dsEntry.getValue()) { RowDataPacket row = getRow(dnName, ds, c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } private static RowDataPacket getRow(String dataNode, PhysicalDatasource ds, String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(dataNode, charset)); row.add(StringUtil.encode(ds.getName(), charset)); row.add(StringUtil.encode(ds.getConfig().getDbType(), charset)); row.add(StringUtil.encode(ds.getConfig().getIp(), charset)); row.add(IntegerUtil.toBytes(ds.getConfig().getPort())); row.add(StringUtil.encode(ds.isReadNode() ? "R" : "W", charset)); row.add(IntegerUtil.toBytes(ds.getActiveCount())); row.add(IntegerUtil.toBytes(ds.getIdleCount())); row.add(IntegerUtil.toBytes(ds.getSize())); row.add(LongUtil.toBytes(ds.getExecuteCount())); row.add(LongUtil.toBytes(ds.getReadCount())); row.add(LongUtil.toBytes(ds.getWriteCount())); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowDatabase.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.util.Map; import java.util.TreeSet; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.config.model.SchemaConfig; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.StringUtil; /** * 查看schema信息 * * @author mycat * @author mycat */ public final class ShowDatabase { private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("DATABASE", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; Map schemas = MycatServer.getInstance().getConfig().getSchemas(); for (String name : new TreeSet(schemas.keySet())) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(name, c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // write lastEof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowDatasourceCluster.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.text.SimpleDateFormat; import java.util.LinkedList; import java.util.List; import java.util.Map; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.heartbeat.DBHeartbeat; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.config.MycatConfig; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.statistic.DataSourceSyncRecorder; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; /** * @author songwie */ public class ShowDatasourceCluster { private static final int FIELD_COUNT = 17; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); private static final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); /*private static final String[] MYSQL_CLUSTER_STAUTS_COLMS = new String[] { "wsrep_incoming_addresses","wsrep_cluster_size","wsrep_cluster_status", "wsrep_connected", "wsrep_flow_control_paused", "wsrep_local_state_comment","wsrep_ready","wsrep_flow_control_paused_ns","wsrep_flow_control_recv","wsrep_local_bf_aborts", "wsrep_local_recv_queue_avg","wsrep_local_send_queue_avg","wsrep_apply_oool","wsrep_apply_oooe"};*/ static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("name", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("host", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("port", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("wsrep_incoming_addresses", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("wsrep_cluster_size", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("wsrep_cluster_status", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("wsrep_connected", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("wsrep_flow_control_paused", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("wsrep_local_state_comment", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("wsrep_ready", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("wsrep_flow_control_paused_ns", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("wsrep_flow_control_recv", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("wsrep_local_bf_aborts", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("wsrep_local_recv_queue_avg", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("wsrep_local_send_queue_avg", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("wsrep_apply_oool", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("wsrep_apply_oooe", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void response(ManagerConnection c,String stmt) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; for (RowDataPacket row : getRows(c.getCharset())) { row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } private static List getRows(String charset) { List list = new LinkedList(); MycatConfig conf = MycatServer.getInstance().getConfig(); // host nodes Map dataHosts = conf.getDataHosts(); for (PhysicalDBPool pool : dataHosts.values()) { for (PhysicalDatasource ds : pool.getAllDataSources()) { DBHeartbeat hb = ds.getHeartbeat(); DataSourceSyncRecorder record = hb.getAsynRecorder(); Map states = record.getRecords(); RowDataPacket row = new RowDataPacket(FIELD_COUNT); if(!states.isEmpty()){ row.add(StringUtil.encode(ds.getName(),charset)); row.add(StringUtil.encode(ds.getConfig().getIp(),charset)); row.add(LongUtil.toBytes(ds.getConfig().getPort())); row.add(StringUtil.encode(states.get("wsrep_incoming_addresses")==null?"":states.get("wsrep_incoming_addresses"),charset)); row.add(StringUtil.encode(states.get("wsrep_cluster_size")==null?"":states.get("wsrep_cluster_size"),charset)); row.add(StringUtil.encode(states.get("wsrep_cluster_status")==null?"":states.get("wsrep_cluster_status"),charset)); row.add(StringUtil.encode(states.get("wsrep_connected")==null?"":states.get("wsrep_connected"),charset)); row.add(StringUtil.encode(states.get("wsrep_flow_control_paused")==null?"":states.get("wsrep_flow_control_paused"),charset)); row.add(StringUtil.encode(states.get("wsrep_local_state_comment")==null?"":states.get("wsrep_local_state_comment"),charset)); row.add(StringUtil.encode(states.get("wsrep_ready")==null?"":states.get("wsrep_ready"),charset)); row.add(StringUtil.encode(states.get("wsrep_flow_control_paused_ns")==null?"":states.get("wsrep_flow_control_paused_ns"),charset)); row.add(StringUtil.encode(states.get("wsrep_flow_control_recv")==null?"":states.get("wsrep_flow_control_recv"),charset)); row.add(StringUtil.encode(states.get("wsrep_local_bf_aborts")==null?"":states.get("wsrep_local_bf_aborts"),charset)); row.add(StringUtil.encode(states.get("wsrep_local_recv_queue_avg")==null?"":states.get("wsrep_local_recv_queue_avg"),charset)); row.add(StringUtil.encode(states.get("wsrep_local_send_queue_avg")==null?"":states.get("wsrep_local_recv_queue_avg"),charset)); row.add(StringUtil.encode(states.get("wsrep_apply_oool")==null?"":states.get("wsrep_apply_oool"),charset)); row.add(StringUtil.encode(states.get("wsrep_apply_oooe")==null?"":states.get("wsrep_apply_oooe"),charset)); list.add(row); } } } return list; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowDatasourceSyn.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.text.SimpleDateFormat; import java.util.LinkedList; import java.util.List; import java.util.Map; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.heartbeat.DBHeartbeat; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.config.MycatConfig; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.statistic.DataSourceSyncRecorder; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; /** * @author songwie */ public class ShowDatasourceSyn { private static final int FIELD_COUNT = 12; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); private static final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("name", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("host", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("port", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("Master_Host", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("Master_Port", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("Master_Use", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("Seconds_Behind_Master", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("Slave_IO_Running", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("Slave_SQL_Running", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("Slave_IO_State", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("Connect_Retry", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("Last_IO_Error", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("Last_SQL_Error", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("Last_SQL_Errno", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void response(ManagerConnection c,String stmt) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; for (RowDataPacket row : getRows(c.getCharset())) { row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } private static List getRows(String charset) { List list = new LinkedList(); MycatConfig conf = MycatServer.getInstance().getConfig(); // host nodes Map dataHosts = conf.getDataHosts(); for (PhysicalDBPool pool : dataHosts.values()) { for (PhysicalDatasource ds : pool.getAllDataSources()) { DBHeartbeat hb = ds.getHeartbeat(); DataSourceSyncRecorder record = hb.getAsynRecorder(); Map states = record.getRecords(); RowDataPacket row = new RowDataPacket(FIELD_COUNT); if(!states.isEmpty()){ row.add(StringUtil.encode(ds.getName(),charset)); row.add(StringUtil.encode(ds.getConfig().getIp(),charset)); row.add(LongUtil.toBytes(ds.getConfig().getPort())); row.add(StringUtil.encode(states.get("Master_Host"),charset)); row.add(LongUtil.toBytes(Long.valueOf(states.get("Master_Port")))); row.add(StringUtil.encode(states.get("Master_Use"),charset)); String secords = states.get("Seconds_Behind_Master"); row.add(secords==null?null:LongUtil.toBytes(Long.valueOf(secords))); row.add(StringUtil.encode(states.get("Slave_IO_Running"),charset)); row.add(StringUtil.encode(states.get("Slave_SQL_Running"),charset)); row.add(StringUtil.encode(states.get("Slave_IO_State"),charset)); row.add(LongUtil.toBytes(Long.valueOf(states.get("Connect_Retry")))); row.add(StringUtil.encode(states.get("Last_IO_Error"),charset)); row.add(StringUtil.encode(states.get("Last_SQL_Error"),charset)); row.add(LongUtil.toBytes(Long.valueOf(states.get("Last_SQL_Errno")))); list.add(row); } } } return list; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowDatasourceSynDetail.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.text.SimpleDateFormat; import java.util.Date; import java.util.LinkedList; import java.util.List; import java.util.Map; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.heartbeat.DBHeartbeat; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.config.MycatConfig; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.route.parser.ManagerParseShow; import io.mycat.statistic.DataSourceSyncRecorder; import io.mycat.statistic.DataSourceSyncRecorder.Record; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; /** * @author songwie */ public class ShowDatasourceSynDetail { private static final int FIELD_COUNT = 8; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("name", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("host", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("port", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("Master_Host", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("Master_Port", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("Master_Use", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("TIME", Fields.FIELD_TYPE_DATETIME); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("Seconds_Behind_Master", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void response(ManagerConnection c,String stmt) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; String name = ManagerParseShow.getWhereParameter(stmt); for (RowDataPacket row : getRows(name,c.getCharset())) { row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } private static List getRows(String name,String charset) { List list = new LinkedList(); MycatConfig conf = MycatServer.getInstance().getConfig(); // host nodes Map dataHosts = conf.getDataHosts(); for (PhysicalDBPool pool : dataHosts.values()) { for (PhysicalDatasource ds : pool.getAllDataSources()) { DBHeartbeat hb = ds.getHeartbeat(); DataSourceSyncRecorder record = hb.getAsynRecorder(); Map states = record.getRecords(); if(name.equals(ds.getName())){ List data = record.getAsynRecords(); for(Record r : data){ RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(ds.getName(),charset)); row.add(StringUtil.encode(ds.getConfig().getIp(),charset)); row.add(LongUtil.toBytes(ds.getConfig().getPort())); row.add(StringUtil.encode(states.get("Master_Host"),charset)); row.add(LongUtil.toBytes(Long.valueOf(states.get("Master_Port")))); row.add(StringUtil.encode(states.get("Master_Use"),charset)); //DateFormat非线程安全 SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); String time = sdf.format(new Date(r.getTime())); row.add(StringUtil.encode(time,charset)); row.add(LongUtil.toBytes((Long)r.getValue())); list.add(row); } break; } } } return list; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowDirectMemory.java ================================================ package io.mycat.manager.response; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.buffer.BufferPool; import io.mycat.buffer.NettyBufferPool; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.memory.MyCatMemory; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.utils.JavaUtils; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.netty.buffer.PoolArenaMetric; import io.netty.buffer.PoolChunkListMetric; import io.netty.buffer.PoolChunkMetric; import io.netty.buffer.PoolSubpageMetric; import sun.rmi.runtime.Log; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; /** * 实现show@@directmemory功能 * * @author zagnix * @version 1.0 * @create 2016-09-21 17:35 */ public class ShowDirectMemory { private static final int DETAILl_FIELD_COUNT = 3; private static final ResultSetHeaderPacket detailHeader = PacketUtil.getHeader(DETAILl_FIELD_COUNT); private static final FieldPacket[] detailFields = new FieldPacket[DETAILl_FIELD_COUNT]; private static final EOFPacket detailEof = new EOFPacket(); private static final int TOTAL_FIELD_COUNT = 5; private static final ResultSetHeaderPacket totalHeader = PacketUtil.getHeader(TOTAL_FIELD_COUNT); private static final FieldPacket[] totalFields = new FieldPacket[TOTAL_FIELD_COUNT]; private static final EOFPacket totalEof = new EOFPacket(); private static int useOffHeapForMerge ; private static int processorBufferPoolType; private static BufferPool bufferPool ; static { int i = 0; byte packetId = 0; detailHeader.packetId = ++packetId; detailFields[i] = PacketUtil.getField("THREAD_ID", Fields.FIELD_TYPE_VAR_STRING); detailFields[i++].packetId = ++packetId; detailFields[i] = PacketUtil.getField("MEM_USE_TYPE", Fields.FIELD_TYPE_VAR_STRING); detailFields[i++].packetId = ++packetId; detailFields[i] = PacketUtil.getField(" SIZE ", Fields.FIELD_TYPE_VAR_STRING); detailFields[i++].packetId = ++packetId; detailEof.packetId = ++packetId; i = 0; packetId = 0; totalHeader.packetId = ++packetId; totalFields[i] = PacketUtil.getField("MDIRECT_MEMORY_MAXED", Fields.FIELD_TYPE_VAR_STRING); totalFields[i++].packetId = ++packetId; totalFields[i] = PacketUtil.getField("DIRECT_MEMORY_USED", Fields.FIELD_TYPE_VAR_STRING); totalFields[i++].packetId = ++packetId; totalFields[i] = PacketUtil.getField("DIRECT_MEMORY_AVAILABLE", Fields.FIELD_TYPE_VAR_STRING); totalFields[i++].packetId = ++packetId; totalFields[i] = PacketUtil.getField("SAFETY_FRACTION", Fields.FIELD_TYPE_VAR_STRING); totalFields[i++].packetId = ++packetId; totalFields[i] = PacketUtil.getField("DIRECT_MEMORY_RESERVED", Fields.FIELD_TYPE_VAR_STRING); totalFields[i++].packetId = ++packetId; totalEof.packetId = ++packetId; } public static void execute(ManagerConnection c, int showtype) { useOffHeapForMerge = MycatServer.getInstance().getConfig(). getSystem().getUseOffHeapForMerge(); processorBufferPoolType = MycatServer.getInstance().getConfig(). getSystem().getProcessorBufferPoolType(); bufferPool = MycatServer.getInstance().getBufferPool(); if (showtype == 1) { showDirectMemoryTotal(c); } else if (showtype == 2) { showDirectMemoryDetail(c); } } public static void showDirectMemoryDetail(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = detailHeader.write(buffer, c, true); // write fields for (FieldPacket field : detailFields) { buffer = field.write(buffer, c, true); } // write eof buffer = detailEof.write(buffer, c, true); // write rows byte packetId = detailEof.packetId; ConcurrentHashMap bufferpoolUsageMap = bufferPool.getNetDirectMemoryUsage(); try { if (useOffHeapForMerge == 1) { ConcurrentHashMap concurrentHashMap = MycatServer.getInstance(). getMyCatMemory(). getResultMergeMemoryManager().getDirectMemorUsage(); for (Long key : concurrentHashMap.keySet()) { RowDataPacket row = new RowDataPacket(DETAILl_FIELD_COUNT); Long value = concurrentHashMap.get(key); row.add(String.valueOf(key).getBytes(c.getCharset())); /** * 该DIRECTMEMORY内存被结果集处理使用了 */ row.add("MergeMemoryPool".getBytes(c.getCharset())); row.add(JavaUtils.bytesToString2(value).getBytes(c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c, true); } } if(processorBufferPoolType == 2){ } else { for (Long key : bufferpoolUsageMap.keySet()) { RowDataPacket row = new RowDataPacket(DETAILl_FIELD_COUNT); Long value = bufferpoolUsageMap.get(key); row.add(String.valueOf(key).getBytes(c.getCharset())); /** * 该DIRECTMEMORY内存属于Buffer Pool管理的！ */ row.add("NetWorkBufferPool".getBytes(c.getCharset())); row.add(JavaUtils.bytesToString2(value).getBytes(c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c, true); } } } catch (UnsupportedEncodingException e) { e.printStackTrace(); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c, true); // write buffer c.write(buffer); } public static void showDirectMemoryTotal(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = totalHeader.write(buffer, c, true); // write fields for (FieldPacket field : totalFields) { buffer = field.write(buffer, c, true); } // write eof buffer = totalEof.write(buffer, c, true); // write rows byte packetId = totalEof.packetId; long usedforMerge = 0; long usedforNetwork = 0; long chunkSizeBytes = 0; int chunkCount = 0; if (processorBufferPoolType == 2 && bufferPool instanceof NettyBufferPool) { /**计算逻辑就是，1.先计算PoolChunk分配的页,表示已经消耗的内存， 2.然后计算小于一页情况，记录小于一页内存使用情况， 上面二者合起来就是整个netty 使用的内存， 已经分配了，但是没有使用的内存的情况*/ List list = ((NettyBufferPool) bufferPool).getAllocator().getAlloc().directArenas(); chunkSizeBytes = ((NettyBufferPool) bufferPool).getAllocator().getChunkSize(); long pageSize = ((NettyBufferPool) bufferPool).getAllocator().getPageSize(); long chunksUsedBytes = 0; /**PoolArenas*/ for (PoolArenaMetric pool : list) { List pcks = pool.chunkLists(); /**针对PoolChunkList*/ for (PoolChunkListMetric pck : pcks) { Iterator it = pck.iterator(); while (it.hasNext()) { chunkCount++; PoolChunkMetric p = it.next(); chunksUsedBytes += (chunkSizeBytes - p.freeBytes()); } } List tinySubpages = pool.tinySubpages(); for (PoolSubpageMetric tiny : tinySubpages) { chunksUsedBytes -= (pageSize - (tiny.maxNumElements() - tiny.numAvailable()) * tiny.elementSize()); } List smallSubpages = pool.smallSubpages(); for (PoolSubpageMetric small : smallSubpages) { chunksUsedBytes -= (pageSize - (small.maxNumElements() - small.numAvailable()) * small.elementSize()); } } usedforNetwork = chunkCount * chunkSizeBytes; } ConcurrentHashMap bufferpoolUsageMap = bufferPool.getNetDirectMemoryUsage(); RowDataPacket row = new RowDataPacket(TOTAL_FIELD_COUNT); try { /** * 通过-XX:MaxDirectMemorySize=2048m设置的值 */ row.add(JavaUtils.bytesToString2(Platform.getMaxDirectMemory()).getBytes(c.getCharset())); if (useOffHeapForMerge == 1) { /** * 结果集合并时，总共消耗的DirectMemory内存 */ ConcurrentHashMap concurrentHashMap = MycatServer.getInstance(). getMyCatMemory(). getResultMergeMemoryManager().getDirectMemorUsage(); for (Map.Entry entry : concurrentHashMap.entrySet()) { usedforMerge += entry.getValue(); } } /** * 网络packet处理，在buffer pool 已经使用DirectMemory内存 */ if (processorBufferPoolType == 2) { usedforNetwork = chunkSizeBytes * chunkCount; } else { for (Map.Entry entry : bufferpoolUsageMap.entrySet()) { usedforNetwork += entry.getValue(); } } row.add(JavaUtils.bytesToString2(usedforMerge + usedforNetwork).getBytes(c.getCharset())); long totalAvailable = 0; if (useOffHeapForMerge == 1) { /** * 设置使用off-heap内存处理结果集时，防止客户把MaxDirectMemorySize设置到物理内存的极限。 * Mycat能使用的DirectMemory是MaxDirectMemorySize*DIRECT_SAFETY_FRACTION大小， * DIRECT_SAFETY_FRACTION为安全系数，为OS，Heap预留空间，避免因大结果集造成系统物理内存被耗尽！ */ totalAvailable = (long) (Platform.getMaxDirectMemory() * MyCatMemory.DIRECT_SAFETY_FRACTION); } else { totalAvailable = Platform.getMaxDirectMemory(); } row.add(JavaUtils.bytesToString2(totalAvailable - usedforMerge - usedforNetwork) .getBytes(c.getCharset())); if (useOffHeapForMerge == 1) { /** * 输出安全系统DIRECT_SAFETY_FRACTION */ row.add(("" + MyCatMemory.DIRECT_SAFETY_FRACTION) .getBytes(c.getCharset())); } else { row.add(("1.0") .getBytes(c.getCharset())); } long resevedForOs = 0; if (useOffHeapForMerge == 1) { /** * 预留OS系统部分内存！！！ */ resevedForOs = (long) ((1 - MyCatMemory.DIRECT_SAFETY_FRACTION) * (Platform.getMaxDirectMemory() - 2 * MycatServer.getInstance().getTotalNetWorkBufferSize())); } row.add(resevedForOs > 0 ? JavaUtils.bytesToString2(resevedForOs).getBytes(c.getCharset()) : "0".getBytes(c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c, true); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c, true); // write buffer c.write(buffer); } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowHeartbeat.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.util.LinkedList; import java.util.List; import java.util.Map; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.heartbeat.DBHeartbeat; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.config.MycatConfig; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.IntegerUtil; import io.mycat.util.LongUtil; /** * @author mycat */ public class ShowHeartbeat { private static final int FIELD_COUNT = 11; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("NAME", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("TYPE", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("HOST", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("PORT", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("RS_CODE", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("RETRY", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("STATUS", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("TIMEOUT", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("EXECUTE_TIME",Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("LAST_ACTIVE_TIME",Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("STOP", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void response(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; for (RowDataPacket row : getRows()) { row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } private static List getRows() { List list = new LinkedList(); MycatConfig conf = MycatServer.getInstance().getConfig(); // host nodes Map dataHosts = conf.getDataHosts(); for (PhysicalDBPool pool : dataHosts.values()) { for (PhysicalDatasource ds : pool.getAllDataSources()) { DBHeartbeat hb = ds.getHeartbeat(); RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(ds.getName().getBytes()); row.add(ds.getConfig().getDbType().getBytes()); if (hb != null) { row.add(ds.getConfig().getIp().getBytes()); row.add(IntegerUtil.toBytes(ds.getConfig().getPort())); row.add(IntegerUtil.toBytes(hb.getStatus())); row.add(IntegerUtil.toBytes(hb.getErrorCount())); row.add(hb.isChecking() ? "checking".getBytes() : "idle".getBytes()); row.add(LongUtil.toBytes(hb.getTimeout())); row.add(hb.getRecorder().get().getBytes()); String lat = hb.getLastActiveTime(); row.add(lat == null ? null : lat.getBytes()); row.add(hb.isStop() ? "true".getBytes() : "false".getBytes()); } else { row.add(null); row.add(null); row.add(null); row.add(null); row.add(null); row.add(null); row.add(null); row.add(null); row.add(null); } list.add(row); } } return list; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowHeartbeatDetail.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.text.SimpleDateFormat; import java.util.Date; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Queue; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.heartbeat.DBHeartbeat; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.config.MycatConfig; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.route.parser.ManagerParseHeartbeat; import io.mycat.route.parser.util.Pair; import io.mycat.statistic.HeartbeatRecorder; import io.mycat.util.IntegerUtil; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; /** * @author songwie */ public class ShowHeartbeatDetail { private static final int FIELD_COUNT = 6; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("NAME", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("TYPE", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("HOST", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("PORT", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("TIME", Fields.FIELD_TYPE_DATETIME); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("EXECUTE_TIME", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void response(ManagerConnection c,String stmt) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; Pair pair = ManagerParseHeartbeat.getPair(stmt); String name = pair.getValue(); for (RowDataPacket row : getRows(name,c.getCharset())) { row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } private static List getRows(String name,String charset) { List list = new LinkedList(); MycatConfig conf = MycatServer.getInstance().getConfig(); // host nodes String type = ""; String ip = ""; int port = 0; DBHeartbeat hb = null; Map dataHosts = conf.getDataHosts(); for (PhysicalDBPool pool : dataHosts.values()) { for (PhysicalDatasource ds : pool.getAllDataSources()) { if(name.equals(ds.getName())){ hb = ds.getHeartbeat(); type = ds.getConfig().getDbType(); ip = ds.getConfig().getIp(); port = ds.getConfig().getPort(); break; } } } if(hb!=null){ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); Queue heatbeartRecorders = hb.getRecorder().getRecordsAll(); for(HeartbeatRecorder.Record record : heatbeartRecorders){ RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(name,charset)); row.add(StringUtil.encode(type,charset)); row.add(StringUtil.encode(ip,charset)); row.add(IntegerUtil.toBytes(port)); long time = record.getTime(); String timeStr = sdf.format(new Date(time)); row.add(StringUtil.encode(timeStr,charset)); row.add(LongUtil.toBytes(record.getValue())); list.add(row); } }else{ RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(null); row.add(null); row.add(null); row.add(null); row.add(null); row.add(null); list.add(row); } return list; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowHelp.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.StringUtil; /** * 打印MycatServer所支持的语句 * * @author mycat * @author mycat */ public final class ShowHelp { private static final int FIELD_COUNT = 2; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("STATEMENT", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("DESCRIPTION", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; for (String key : keys) { RowDataPacket row = getRow(key, helps.get(key), c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } private static RowDataPacket getRow(String stmt, String desc, String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(stmt, charset)); row.add(StringUtil.encode(desc, charset)); return row; } private static final Map helps = new LinkedHashMap(); private static final List keys = new LinkedList(); static { // show helps.put("show @@time.current", "Report current timestamp"); helps.put("show @@time.startup", "Report startup timestamp"); helps.put("show @@version", "Report Mycat Server version"); helps.put("show @@server", "Report server status"); helps.put("show @@threadpool", "Report threadPool status"); helps.put("show @@database", "Report databases"); helps.put("show @@datanode", "Report dataNodes"); helps.put("show @@datanode where schema = ?", "Report dataNodes"); helps.put("show @@datasource", "Report dataSources"); helps.put("show @@datasource where dataNode = ?", "Report dataSources"); helps.put("show @@datasource.synstatus", "Report datasource data synchronous"); helps.put("show @@datasource.syndetail where name=?", "Report datasource data synchronous detail"); helps.put("show @@datasource.cluster", "Report datasource galary cluster variables"); helps.put("show @@processor", "Report processor status"); helps.put("show @@command", "Report commands status"); helps.put("show @@connection", "Report connection status"); helps.put("show @@cache", "Report system cache usage"); helps.put("show @@backend", "Report backend connection status"); helps.put("show @@session", "Report front session details"); helps.put("show @@connection.sql", "Report connection sql"); helps.put("show @@sql.execute", "Report execute status"); helps.put("show @@sql.detail where id = ?", "Report execute detail status"); helps.put("show @@sql", "Report SQL list"); // helps.put("show @@sql where id = ?", "Report specify SQL"); helps.put("show @@sql.high", "Report Hight Frequency SQL"); helps.put("show @@sql.slow", "Report slow SQL"); helps.put("show @@sql.resultset", "Report BIG RESULTSET SQL"); helps.put("show @@sql.sum", "Report User RW Stat "); helps.put("show @@sql.sum.user", "Report User RW Stat "); helps.put("show @@sql.sum.table", "Report Table RW Stat "); helps.put("show @@parser", "Report parser status"); helps.put("show @@router", "Report router status"); helps.put("show @@heartbeat", "Report heartbeat status"); helps.put("show @@heartbeat.detail where name=?", "Report heartbeat current detail"); helps.put("show @@slow where schema = ?", "Report schema slow sql"); helps.put("show @@slow where datanode = ?", "Report datanode slow sql"); helps.put("show @@sysparam", "Report system param"); helps.put("show @@syslog limit=?", "Report system mycat.log"); helps.put("show @@white", "show mycat white host "); helps.put("show @@white.set=?,?", "set mycat white host,[ip,user]"); helps.put("show @@directmemory=1 or 2", "show mycat direct memory usage"); helps.put("show @@check_global -SCHEMA= ? -TABLE=? -retry=? -interval=?", "check mycat global table consistency "); // switch helps.put("switch @@datasource name:index", "Switch dataSource"); // kill helps.put("kill @@connection id1,id2,...", "Kill the specified connections"); // stop helps.put("stop @@heartbeat name:time", "Pause dataNode heartbeat"); // reload helps.put("reload @@config", "Reload basic config from file"); helps.put("reload @@config_all", "Reload all config from file"); helps.put("reload @@route", "Reload route config from file"); helps.put("reload @@user", "Reload user config from file"); helps.put("reload @@sqlslow=", "Set Slow SQL Time(ms)"); helps.put("reload @@user_stat", "Reset show @@sql @@sql.sum @@sql.slow"); // rollback helps.put("rollback @@config", "Rollback all config from memory"); helps.put("rollback @@route", "Rollback route config from memory"); helps.put("rollback @@user", "Rollback user config from memory"); // open/close sql stat helps.put("reload @@sqlstat=open", "Open real-time sql stat analyzer"); helps.put("reload @@sqlstat=close", "Close real-time sql stat analyzer"); // offline/online helps.put("offline", "Change MyCat status to OFF"); helps.put("online", "Change MyCat status to ON"); // clear helps.put("clear @@slow where schema = ?", "Clear slow sql by schema"); helps.put("clear @@slow where datanode = ?", "Clear slow sql by datanode"); // list sort keys.addAll(helps.keySet()); } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowParser.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; /** * @author mycat */ public final class ShowParser { private static final int FIELD_COUNT = 7; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("PROCESSOR_NAME", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("PARSE_COUNT", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("TIME_COUNT", Fields.FIELD_TYPE_DOUBLE); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("MAX_PARSE_TIME", Fields.FIELD_TYPE_DOUBLE); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("MAX_PARSE_SQL_ID", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("CACHED_COUNT", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("CACHE_SIZE", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; for (int i = 0; i < 1; i++) { RowDataPacket row = getRow(c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } private static RowDataPacket getRow(String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(null); row.add(null); row.add(null); row.add(null); row.add(null); row.add(null); row.add(null); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowProcessor.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.buffer.BufferPool; import io.mycat.buffer.ByteBufferArena; import io.mycat.buffer.DirectByteBufferPool; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.NIOProcessor; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.IntegerUtil; import io.mycat.util.LongUtil; import java.nio.ByteBuffer; /** * 查看处理器状态 * * @author mycat * @author mycat */ public final class ShowProcessor { private static final int FIELD_COUNT = 12; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("NAME", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("NET_IN", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("NET_OUT", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("REACT_COUNT", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("R_QUEUE", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("W_QUEUE", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("FREE_BUFFER", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("TOTAL_BUFFER", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("BU_PERCENT", Fields.FIELD_TYPE_TINY); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("BU_WARNS", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("FC_COUNT", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("BC_COUNT", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c, true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c, true); } // write eof buffer = eof.write(buffer, c, true); // write rows byte packetId = eof.packetId; for (NIOProcessor p : MycatServer.getInstance().getProcessors()) { RowDataPacket row = getRow(p, c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c, true); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c, true); // write buffer c.write(buffer); } private static RowDataPacket getRow(NIOProcessor processor, String charset) { BufferPool pool = processor.getBufferPool(); RowDataPacket row = new RowDataPacket(FIELD_COUNT); long bufferSize = 0;//总数 long bufferCapacity = 0;//使用 long free = 0; long bufferUsagePercent = 0; long bufferSharedOpts = pool.getSharedOptsCount(); try { if (pool instanceof ByteBufferArena) { ByteBufferArena bufferPool = (ByteBufferArena) pool; bufferSize = bufferPool.size();//总大小 bufferCapacity = bufferPool.capacity();//使用 free = bufferSize - bufferCapacity; bufferUsagePercent = (long) ((bufferCapacity) * 100.0 / bufferSize); } else if (pool instanceof DirectByteBufferPool) { DirectByteBufferPool bufferPool = (DirectByteBufferPool) pool; bufferSize = bufferPool.size();//总大小 bufferCapacity = bufferPool.getNetDirectMemoryUsage().values().stream() .mapToLong(i -> i).count(); free = bufferSize - bufferCapacity; bufferSharedOpts = bufferPool.getSharedOptsCount(); bufferUsagePercent = (long) ((bufferCapacity) * 100.0 / bufferSize); } else { return row; } row.add(processor.getName().getBytes()); row.add(LongUtil.toBytes(processor.getNetInBytes())); row.add(LongUtil.toBytes(processor.getNetOutBytes())); row.add(LongUtil.toBytes(0)); row.add(IntegerUtil.toBytes(0)); row.add(IntegerUtil.toBytes(processor.getWriteQueueSize())); row.add(LongUtil.toBytes(free)); row.add(LongUtil.toBytes(bufferSize)); row.add(LongUtil.toBytes(bufferUsagePercent)); row.add(LongUtil.toBytes(bufferSharedOpts)); row.add(IntegerUtil.toBytes(processor.getFrontends().size())); row.add(IntegerUtil.toBytes(processor.getBackends().size())); } finally { return row; } } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowRouter.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.text.DecimalFormat; import java.text.NumberFormat; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.NIOProcessor; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; /** * @author mycat */ public final class ShowRouter { private static final int FIELD_COUNT = 5; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("PROCESSOR_NAME", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("ROUTE_COUNT", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("TIME_COUNT", Fields.FIELD_TYPE_FLOAT); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("MAX_ROUTE_TIME", Fields.FIELD_TYPE_FLOAT); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("MAX_ROUTE_SQL_ID", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; for (NIOProcessor p : MycatServer.getInstance().getProcessors()) { RowDataPacket row = getRow(p, c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } private static final NumberFormat nf = DecimalFormat.getInstance(); static { nf.setMaximumFractionDigits(3); } private static RowDataPacket getRow(NIOProcessor processor, String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(processor.getName().getBytes()); row.add(null); row.add(null); row.add(null); row.add(null); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowSQL.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.util.List; import java.util.Map; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.statistic.stat.UserSqlLastStat; import io.mycat.statistic.stat.UserStat; import io.mycat.statistic.stat.UserStatAnalyzer; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; /** * 查询用户最近执行的SQL记录 * * @author mycat * @author zhuam */ public final class ShowSQL { private static final int FIELD_COUNT = 6; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("ID", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("USER", Fields.FIELD_TYPE_VARCHAR); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("START_TIME", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("EXECUTE_TIME", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("SQL", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("IP", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c, boolean isClear) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; Map statMap = UserStatAnalyzer.getInstance().getUserStatMap(); for (UserStat userStat : statMap.values()) { String user = userStat.getUser(); List sqls = userStat.getSqlLastStat().getSqls(); int i = 1; for (UserSqlLastStat.SqlLast sqlLast : sqls) { if (sqlLast != null) { RowDataPacket row = getRow(user, sqlLast, i, c.getCharset()); row.packetId = ++packetId; i++; buffer = row.write(buffer, c,true); } } //读取SQL监控后清理 if ( isClear ) { userStat.getSqlLastStat().clear(); } } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } private static RowDataPacket getRow(String user, UserSqlLastStat.SqlLast sql, int idx, String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(LongUtil.toBytes(idx)); row.add( StringUtil.encode( user, charset) ); row.add( LongUtil.toBytes( sql.getStartTime() ) ); row.add( LongUtil.toBytes( sql.getExecuteTime() ) ); row.add( StringUtil.encode( sql.getSql(), charset) ); row.add(StringUtil.encode( sql.getHost(),charset )); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowSQLCondition.java ================================================ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.statistic.stat.QueryConditionAnalyzer; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; /** * SQL 查询条件 值统计 * * @author zhuam * */ public class ShowSQLCondition { private static final int FIELD_COUNT = 4; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("ID", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("KEY", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("VALUE", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("COUNT", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; String key = QueryConditionAnalyzer.getInstance().getKey(); List> list = QueryConditionAnalyzer.getInstance().getValues(); if ( list != null ) { int size = list.size(); long total = 0L; for (int i = 0; i < size; i++) { Map.Entry entry = list.get(i); Object value = entry.getKey(); Long count = entry.getValue().get(); total += count; RowDataPacket row = getRow(i, key, value.toString(), count, c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } RowDataPacket vk_row = getRow(size + 1, key + ".valuekey", "size", size, c.getCharset()); vk_row.packetId = ++packetId; buffer = vk_row.write(buffer, c,true); RowDataPacket vc_row = getRow(size + 2, key + ".valuecount", "total", total, c.getCharset()); vc_row.packetId = ++packetId; buffer = vc_row.write(buffer, c,true); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } private static RowDataPacket getRow(int i, String key, String value, long count, String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add( LongUtil.toBytes( i ) ); row.add( StringUtil.encode(key, charset) ); row.add( StringUtil.encode(value, charset) ); row.add( LongUtil.toBytes( count ) ); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowSQLDetail.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.text.DecimalFormat; import java.text.NumberFormat; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; /** * 查询指定SQL在各个pool中的执行情况 * * @author mycat * @author mycat */ public final class ShowSQLDetail { private static final NumberFormat nf = DecimalFormat.getInstance(); private static final int FIELD_COUNT = 5; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { nf.setMaximumFractionDigits(3); int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("DATA_SOURCE", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("EXECUTE", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("TIME", Fields.FIELD_TYPE_DOUBLE); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("LAST_EXECUTE_TIMESTAMP", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("LAST_TIME", Fields.FIELD_TYPE_DOUBLE); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c, long sql) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; for (int i = 0; i < 3; i++) { RowDataPacket row = getRow(sql, c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } private static RowDataPacket getRow(long sql, String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add("mysql_1".getBytes()); row.add(LongUtil.toBytes(123L)); row.add(StringUtil.encode(nf.format(2.3), charset)); row.add(LongUtil.toBytes(1279188420682L)); row.add(StringUtil.encode(nf.format(3.42), charset)); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowSQLExecute.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.text.DecimalFormat; import java.text.NumberFormat; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; /** * 查询各SQL在所有pool中的执行情况 * * @author mycat */ public final class ShowSQLExecute { private static final NumberFormat nf = DecimalFormat.getInstance(); private static final int FIELD_COUNT = 5; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { nf.setMaximumFractionDigits(3); int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("SQL_ID", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("EXECUTE", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("TIME", Fields.FIELD_TYPE_DOUBLE); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("MAX_TIME", Fields.FIELD_TYPE_DOUBLE); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("MIN_TIME", Fields.FIELD_TYPE_DOUBLE); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; for (int i = 0; i < 3; i++) { RowDataPacket row = getRow(1000 * (i + 1), c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } private static RowDataPacket getRow(long id, String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(LongUtil.toBytes(id)); row.add(LongUtil.toBytes(100L)); row.add(StringUtil.encode(nf.format(898.9), charset)); row.add(StringUtil.encode(nf.format(8.8), charset)); row.add(StringUtil.encode(nf.format(1.0), charset)); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowSQLHigh.java ================================================ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.util.List; import java.util.Map; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.statistic.stat.SqlFrequency; import io.mycat.statistic.stat.UserStat; import io.mycat.statistic.stat.UserStatAnalyzer; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; /** * 查询高频 SQL * * @author zhuam * */ public final class ShowSQLHigh { private static final int FIELD_COUNT = 10; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("ID", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("USER", Fields.FIELD_TYPE_VARCHAR); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("FREQUENCY", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("AVG_TIME", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("MAX_TIME", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("MIN_TIME", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("EXECUTE_TIME", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("LAST_TIME", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("SQL", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("IP", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c, boolean isClear) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; Map statMap = UserStatAnalyzer.getInstance().getUserStatMap(); for (UserStat userStat : statMap.values()) { String user = userStat.getUser(); List list=userStat.getSqlHigh().getSqlFrequency( isClear ); if ( list != null ) { int i = 1; for (SqlFrequency sqlFrequency : list) { if(sqlFrequency != null){ RowDataPacket row = getRow(i, user, sqlFrequency.getSql(), sqlFrequency.getCount(), sqlFrequency.getAvgTime(), sqlFrequency.getMaxTime(), sqlFrequency.getMinTime(), sqlFrequency.getExecuteTime(), sqlFrequency.getLastTime(), c.getCharset(),sqlFrequency.getHost()); row.packetId = ++packetId; buffer = row.write(buffer, c,true); i++; } } } } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } private static RowDataPacket getRow(int i, String user, String sql, long count, long avgTime, long maxTime, long minTime, long executTime, long lastTime, String charset,String host) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(LongUtil.toBytes(i)); row.add(StringUtil.encode(user, charset)); row.add(LongUtil.toBytes(count)); row.add(LongUtil.toBytes(avgTime)); row.add(LongUtil.toBytes(maxTime)); row.add(LongUtil.toBytes(minTime)); row.add(LongUtil.toBytes(executTime)); row.add(LongUtil.toBytes(lastTime)); row.add(StringUtil.encode(sql, charset)); row.add(StringUtil.encode(host,charset )); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowSQLLarge.java ================================================ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.util.List; import java.util.Map; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.statistic.stat.UserSqlLargeStat; import io.mycat.statistic.stat.UserStat; import io.mycat.statistic.stat.UserStatAnalyzer; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; /** * * 查询每个用户大集合返回的 SQL * * @author zhuam * */ public class ShowSQLLarge { private static final int FIELD_COUNT = 6; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("USER", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("ROWS", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("START_TIME", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("EXECUTE_TIME", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("SQL", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("IP", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c, boolean isClear) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; Map statMap = UserStatAnalyzer.getInstance().getUserStatMap(); for (UserStat userStat : statMap.values()) { String user = userStat.getUser(); List sqls = userStat.getSqlLargeRowStat().getSqls(); for (UserSqlLargeStat.SqlLarge sql : sqls) { if (sql != null) { RowDataPacket row = getRow(user, sql, c.getCharset(),sql.getHost()); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } } if ( isClear ) { userStat.getSqlLargeRowStat().clear();//读取大结果集SQL后，清理 } } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } private static RowDataPacket getRow(String user, UserSqlLargeStat.SqlLarge sql, String charset, String host) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add( StringUtil.encode(user, charset) ); row.add( LongUtil.toBytes( sql.getSqlRows() ) ); row.add( LongUtil.toBytes(sql.getStartTime() ) ); row.add( LongUtil.toBytes(sql.getExecuteTime() ) ); row.add( StringUtil.encode(sql.getSql(), charset) ); row.add(StringUtil.encode(host,charset) ); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowSQLSlow.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.Map; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.statistic.SQLRecord; import io.mycat.statistic.stat.UserStat; import io.mycat.statistic.stat.UserStatAnalyzer; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; /** * 查询每个用户的执行时间超过设定阈值的SQL, 默认TOP10 * * @author mycat * @author zhuam */ public final class ShowSQLSlow { private static final int FIELD_COUNT = 7; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("USER", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("DATASOURCE", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("START_TIME", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("EXECUTE_TIME", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("SQL", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("IP", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("SCHEMA", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } /** * * @param c * @param isClear 查询完成后是否清理掉内存里面的所有慢SQL数据 * @param schema 逻辑库的schema，如果不为空 ，仅仅展示此schema的慢SQL */ public static void execute(ManagerConnection c, boolean isClear, String schema) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; Map statMap = UserStatAnalyzer.getInstance().getUserStatMap(); for (UserStat userStat : statMap.values()) { String user = userStat.getUser(); List keyList = userStat.getSqlRecorder().getRecords(); for (SQLRecord key : keyList) { if (key != null) { // 如果带schema条件，过滤下信息 if (schema != null && !schema.equalsIgnoreCase(key.schema)) { continue; } RowDataPacket row = getRow(user, key, c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } } if ( isClear ) { userStat.getSqlRecorder().clear();//读取慢SQL后，清理 } } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } private static RowDataPacket getRow(String user, SQLRecord sql, String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add( StringUtil.encode(user, charset) ); row.add( StringUtil.encode(sql.dataNode, charset) ); row.add( LongUtil.toBytes(sql.startTime) ); row.add( LongUtil.toBytes(sql.executeTime) ); row.add( StringUtil.encode(sql.statement, charset) ); row.add(StringUtil.encode(sql.host,charset )); row.add(StringUtil.encode(sql.schema, charset)); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowSQLSumTable.java ================================================ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.text.DecimalFormat; import java.util.List; import java.util.Map; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.statistic.stat.TableStat; import io.mycat.statistic.stat.TableStatAnalyzer; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; public class ShowSQLSumTable { private static DecimalFormat decimalFormat = new DecimalFormat("0.00"); private static final int FIELD_COUNT = 8; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("ID", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("TABLE", Fields.FIELD_TYPE_VARCHAR); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("R", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("W", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("R%", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("RELATABLE", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("RELACOUNT", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("LAST_TIME", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c, boolean isClear) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; /* int i=0; Map statMap = TableStatAnalyzer.getInstance().getTableStatMap(); for (TableStat tableStat : statMap.values()) { i++; RowDataPacket row = getRow(tableStat,i, c.getCharset());//getRow(sqlStat,sql, c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } */ List list = TableStatAnalyzer.getInstance().getTableStats(isClear); if ( list != null ) { int i = 1; for (TableStat tableStat : list) { if(tableStat!=null){ RowDataPacket row = getRow(tableStat,i, c.getCharset()); i++; row.packetId = ++packetId; buffer = row.write(buffer, c,true); } } } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } private static RowDataPacket getRow(TableStat tableStat, long idx, String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(LongUtil.toBytes(idx)); if (tableStat == null){ row.add(StringUtil.encode(("not fond"), charset)); return row; } String table = tableStat.getTable(); long R = tableStat.getRCount(); long W = tableStat.getWCount(); String __R = decimalFormat.format( 1.0D * R / (R + W) ); StringBuffer relaTableNameBuffer = new StringBuffer(); StringBuffer relaTableCountBuffer = new StringBuffer(); List relaTables = tableStat.getRelaTables(); if ( !relaTables.isEmpty() ) { for(TableStat.RelaTable relaTable: relaTables) { relaTableNameBuffer.append( relaTable.getTableName() ).append(", "); relaTableCountBuffer.append( relaTable.getCount() ).append(", "); } } else { relaTableNameBuffer.append("NULL"); relaTableCountBuffer.append("NULL"); } row.add( StringUtil.encode( table, charset) ); row.add( LongUtil.toBytes( R ) ); row.add( LongUtil.toBytes( W ) ); row.add( StringUtil.encode( String.valueOf( __R ), charset) ); row.add( StringUtil.encode( relaTableNameBuffer.toString(), charset) ); row.add( StringUtil.encode( relaTableCountBuffer.toString(), charset) ); row.add( LongUtil.toBytes( tableStat.getLastExecuteTime() ) ); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowSQLSumUser.java ================================================ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.text.DecimalFormat; import java.util.Map; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.statistic.stat.UserSqlRWStat; import io.mycat.statistic.stat.UserStat; import io.mycat.statistic.stat.UserStatAnalyzer; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; /** * 查询用户的 SQL 执行情况 * * 1、用户 R/W数、读占比、并发数 * 2、请求时间范围 * 3、请求的耗时范围 * 4、Net 进/出 字节数 * * @author zhuam */ public class ShowSQLSumUser { private static DecimalFormat decimalFormat = new DecimalFormat("0.00"); private static final int FIELD_COUNT = 11; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("ID", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("USER", Fields.FIELD_TYPE_VARCHAR); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("R", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("W", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("R%", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("MAX", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("NET_IN", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("NET_OUT", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; //22-06h, 06-13h, 13-18h, 18-22h fields[i] = PacketUtil.getField("TIME_COUNT", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; //<10ms, 10ms-200ms, 200ms-1s, >1s fields[i] = PacketUtil.getField("TTL_COUNT", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("LAST_TIME", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c, boolean isClear) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; int i=0; Map statMap = UserStatAnalyzer.getInstance().getUserStatMap(); for (UserStat userStat : statMap.values()) { i++; RowDataPacket row = getRow(userStat,i, c.getCharset());//getRow(sqlStat,sql, c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c,true); if ( isClear ) { userStat.clearRwStat(); } } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } private static RowDataPacket getRow(UserStat userStat, long idx, String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(LongUtil.toBytes(idx)); if (userStat == null){ row.add(StringUtil.encode(("not fond"), charset)); return row; } String user = userStat.getUser(); UserSqlRWStat rwStat = userStat.getRWStat(); long R = rwStat.getRCount(); long W = rwStat.getWCount(); String __R = decimalFormat.format( 1.0D * R / (R + W) ); int MAX = rwStat.getConcurrentMax(); row.add( StringUtil.encode( user, charset) ); row.add( LongUtil.toBytes( R ) ); row.add( LongUtil.toBytes( W ) ); row.add( StringUtil.encode( String.valueOf( __R ), charset) ); row.add( StringUtil.encode( String.valueOf( MAX ), charset) ); row.add( LongUtil.toBytes( rwStat.getNetInBytes() ) ); row.add( LongUtil.toBytes( rwStat.getNetOutBytes() ) ); row.add( StringUtil.encode( rwStat.getExecuteHistogram().toString(), charset) ); row.add( StringUtil.encode( rwStat.getTimeHistogram().toString(), charset) ); row.add( LongUtil.toBytes( rwStat.getLastExecuteTime() ) ); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowServer.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.FormatUtil; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; import io.mycat.util.TimeUtil; /** * 服务器状态报告 * * @author mycat * @author mycat */ public final class ShowServer { private static final int FIELD_COUNT = 8; private static final ResultSetHeaderPacket header = PacketUtil .getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("UPTIME", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("USED_MEMORY", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("TOTAL_MEMORY", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("MAX_MEMORY", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("RELOAD_TIME", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("ROLLBACK_TIME", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil .getField("CHARSET", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("STATUS", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c, true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c, true); } // write eof buffer = eof.write(buffer, c, true); // write rows byte packetId = eof.packetId; RowDataPacket row = getRow(c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c, true); // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c, true); // write buffer c.write(buffer); } private static RowDataPacket getRow(String charset) { MycatServer server = MycatServer.getInstance(); long startupTime = server.getStartupTime(); long now = TimeUtil.currentTimeMillis(); long uptime = now - startupTime; Runtime rt = Runtime.getRuntime(); long total = rt.totalMemory(); long max = rt.maxMemory(); long used = (total - rt.freeMemory()); RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(FormatUtil.formatTime(uptime, 3), charset)); row.add(LongUtil.toBytes(used)); row.add(LongUtil.toBytes(total)); row.add(LongUtil.toBytes(max)); row.add(LongUtil.toBytes(server.getConfig().getReloadTime())); row.add(LongUtil.toBytes(server.getConfig().getRollbackTime())); row.add(StringUtil.encode(charset, charset)); row.add(StringUtil.encode(MycatServer.getInstance().isOnline() ? "ON" : "OFF", charset)); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowSession.java ================================================ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.util.Collection; import io.mycat.MycatServer; import io.mycat.backend.BackendConnection; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.FrontendConnection; import io.mycat.net.NIOProcessor; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.NonBlockingSession; import io.mycat.server.ServerConnection; import io.mycat.util.StringUtil; /** * show front session detail info * * @author wuzhih * */ public class ShowSession { private static final int FIELD_COUNT = 3; private static final ResultSetHeaderPacket header = PacketUtil .getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("SESSION", Fields.FIELD_TYPE_VARCHAR); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("DN_COUNT", Fields.FIELD_TYPE_VARCHAR); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("DN_LIST", Fields.FIELD_TYPE_VARCHAR); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c, true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c, true); } // write eof buffer = eof.write(buffer, c, true); // write rows byte packetId = eof.packetId; for (NIOProcessor process : MycatServer.getInstance().getProcessors()) { for (FrontendConnection front : process.getFrontends().values()) { if (!(front instanceof ServerConnection)) { continue; } ServerConnection sc = (ServerConnection) front; RowDataPacket row = getRow(sc, c.getCharset()); if (row != null) { row.packetId = ++packetId; buffer = row.write(buffer, c, true); } } } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c, true); // write buffer c.write(buffer); } private static RowDataPacket getRow(ServerConnection sc, String charset) { StringBuilder sb = new StringBuilder(); NonBlockingSession ssesion = sc.getSession2(); Collection backConnections = ssesion.getTargetMap() .values(); int cncount = backConnections.size(); if (cncount == 0) { return null; } for (BackendConnection backCon : backConnections) { sb.append(backCon).append("\r\n"); } RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(sc.getId() + "", charset)); row.add(StringUtil.encode(cncount + "", charset)); row.add(StringUtil.encode(sb.toString(), charset)); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowSqlResultSet.java ================================================ package io.mycat.manager.response; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.statistic.stat.SqlResultSet; import io.mycat.statistic.stat.UserStat; import io.mycat.statistic.stat.UserStatAnalyzer; import io.mycat.util.IntegerUtil; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; import java.nio.ByteBuffer; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; /** * show 大结果集 SQL * * @author songgw * */ public final class ShowSqlResultSet { private static final int FIELD_COUNT = 5; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("ID", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("USER", Fields.FIELD_TYPE_VARCHAR); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("FREQUENCY", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("SQL", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("RESULTSET_SIZE", Fields.FIELD_TYPE_INT24); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; int i=0; Map statMap = UserStatAnalyzer.getInstance().getUserStatMap(); for (UserStat userStat : statMap.values()) { String user = userStat.getUser(); ConcurrentHashMap map=userStat.getSqlResultSizeRecorder().getSqlResultSet(); if ( map != null ) { for (SqlResultSet sqlResultSet:map.values()) { RowDataPacket row = getRow(++i, user,sqlResultSet.getSql(), sqlResultSet.getCount(), sqlResultSet.getResultSetSize(),c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } } } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } private static RowDataPacket getRow(int i, String user,String sql, int count, int resultSetSize,String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add( LongUtil.toBytes( i ) ); row.add( StringUtil.encode( user, charset) ); row.add( LongUtil.toBytes( count ) ); row.add( StringUtil.encode(sql, charset) ); row.add( IntegerUtil.toBytes(resultSetSize) ); return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowSysLog.java ================================================ package io.mycat.manager.response; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.config.model.SystemConfig; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.StringUtil; /** * Show @@SYSLOG LIMIT=50 * * @author zhuam * */ public class ShowSysLog { private static final int FIELD_COUNT = 2; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("DATE", Fields.FIELD_TYPE_VARCHAR); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("LOG", Fields.FIELD_TYPE_VARCHAR); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c, int numLines) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c, true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c, true); } // write eof buffer = eof.write(buffer, c, true); // write rows byte packetId = eof.packetId; String filename = SystemConfig.getHomePath() + File.separator + "logs" + File.separator + "mycat.log"; String[] lines = getLinesByLogFile(filename, numLines); boolean linesIsEmpty = true; for(int i = 0; i < lines.length ; i++){ String line = lines[i]; if ( line != null ) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode( line.substring(0,19), c.getCharset())); row.add(StringUtil.encode( line.substring(19,line.length()), c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); linesIsEmpty = false; } } if ( linesIsEmpty ) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode( "NULL", c.getCharset())); row.add(StringUtil.encode( "NULL", c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c, true); // write buffer c.write(buffer); } private static String[] getLinesByLogFile(String filename, int numLines) { String lines[] = new String[numLines]; BufferedReader in = null; try { //获取长度 int start = 0; int totalNumLines = 0; File logFile = new File(filename); in = new BufferedReader(new InputStreamReader(new FileInputStream(logFile), "UTF-8")); String line; while ((line=in.readLine()) != null) { totalNumLines++; } in.close(); // in = new BufferedReader(new InputStreamReader(new FileInputStream(logFile), "UTF-8")); // 跳过行 start = totalNumLines - numLines; if (start < 0) { start = 0; } for (int i=0; i paramValues = new ArrayList(); paramValues.add(sysConfig.getProcessors() + ""); paramValues.add(sysConfig.getBufferPoolChunkSize() + "B"); paramValues.add(sysConfig.getBufferPoolPageSize() + "B"); paramValues.add(sysConfig.getProcessorBufferLocalPercent() + ""); paramValues.add(sysConfig.getProcessorExecutor() + ""); paramValues.add(sysConfig.getSequenceHandlerType() == 1 ? "数据库方式" : "本地文件方式"); paramValues.add(sysConfig.getPacketHeaderSize() + "B"); paramValues.add(sysConfig.getMaxPacketSize()/1024/1024 + "M"); paramValues.add(sysConfig.getIdleTimeout()/1000/60 + "分钟"); paramValues.add(sysConfig.getCharset() + ""); paramValues.add(ISOLATIONS[sysConfig.getTxIsolation()]); paramValues.add(sysConfig.getSqlExecuteTimeout() + "秒"); paramValues.add(sysConfig.getProcessorCheckPeriod()/1000 + "秒"); paramValues.add(sysConfig.getDataNodeIdleCheckPeriod()/1000 + "秒"); paramValues.add(sysConfig.getDataNodeHeartbeatPeriod()/1000 + "秒"); paramValues.add(sysConfig.getBindIp() + ""); paramValues.add(sysConfig.getServerPort()+ ""); paramValues.add(sysConfig.getManagerPort() + ""); paramValues.add(sysConfig.getUseSqlStat() + ""); for(int i = 0; i < PARAMNAMES.length ; i++){ RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(PARAMNAMES[i], c.getCharset())); row.add(StringUtil.encode(paramValues.get(i), c.getCharset())); row.add(StringUtil.encode(PARAM_DESCRIPTION[i], c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c, true); // write buffer c.write(buffer); } private static final String[] PARAMNAMES = { "processors", "processorBufferChunk", "processorBufferPool", "processorBufferLocalPercent", "processorExecutor", "sequnceHandlerType", "Mysql_packetHeaderSize", "Mysql_maxPacketSize", "Mysql_idleTimeout", "Mysql_charset", "Mysql_txIsolation", "Mysql_sqlExecuteTimeout", "Mycat_processorCheckPeriod", "Mycat_dataNodeIdleCheckPeriod", "Mycat_dataNodeHeartbeatPeriod", "Mycat_bindIp", "Mycat_serverPort", "Mycat_managerPort", "Mycat_useSqlStat"}; private static final String[] PARAM_DESCRIPTION = { "主要用于指定系统可用的线程数，默认值为Runtime.getRuntime().availableProcessors()方法返回的值。主要影响processorBufferPool、processorBufferLocalPercent、processorExecutor属性。NIOProcessor的个数也是由这个属性定义的，所以调优的时候可以适当的调高这个属性。", "指定每次分配Socket Direct Buffer的大小，默认是4096个字节。这个属性也影响buffer pool的长度。", "指定bufferPool计算 比例值。由于每次执行NIO读、写操作都需要使用到buffer，系统初始化的时候会建立一定长度的buffer池来加快读、写的效率，减少建立buffer的时间", "就是用来控制分配这个pool的大小用的，但其也并不是一个准确的值，也是一个比例值。这个属性默认值为100。线程缓存百分比 = bufferLocalPercent / processors属性。", "主要用于指定NIOProcessor上共享的businessExecutor固定线程池大小。mycat在需要处理一些异步逻辑的时候会把任务提交到这个线程池中。新版本中这个连接池的使用频率不是很大了，可以设置一个较小的值。", "指定使用Mycat全局序列的类型。", "指定Mysql协议中的报文头长度。默认4", "指定Mysql协议可以携带的数据最大长度。默认16M", "指定连接的空闲超时时间。某连接在发起空闲检查下，发现距离上次使用超过了空闲时间，那么这个连接会被回收，就是被直接的关闭掉。默认30分钟", "连接的初始化字符集。默认为utf8", "前端连接的初始化事务隔离级别，只在初始化的时候使用，后续会根据客户端传递过来的属性对后端数据库连接进行同步。默认为REPEATED_READ", "SQL执行超时的时间，Mycat会检查连接上最后一次执行SQL的时间，若超过这个时间则会直接关闭这连接。默认时间为300秒", "清理NIOProcessor上前后端空闲、超时和关闭连接的间隔时间。默认是1秒", "对后端连接进行空闲、超时检查的时间间隔，默认是300秒", "对后端所有读、写库发起心跳的间隔时间，默认是10秒", "mycat服务监听的IP地址，默认值为0.0.0.0", "mycat的使用端口，默认值为8066", "mycat的管理端口，默认值为9066", "mycat是否开启SQL统计,1开启，0未开启"}; public static final String[] ISOLATIONS = {"", "READ_UNCOMMITTED", "READ_COMMITTED", "REPEATED_READ", "SERIALIZABLE"}; } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowThreadPool.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.util.LinkedList; import java.util.List; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.IntegerUtil; import io.mycat.util.LongUtil; import io.mycat.util.NameableExecutor; import io.mycat.util.StringUtil; /** * 查看线程池状态 * * @author mycat * @author mycat */ public final class ShowThreadPool { private static final int FIELD_COUNT = 6; private static final ResultSetHeaderPacket header = PacketUtil .getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("NAME", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("POOL_SIZE", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("ACTIVE_COUNT", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("TASK_QUEUE_SIZE", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("COMPLETED_TASK", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("TOTAL_TASK", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c, true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c, true); } // write eof buffer = eof.write(buffer, c, true); // write rows byte packetId = eof.packetId; List executors = getExecutors(); for (NameableExecutor exec : executors) { if (exec != null) { RowDataPacket row = getRow(exec, c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c, true); } } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c, true); // write buffer c.write(buffer); } private static RowDataPacket getRow(NameableExecutor exec, String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(exec.getName(), charset)); row.add(IntegerUtil.toBytes(exec.getPoolSize())); row.add(IntegerUtil.toBytes(exec.getActiveCount())); row.add(IntegerUtil.toBytes(exec.getQueue().size())); row.add(LongUtil.toBytes(exec.getCompletedTaskCount())); row.add(LongUtil.toBytes(exec.getTaskCount())); return row; } private static List getExecutors() { List list = new LinkedList(); MycatServer server = MycatServer.getInstance(); list.add(server.getTimerExecutor()); // list.add(server.getAioExecutor()); list.add(server.getBusinessExecutor()); // for (NIOProcessor pros : server.getProcessors()) { // list.add(pros.getExecutor()); // } return list; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowTime.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.route.parser.ManagerParseShow; import io.mycat.util.LongUtil; /** * @author mycat */ public final class ShowTime { private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("TIMESTAMP", Fields.FIELD_TYPE_LONGLONG); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c, int type) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; RowDataPacket row = getRow(type); row.packetId = ++packetId; buffer = row.write(buffer, c,true); // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } private static RowDataPacket getRow(int type) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); switch (type) { case ManagerParseShow.TIME_CURRENT: row.add(LongUtil.toBytes(System.currentTimeMillis())); break; case ManagerParseShow.TIME_STARTUP: row.add(LongUtil.toBytes(MycatServer.getInstance().getStartupTime())); break; default: row.add(LongUtil.toBytes(0L)); } return row; } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowVariables.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.StringUtil; /** * @author mycat */ public final class ShowVariables { private static final int FIELD_COUNT = 2; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("VARIABLE_NAME", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("VALUE", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; for (Map.Entry e : variables.entrySet()) { RowDataPacket row = getRow(e.getKey(), e.getValue(), c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // write lastEof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } private static RowDataPacket getRow(String name, String value, String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(name, charset)); row.add(StringUtil.encode(value, charset)); return row; } private static final Map variables = new HashMap(); static { variables.put("character_set_client", "utf8"); variables.put("character_set_connection", "utf8"); variables.put("character_set_results", "utf8"); variables.put("character_set_server", "utf8"); variables.put("init_connect", ""); variables.put("interactive_timeout", "172800"); variables.put("lower_case_table_names", "1"); variables.put("max_allowed_packet", "16777216"); variables.put("net_buffer_length", "8192"); variables.put("net_write_timeout", "60"); variables.put("query_cache_size", "0"); variables.put("query_cache_type", "OFF"); variables.put("sql_mode", "STRICT_TRANS_TABLES"); variables.put("system_time_zone", "CST"); variables.put("time_zone", "SYSTEM"); variables.put("lower_case_table_names", "1"); variables.put("tx_isolation", "REPEATABLE-READ"); variables.put("wait_timeout", "172800"); } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowVersion.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.nio.ByteBuffer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.config.Versions; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; /** * 查看CobarServer版本 * * @author mycat */ public final class ShowVersion { private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("VERSION", Fields.FIELD_TYPE_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(Versions.SERVER_VERSION); row.packetId = ++packetId; buffer = row.write(buffer, c,true); // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } } ================================================ FILE: src/main/java/io/mycat/manager/response/ShowWhiteHost.java ================================================ package io.mycat.manager.response; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.ErrorCode; import io.mycat.config.Fields; import io.mycat.config.model.FirewallConfig; import io.mycat.config.model.UserConfig; import io.mycat.config.util.ConfigException; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.OkPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.StringUtil; public final class ShowWhiteHost { private static final Logger LOGGER = LoggerFactory.getLogger(ShowWhiteHost.class); private static final int FIELD_COUNT = 2; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("IP", Fields.FIELD_TYPE_VARCHAR); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("USER", Fields.FIELD_TYPE_VARCHAR); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void execute(ManagerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; Map> map=MycatServer.getInstance().getConfig().getFirewall().getWhitehost(); for (String key : map.keySet()) { List userConfigs=map.get(key); String users=""; for (int i = 0; i < userConfigs.size(); i++) { if(i>0) { users += "," + userConfigs.get(i).getName(); } else { users += userConfigs.get(i).getName(); } } RowDataPacket row = getRow(key, users, c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } Map> map2=MycatServer.getInstance().getConfig().getFirewall().getWhitehostMask(); for (Pattern key : map2.keySet()) { List userConfigs=map2.get(key); String users=""; for (int i = 0; i < userConfigs.size(); i++) { if(i>0) { users += "," + userConfigs.get(i).getName(); } else { users += userConfigs.get(i).getName(); } } RowDataPacket row = getRow(FirewallConfig.getHost(key), users, c.getCharset()); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } private static RowDataPacket getRow(String ip, String user, String charset) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add( StringUtil.encode( ip, charset) ); row.add( StringUtil.encode( user, charset) ); return row; } public static String parseString(String stmt) { int offset = stmt.indexOf(','); if (offset != -1 && stmt.length() > ++offset) { String txt = stmt.substring(offset).trim(); return txt; } return null; } public static synchronized void setHost(ManagerConnection c,String ips) { OkPacket ok = new OkPacket(); String []users = ips.split(","); if (users.length<2){ c.writeErrMessage(ErrorCode.ER_YES, "white host info error."); return; } String host=""; List userConfigs = new ArrayList(); int i=0; for(String user : users){ if (i==0){ host=user; i++; } else { i++; UserConfig uc = MycatServer.getInstance().getConfig().getUsers().get(user); if (null == uc) { c.writeErrMessage(ErrorCode.ER_YES, "user doesn't exist in host."); return; } if (uc.getSchemas() == null || uc.getSchemas().size() == 0) { c.writeErrMessage(ErrorCode.ER_YES, "host contains one root privileges user."); return; } userConfigs.add(uc); } } if (MycatServer.getInstance().getConfig().getFirewall().addWhitehost(host, userConfigs)) { try{ FirewallConfig.updateToFile(host, userConfigs); }catch(Exception e){ LOGGER.warn("set while host error : " + e.getMessage()); c.writeErrMessage(ErrorCode.ER_YES, "white host set success ,but write to file failed :" + e.getMessage()); } ok.packetId = 1; ok.affectedRows = 1; ok.serverStatus = 2; ok.message = "white host set to succeed.".getBytes(); ok.write(c); } else { c.writeErrMessage(ErrorCode.ER_YES, "host duplicated."); } } } ================================================ FILE: src/main/java/io/mycat/manager/response/StopHeartbeat.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.OkPacket; import io.mycat.route.parser.ManagerParseStop; import io.mycat.route.parser.util.Pair; import io.mycat.util.FormatUtil; import io.mycat.util.TimeUtil; /** * 暂停数据节点心跳检测 * * @author mycat */ public final class StopHeartbeat { private static final Logger logger = LoggerFactory.getLogger(StopHeartbeat.class); public static void execute(String stmt, ManagerConnection c) { int count = 0; Pair keys = ManagerParseStop.getPair(stmt); if (keys.getKey() != null && keys.getValue() != null) { long time = keys.getValue().intValue() * 1000L; Map dns = MycatServer.getInstance().getConfig().getDataHosts(); for (String key : keys.getKey()) { PhysicalDBPool dn = dns.get(key); if (dn != null) { dn.getSource().setHeartbeatRecoveryTime(TimeUtil.currentTimeMillis() + time); ++count; StringBuilder s = new StringBuilder(); s.append(dn.getHostName()).append(" stop heartbeat '"); logger.warn(s.append(FormatUtil.formatTime(time, 3)).append("' by manager.").toString()); } } } OkPacket packet = new OkPacket(); packet.packetId = 1; packet.affectedRows = count; packet.serverStatus = 2; packet.write(c); } } ================================================ FILE: src/main/java/io/mycat/manager/response/SwitchDataSource.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.manager.response; import java.util.Map; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.manager.ManagerConnection; import io.mycat.net.mysql.OkPacket; import io.mycat.route.parser.ManagerParseSwitch; import io.mycat.route.parser.util.Pair; /** * 切换数据节点的数据源 * * @author mycat */ public final class SwitchDataSource { public static void response(String stmt, ManagerConnection c) { int count = 0; Pair pair = ManagerParseSwitch.getPair(stmt); Map dns = MycatServer.getInstance().getConfig().getDataHosts(); Integer idx = pair.getValue(); for (String key : pair.getKey()) { PhysicalDBPool dn = dns.get(key); if (dn != null) { int m = dn.getActivedIndex(); int n = (idx == null) ? dn.next(m) : idx.intValue(); if(!dn.notSwitchSource(n)) { //todo 如果是zk集群 需要将结果写入到zk中再来切换. if(MycatServer.getInstance().isUseZkSwitch()) { MycatServer.getInstance().saveDataHostIndexToZk(dn.getHostName(), n); ++count; //return switchSourceVoted( n, isAlarm, reason); } else { if (dn.switchSource(n, false, "MANAGER")) { ++count; } } } } } OkPacket packet = new OkPacket(); packet.packetId = 1; packet.affectedRows = count; packet.serverStatus = 2; packet.write(c); } } ================================================ FILE: src/main/java/io/mycat/memory/MyCatMemory.java ================================================ package io.mycat.memory; import com.google.common.annotations.VisibleForTesting; import io.mycat.config.model.SystemConfig; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager; import io.mycat.memory.unsafe.memory.mm.MemoryManager; import io.mycat.memory.unsafe.memory.mm.ResultMergeMemoryManager; import io.mycat.memory.unsafe.storage.DataNodeDiskManager; import io.mycat.memory.unsafe.storage.SerializerManager; import io.mycat.memory.unsafe.utils.JavaUtils; import io.mycat.memory.unsafe.utils.MycatPropertyConf; import org.apache.log4j.Logger; /** * Created by zagnix on 2016/6/2. * Mycat内存管理工具类 * 规划为三部分内存:结果集处理内存,系统预留内存,网络处理内存 * 其中网络处理内存部分全部为Direct Memory * 结果集内存分为Direct Memory 和 Heap Memory，但目前仅使用Direct Memory * 系统预留内存为 Heap Memory。 * 系统运行时，必须设置-XX:MaxDirectMemorySize 和 -Xmx JVM参数 * -Xmx1024m -Xmn512m -XX:MaxDirectMemorySize=2048m -Xss256K -XX:+UseParallelGC */ public class MyCatMemory { private static Logger LOGGER = Logger.getLogger(MyCatMemory.class); public final static double DIRECT_SAFETY_FRACTION = 0.7; private final long systemReserveBufferSize; private final long memoryPageSize; private final long spillsFileBufferSize; private final long resultSetBufferSize; private final int numCores; /** * 内存管理相关关键类 */ private final MycatPropertyConf conf; private final MemoryManager resultMergeMemoryManager; private final DataNodeDiskManager blockManager; private final SerializerManager serializerManager; private final SystemConfig system; public MyCatMemory(SystemConfig system,long totalNetWorkBufferSize) throws NoSuchFieldException, IllegalAccessException { this.system = system; LOGGER.info("useOffHeapForMerge = " + system.getUseOffHeapForMerge()); LOGGER.info("memoryPageSize = " + system.getMemoryPageSize()); LOGGER.info("spillsFileBufferSize = " + system.getSpillsFileBufferSize()); LOGGER.info("useStreamOutput = " + system.getUseStreamOutput()); LOGGER.info("systemReserveMemorySize = " + system.getSystemReserveMemorySize()); LOGGER.info("totalNetWorkBufferSize = " + JavaUtils.bytesToString2(totalNetWorkBufferSize)); LOGGER.info("dataNodeSortedTempDir = " + system.getDataNodeSortedTempDir()); this.conf = new MycatPropertyConf(); numCores = Runtime.getRuntime().availableProcessors(); this.systemReserveBufferSize = JavaUtils. byteStringAsBytes(system.getSystemReserveMemorySize()); this.memoryPageSize = JavaUtils. byteStringAsBytes(system.getMemoryPageSize()); this.spillsFileBufferSize = JavaUtils. byteStringAsBytes(system.getSpillsFileBufferSize()); /** * 目前merge，order by ，limit 没有使用On Heap内存 */ long maxOnHeapMemory = (Platform.getMaxHeapMemory()-systemReserveBufferSize); assert maxOnHeapMemory > 0; resultSetBufferSize = (long)((Platform.getMaxDirectMemory()-2*totalNetWorkBufferSize)*DIRECT_SAFETY_FRACTION); assert resultSetBufferSize > 0; /** * mycat.merge.memory.offHeap.enabled * mycat.buffer.pageSize * mycat.memory.offHeap.size * mycat.merge.file.buffer * mycat.direct.output.result * mycat.local.dir */ if(system.getUseOffHeapForMerge()== 1){ conf.set("mycat.memory.offHeap.enabled","true"); }else{ conf.set("mycat.memory.offHeap.enabled","false"); } if(system.getUseStreamOutput() == 1){ conf.set("mycat.stream.output.result","true"); }else{ conf.set("mycat.stream.output.result","false"); } if(system.getMemoryPageSize() != null){ conf.set("mycat.buffer.pageSize",system.getMemoryPageSize()); }else{ conf.set("mycat.buffer.pageSize","32k"); } if(system.getSpillsFileBufferSize() != null){ conf.set("mycat.merge.file.buffer",system.getSpillsFileBufferSize()); }else{ conf.set("mycat.merge.file.buffer","32k"); } conf.set("mycat.pointer.array.len","1k") .set("mycat.memory.offHeap.size", JavaUtils.bytesToString2(resultSetBufferSize)); LOGGER.info("mycat.memory.offHeap.size: " + JavaUtils.bytesToString2(resultSetBufferSize)); resultMergeMemoryManager = new ResultMergeMemoryManager(conf,numCores,maxOnHeapMemory); serializerManager = new SerializerManager(); blockManager = new DataNodeDiskManager(conf,true,serializerManager); } @VisibleForTesting public MyCatMemory() throws NoSuchFieldException, IllegalAccessException { this.system = null; this.systemReserveBufferSize = 0; this.memoryPageSize = 0; this.spillsFileBufferSize = 0; conf = new MycatPropertyConf(); numCores = Runtime.getRuntime().availableProcessors(); long maxOnHeapMemory = (Platform.getMaxHeapMemory()); assert maxOnHeapMemory > 0; resultSetBufferSize = (long)((Platform.getMaxDirectMemory())*DIRECT_SAFETY_FRACTION); assert resultSetBufferSize > 0; /** * mycat.memory.offHeap.enabled * mycat.buffer.pageSize * mycat.memory.offHeap.size * mycat.testing.memory * mycat.merge.file.buffer * mycat.direct.output.result * mycat.local.dir */ conf.set("mycat.memory.offHeap.enabled","true") .set("mycat.pointer.array.len","8K") .set("mycat.buffer.pageSize","1m") .set("mycat.memory.offHeap.size", JavaUtils.bytesToString2(resultSetBufferSize)) .set("mycat.stream.output.result","false"); LOGGER.info("mycat.memory.offHeap.size: " + JavaUtils.bytesToString2(resultSetBufferSize)); resultMergeMemoryManager = new ResultMergeMemoryManager(conf,numCores,maxOnHeapMemory); serializerManager = new SerializerManager(); blockManager = new DataNodeDiskManager(conf,true,serializerManager); } public MycatPropertyConf getConf() { return conf; } public long getResultSetBufferSize() { return resultSetBufferSize; } public MemoryManager getResultMergeMemoryManager() { return resultMergeMemoryManager; } public SerializerManager getSerializerManager() { return serializerManager; } public DataNodeDiskManager getBlockManager() { return blockManager; } } ================================================ FILE: src/main/java/io/mycat/memory/environment/EnvironmentInformation.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.environment; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.InputStream; import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; import java.lang.management.RuntimeMXBean; import java.lang.reflect.Method; import java.util.List; import java.util.Properties; /** * Utility class that gives access to the execution environment of the JVM, like * the executing user, startup options, or the JVM version. */ public class EnvironmentInformation { private static final Logger LOG = LoggerFactory.getLogger(EnvironmentInformation.class); public static final String UNKNOWN = ""; /** * Returns the version of the code as String. If version == null, then the JobManager does not run from a * Maven build. An example is a source code checkout, compile, and run from inside an IDE. * * @return The version string. */ public static String getVersion() { String version = EnvironmentInformation.class.getPackage().getImplementationVersion(); return version != null ? version : UNKNOWN; } /** * Returns the code revision (commit and commit date) of Flink, as generated by the Maven builds. * * @return The code revision. */ public static RevisionInformation getRevisionInformation() { String revision = UNKNOWN; String commitDate = UNKNOWN; try (InputStream propFile = EnvironmentInformation.class.getClassLoader().getResourceAsStream(".version.properties")) { if (propFile != null) { Properties properties = new Properties(); properties.load(propFile); String propRevision = properties.getProperty("git.commit.id.abbrev"); String propCommitDate = properties.getProperty("git.commit.time"); revision = propRevision != null ? propRevision : UNKNOWN; commitDate = propCommitDate != null ? propCommitDate : UNKNOWN; } } catch (Throwable t) { if (LOG.isDebugEnabled()) { LOG.debug("Cannot determine code revision: Unable to read version property file.", t); } else { LOG.info("Cannot determine code revision: Unable to read version property file."); } } return new RevisionInformation(revision, commitDate); } /** * Gets the name of the user that is running the JVM. * * @return The name of the user that is running the JVM. */ public static String getUserRunning() { String user = System.getProperty("user.name"); if (user == null) { user = UNKNOWN; if (LOG.isDebugEnabled()) { LOG.debug("Cannot determine user/group information for the current user."); } } return user; } /** * The maximum JVM heap size, in bytes. * * @return The maximum JVM heap size, in bytes. */ public static long getMaxJvmHeapMemory() { long maxMemory = Runtime.getRuntime().maxMemory(); if (maxMemory == Long.MAX_VALUE) { // amount of free memory unknown try { // workaround for Oracle JDK OperatingSystemMXBean operatingSystemMXBean = ManagementFactory.getOperatingSystemMXBean(); Class clazz = Class.forName("com.sun.management.OperatingSystemMXBean"); Method method = clazz.getMethod("getTotalPhysicalMemorySize"); maxMemory = (Long) method.invoke(operatingSystemMXBean) / 4; } catch (Throwable e) { throw new RuntimeException("Could not determine the amount of free memory.\n" + "Please set the maximum memory for the JVM, e.g. -Xmx512M for 512 megabytes."); } } return maxMemory; } /** * Gets an estimate of the size of the free heap memory. * * NOTE: This method is heavy-weight. It triggers a garbage collection to reduce fragmentation and get * a better estimate at the size of free memory. It is typically more accurate than the plain version * {@link #getSizeOfFreeHeapMemory()}. * * @return An estimate of the size of the free heap memory, in bytes. */ public static long getSizeOfFreeHeapMemoryWithDefrag() { // trigger a garbage collection, to reduce fragmentation System.gc(); return getSizeOfFreeHeapMemory(); } /** * Gets an estimate of the size of the free heap memory. The estimate may vary, depending on the current * level of memory fragmentation and the number of dead objects. For a better (but more heavy-weight) * estimate, use {@link #getSizeOfFreeHeapMemoryWithDefrag()}. * * @return An estimate of the size of the free heap memory, in bytes. */ public static long getSizeOfFreeHeapMemory() { Runtime r = Runtime.getRuntime(); long maxMemory = r.maxMemory(); if (maxMemory == Long.MAX_VALUE) { // amount of free memory unknown try { // workaround for Oracle JDK OperatingSystemMXBean operatingSystemMXBean = ManagementFactory.getOperatingSystemMXBean(); Class clazz = Class.forName("com.sun.management.OperatingSystemMXBean"); Method method = clazz.getMethod("getTotalPhysicalMemorySize"); maxMemory = (Long) method.invoke(operatingSystemMXBean) / 4; } catch (Throwable e) { throw new RuntimeException("Could not determine the amount of free memory.\n" + "Please set the maximum memory for the JVM, e.g. -Xmx512M for 512 megabytes."); } } return maxMemory - r.totalMemory() + r.freeMemory(); } /** * Gets the version of the JVM in the form "VM_Name - Vendor - Spec/Version". * * @return The JVM version. */ public static String getJvmVersion() { try { final RuntimeMXBean bean = ManagementFactory.getRuntimeMXBean(); return bean.getVmName() + " - " + bean.getVmVendor() + " - " + bean.getSpecVersion() + '/' + bean.getVmVersion(); } catch (Throwable t) { return UNKNOWN; } } /** * Gets the system parameters and environment parameters that were passed to the JVM on startup. * * @return The options passed to the JVM on startup. */ public static String getJvmStartupOptions() { try { final RuntimeMXBean bean = ManagementFactory.getRuntimeMXBean(); final StringBuilder bld = new StringBuilder(); for (String s : bean.getInputArguments()) { bld.append(s).append(' '); } return bld.toString(); } catch (Throwable t) { return UNKNOWN; } } /** * Gets the system parameters and environment parameters that were passed to the JVM on startup. * * @return The options passed to the JVM on startup. */ public static String[] getJvmStartupOptionsArray() { try { RuntimeMXBean bean = ManagementFactory.getRuntimeMXBean(); List options = bean.getInputArguments(); return options.toArray(new String[options.size()]); } catch (Throwable t) { return new String[0]; } } /** * Gets the directory for temporary files, as returned by the JVM system property "java.io.tmpdir". * * @return The directory for temporary files. */ public static String getTemporaryFileDirectory() { return System.getProperty("java.io.tmpdir"); } /** * Tries to retrieve the maximum number of open file handles. This method will only work on * UNIX-based operating systems with Sun/Oracle Java versions. * *

If the number of max open file handles cannot be determined, this method returns {@code -1}.

* * @return The limit of open file handles, or {@code -1}, if the limit could not be determined. */ public static long getOpenFileHandlesLimit() { Class sunBeanClass; try { sunBeanClass = Class.forName("com.sun.management.UnixOperatingSystemMXBean"); } catch (ClassNotFoundException e) { return -1L; } try { Method fhLimitMethod = sunBeanClass.getMethod("getMaxFileDescriptorCount"); Object result = fhLimitMethod.invoke(ManagementFactory.getOperatingSystemMXBean()); return (Long) result; } catch (Throwable t) { LOG.warn("Unexpected error when accessing file handle limit", t); return -1L; } } /** * Logs a information about the environment, like code revision, current user, java version, * and JVM parameters. * * @param log The logger to log the information to. * @param componentName The component name to mention in the log. * @param commandLineArgs The arguments accompanying the starting the component. */ public static void logEnvironmentInfo(Logger log, String componentName, String[] commandLineArgs) { if (log.isInfoEnabled()) { RevisionInformation rev = getRevisionInformation(); String version = getVersion(); String user = getUserRunning(); String jvmVersion = getJvmVersion(); String[] options = getJvmStartupOptionsArray(); String javaHome = System.getenv("JAVA_HOME"); long maxHeapMegabytes = getMaxJvmHeapMemory() >>> 20; log.info("--------------------------------------------------------------------------------"); log.info(" Starting " + componentName + " (Version: " + version + ", " + "Rev:" + rev.commitId + ", " + "Date:" + rev.commitDate + ")"); log.info(" Current user: " + user); log.info(" JVM: " + jvmVersion); log.info(" Maximum heap size: " + maxHeapMegabytes + " MiBytes"); log.info(" JAVA_HOME: " + (javaHome == null ? "(not set)" : javaHome)); if (options.length == 0) { log.info(" JVM Options: (none)"); } else { log.info(" JVM Options:"); for (String s: options) { log.info(" " + s); } } if (commandLineArgs == null || commandLineArgs.length == 0) { log.info(" Program Arguments: (none)"); } else { log.info(" Program Arguments:"); for (String s: commandLineArgs) { log.info(" " + s); } } log.info(" Classpath: " + System.getProperty("java.class.path")); log.info("--------------------------------------------------------------------------------"); } } // -------------------------------------------------------------------------------------------- /** Don't instantiate this class */ private EnvironmentInformation() {} // -------------------------------------------------------------------------------------------- /** * Revision information encapsulates information about the source code revision of the Flink * code. */ public static class RevisionInformation { /** The git commit id (hash) */ public final String commitId; /** The git commit date */ public final String commitDate; public RevisionInformation(String commitId, String commitDate) { this.commitId = commitId; this.commitDate = commitDate; } } } ================================================ FILE: src/main/java/io/mycat/memory/environment/Hardware.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.environment; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; import java.util.regex.Matcher; import java.util.regex.Pattern; /** * Convenience class to extract hardware specifics of the computer executing this class */ public class Hardware { private static final Logger LOG = LoggerFactory.getLogger(Hardware.class); private static final String LINUX_MEMORY_INFO_PATH = "/proc/meminfo"; private static final Pattern LINUX_MEMORY_REGEX = Pattern.compile("^MemTotal:\\s*(\\d+)\\s+kB$"); /** * Gets the number of CPU cores (hardware contexts) that the JVM has access to. * * @return The number of CPU cores. */ public static int getNumberCPUCores() { return Runtime.getRuntime().availableProcessors(); } /** * Returns the size of the physical memory in bytes. * * @return the size of the physical memory in bytes or -1 if * the size could not be determined */ public static long getSizeOfPhysicalMemory() { switch (OperatingSystem.getCurrentOperatingSystem()) { case LINUX: return getSizeOfPhysicalMemoryForLinux(); case WINDOWS: return getSizeOfPhysicalMemoryForWindows(); case MAC_OS: return getSizeOfPhysicalMemoryForMac(); case FREE_BSD: return getSizeOfPhysicalMemoryForFreeBSD(); case UNKNOWN: LOG.error("Cannot determine size of physical memory for unknown operating system"); return -1; default: LOG.error("Unrecognized OS: " + OperatingSystem.getCurrentOperatingSystem()); return -1; } } /** * Returns the size of the physical memory in bytes on a Linux-based * operating system. * * @return the size of the physical memory in bytes or -1 if * the size could not be determined */ private static long getSizeOfPhysicalMemoryForLinux() { try (BufferedReader lineReader = new BufferedReader(new FileReader(LINUX_MEMORY_INFO_PATH))) { String line; while ((line = lineReader.readLine()) != null) { Matcher matcher = LINUX_MEMORY_REGEX.matcher(line); if (matcher.matches()) { String totalMemory = matcher.group(1); return Long.parseLong(totalMemory) * 1024L; // Convert from kilobyte to byte } } // expected line did not come LOG.error("Cannot determine the size of the physical memory for Linux host (using '/proc/meminfo'). Unexpected format."); return -1; } catch (NumberFormatException e) { LOG.error("Cannot determine the size of the physical memory for Linux host (using '/proc/meminfo'). Unexpected format."); return -1; } catch (Throwable t) { LOG.error("Cannot determine the size of the physical memory for Linux host (using '/proc/meminfo'): " + t.getMessage(), t); return -1; } } /** * Returns the size of the physical memory in bytes on a Mac OS-based * operating system * * @return the size of the physical memory in bytes or -1 if * the size could not be determined */ private static long getSizeOfPhysicalMemoryForMac() { BufferedReader bi = null; try { Process proc = Runtime.getRuntime().exec("sysctl hw.memsize"); bi = new BufferedReader( new InputStreamReader(proc.getInputStream())); String line; while ((line = bi.readLine()) != null) { if (line.startsWith("hw.memsize")) { long memsize = Long.parseLong(line.split(":")[1].trim()); bi.close(); proc.destroy(); return memsize; } } } catch (Throwable t) { LOG.error("Cannot determine physical memory of machine for MacOS host: " + t.getMessage(), t); return -1; } finally { if (bi != null) { try { bi.close(); } catch (IOException ignored) {} } } return -1; } /** * Returns the size of the physical memory in bytes on FreeBSD. * * @return the size of the physical memory in bytes or -1 if * the size could not be determined */ private static long getSizeOfPhysicalMemoryForFreeBSD() { BufferedReader bi = null; try { Process proc = Runtime.getRuntime().exec("sysctl hw.physmem"); bi = new BufferedReader(new InputStreamReader(proc.getInputStream())); String line; while ((line = bi.readLine()) != null) { if (line.startsWith("hw.physmem")) { long memsize = Long.parseLong(line.split(":")[1].trim()); bi.close(); proc.destroy(); return memsize; } } LOG.error("Cannot determine the size of the physical memory for FreeBSD host (using 'sysctl hw.physmem')."); return -1; } catch (Throwable t) { LOG.error("Cannot determine the size of the physical memory for FreeBSD host (using 'sysctl hw.physmem'): " + t.getMessage(), t); return -1; } finally { if (bi != null) { try { bi.close(); } catch (IOException ignored) {} } } } /** * Returns the size of the physical memory in bytes on Windows. * * @return the size of the physical memory in bytes or -1 if * the size could not be determined */ private static long getSizeOfPhysicalMemoryForWindows() { BufferedReader bi = null; try { Process proc = Runtime.getRuntime().exec("wmic memorychip get capacity"); bi = new BufferedReader(new InputStreamReader(proc.getInputStream())); String line = bi.readLine(); if (line == null) { return -1L; } if (!line.startsWith("Capacity")) { return -1L; } long sizeOfPhyiscalMemory = 0L; while ((line = bi.readLine()) != null) { if (line.isEmpty()) { continue; } line = line.replaceAll(" ", ""); sizeOfPhyiscalMemory += Long.parseLong(line); } return sizeOfPhyiscalMemory; } catch (Throwable t) { LOG.error("Cannot determine the size of the physical memory for Windows host (using 'wmic memorychip'): " + t.getMessage(), t); return -1L; } finally { if (bi != null) { try { bi.close(); } catch (Throwable ignored) {} } } } // -------------------------------------------------------------------------------------------- private Hardware() {} } ================================================ FILE: src/main/java/io/mycat/memory/environment/HardwareDescription.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.environment; import java.io.Serializable; /** * A hardware description describes the resources available to a task manager. */ public final class HardwareDescription implements Serializable { private static final long serialVersionUID = 3380016608300325361L; /** The number of CPU cores available to the JVM on the compute node. */ private int numberOfCPUCores; /** The size of physical memory in bytes available on the compute node. */ private long sizeOfPhysicalMemory; /** The size of the JVM heap memory */ private long sizeOfJvmHeap; /** The size of the memory managed by the system for caching, hashing, sorting, ... */ private long sizeOfManagedMemory; /** * Public default constructor used for serialization process. */ public HardwareDescription() {} /** * Constructs a new hardware description object. * * @param numberOfCPUCores The number of CPU cores available to the JVM on the compute node. * @param sizeOfPhysicalMemory The size of physical memory in bytes available on the compute node. * @param sizeOfJvmHeap The size of the JVM heap memory. * @param sizeOfManagedMemory The size of the memory managed by the system for caching, hashing, sorting, ... */ public HardwareDescription(int numberOfCPUCores, long sizeOfPhysicalMemory, long sizeOfJvmHeap, long sizeOfManagedMemory) { this.numberOfCPUCores = numberOfCPUCores; this.sizeOfPhysicalMemory = sizeOfPhysicalMemory; this.sizeOfJvmHeap = sizeOfJvmHeap; this.sizeOfManagedMemory = sizeOfManagedMemory; } /** * Returns the number of CPU cores available to the JVM on the compute node. * * @return the number of CPU cores available to the JVM on the compute node */ public int getNumberOfCPUCores() { return this.numberOfCPUCores; } /** * Returns the size of physical memory in bytes available on the compute node. * * @return the size of physical memory in bytes available on the compute node */ public long getSizeOfPhysicalMemory() { return this.sizeOfPhysicalMemory; } /** * Returns the size of the JVM heap memory * * @return The size of the JVM heap memory */ public long getSizeOfJvmHeap() { return this.sizeOfJvmHeap; } /** * Returns the size of the memory managed by the system for caching, hashing, sorting, ... * * @return The size of the memory managed by the system. */ public long getSizeOfManagedMemory() { return this.sizeOfManagedMemory; } // -------------------------------------------------------------------------------------------- // Utils // -------------------------------------------------------------------------------------------- @Override public String toString() { return String.format("cores=%d, physMem=%d, heap=%d, managed=%d", numberOfCPUCores, sizeOfPhysicalMemory, sizeOfJvmHeap, sizeOfManagedMemory); } // -------------------------------------------------------------------------------------------- // Factory // -------------------------------------------------------------------------------------------- public static HardwareDescription extractFromSystem(long managedMemory) { final int numberOfCPUCores = Hardware.getNumberCPUCores(); final long sizeOfJvmHeap = Runtime.getRuntime().maxMemory(); final long sizeOfPhysicalMemory = Hardware.getSizeOfPhysicalMemory(); return new HardwareDescription(numberOfCPUCores, sizeOfPhysicalMemory, sizeOfJvmHeap, managedMemory); } } ================================================ FILE: src/main/java/io/mycat/memory/environment/OperatingSystem.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.environment; /** * An enumeration indicating the operating system that the JVM runs on. */ public enum OperatingSystem { LINUX, WINDOWS, MAC_OS, FREE_BSD, UNKNOWN; // ------------------------------------------------------------------------ /** * Gets the operating system that the JVM runs on from the java system properties. * this method returns UNKNOWN, if the operating system was not successfully determined. * * @return The enum constant for the operating system, or UNKNOWN, if it was not possible to determine. */ public static OperatingSystem getCurrentOperatingSystem() { return os; } /** * Checks whether the operating system this JVM runs on is Windows. * * @return true if the operating system this JVM runs on is * Windows, false otherwise */ public static boolean isWindows() { return getCurrentOperatingSystem() == WINDOWS; } /** * Checks whether the operating system this JVM runs on is Linux. * * @return true if the operating system this JVM runs on is * Linux, false otherwise */ public static boolean isLinux() { return getCurrentOperatingSystem() == LINUX; } /** * Checks whether the operating system this JVM runs on is Windows. * * @return true if the operating system this JVM runs on is * Windows, false otherwise */ public static boolean isMac() { return getCurrentOperatingSystem() == MAC_OS; } /** * Checks whether the operating system this JVM runs on is FreeBSD. * * @return true if the operating system this JVM runs on is * FreeBSD, false otherwise */ public static boolean isFreeBSD() { return getCurrentOperatingSystem() == FREE_BSD; } /** * The enum constant for the operating system. */ private static final OperatingSystem os = readOSFromSystemProperties(); /** * Parses the operating system that the JVM runs on from the java system properties. * If the operating system was not successfully determined, this method returns {@code UNKNOWN}. * * @return The enum constant for the operating system, or {@code UNKNOWN}, if it was not possible to determine. */ private static OperatingSystem readOSFromSystemProperties() { String osName = System.getProperty(OS_KEY); if (osName.startsWith(LINUX_OS_PREFIX)) { return LINUX; } if (osName.startsWith(WINDOWS_OS_PREFIX)) { return WINDOWS; } if (osName.startsWith(MAC_OS_PREFIX)) { return MAC_OS; } if (osName.startsWith(FREEBSD_OS_PREFIX)) { return FREE_BSD; } return UNKNOWN; } // -------------------------------------------------------------------------------------------- // Constants to extract the OS type from the java environment // -------------------------------------------------------------------------------------------- /** * The key to extract the operating system name from the system properties. */ private static final String OS_KEY = "os.name"; /** * The expected prefix for Linux operating systems. */ private static final String LINUX_OS_PREFIX = "Linux"; /** * The expected prefix for Windows operating systems. */ private static final String WINDOWS_OS_PREFIX = "Windows"; /** * The expected prefix for Mac OS operating systems. */ private static final String MAC_OS_PREFIX = "Mac"; /** * The expected prefix for FreeBSD. */ private static final String FREEBSD_OS_PREFIX = "FreeBSD"; } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/KVIterator.java ================================================ package io.mycat.memory.unsafe; import java.io.IOException; public abstract class KVIterator { public abstract boolean next() throws IOException; public abstract K getKey(); public abstract V getValue(); public abstract void close(); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/Platform.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe; import io.mycat.MycatServer; import io.mycat.memory.unsafe.utils.BytesTools; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import sun.misc.Cleaner; import sun.misc.Unsafe; import sun.nio.ch.DirectBuffer; import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.security.AccessController; import java.security.PrivilegedAction; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; public final class Platform { private final static Logger logger = LoggerFactory.getLogger(Platform.class); private static final Pattern MAX_DIRECT_MEMORY_SIZE_ARG_PATTERN = Pattern.compile("\\s*-XX:MaxDirectMemorySize\\s*=\\s*([0-9]+)\\s*([kKmMgG]?)\\s*$"); private static final Unsafe _UNSAFE; public static final int BYTE_ARRAY_OFFSET; public static final int SHORT_ARRAY_OFFSET; public static final int INT_ARRAY_OFFSET; public static final int LONG_ARRAY_OFFSET; public static final int FLOAT_ARRAY_OFFSET; public static final int DOUBLE_ARRAY_OFFSET; private static final long MAX_DIRECT_MEMORY; private static final boolean unaligned; public static final boolean littleEndian = ByteOrder.nativeOrder() .equals(ByteOrder.LITTLE_ENDIAN); public static final long sqlTimeout = MycatServer.getInstance().getConfig().getSystem().getSqlExecuteTimeout() * 1000L; static { boolean _unaligned; // use reflection to access unaligned field try { Class bitsClass = Class.forName("java.nio.Bits", false, ClassLoader.getSystemClassLoader()); Method unalignedMethod = bitsClass.getDeclaredMethod("unaligned"); unalignedMethod.setAccessible(true); _unaligned = Boolean.TRUE.equals(unalignedMethod.invoke(null)); } catch (Throwable t) { // We at least know x86 and x64 support unaligned access. String arch = System.getProperty("os.arch", ""); //noinspection DynamicRegexReplaceableByCompiledPattern _unaligned = arch.matches("^(i[3-6]86|x86(_64)?|x64|amd64)$"); } unaligned = _unaligned; MAX_DIRECT_MEMORY = maxDirectMemory(); } private static ClassLoader getSystemClassLoader() { return System.getSecurityManager() == null ? ClassLoader.getSystemClassLoader() : (ClassLoader) AccessController.doPrivileged(new PrivilegedAction() { public ClassLoader run() { return ClassLoader.getSystemClassLoader(); } }); } /** * GET MaxDirectMemory Size,from Netty Project! */ private static long maxDirectMemory() { long maxDirectMemory = 0L; Class t; try { t = Class.forName("sun.misc.VM", true, getSystemClassLoader()); Method runtimeClass = t.getDeclaredMethod("maxDirectMemory", new Class[0]); maxDirectMemory = ((Number) runtimeClass.invoke((Object) null, new Object[0])).longValue(); } catch (Throwable var8) { ; } if (maxDirectMemory > 0L) { return maxDirectMemory; } else { try { t = Class.forName("java.lang.management.ManagementFactory", true, getSystemClassLoader()); Class var10 = Class.forName("java.lang.management.RuntimeMXBean", true, getSystemClassLoader()); Object runtime = t.getDeclaredMethod("getRuntimeMXBean", new Class[0]).invoke((Object) null, new Object[0]); List vmArgs = (List) var10.getDeclaredMethod("getInputArguments", new Class[0]).invoke(runtime, new Object[0]); label41: for (int i = vmArgs.size() - 1; i >= 0; --i) { Matcher m = MAX_DIRECT_MEMORY_SIZE_ARG_PATTERN.matcher((CharSequence) vmArgs.get(i)); if (m.matches()) { maxDirectMemory = Long.parseLong(m.group(1)); switch (m.group(2).charAt(0)) { case 'G': case 'g': maxDirectMemory *= 1073741824L; break label41; case 'K': case 'k': maxDirectMemory *= 1024L; break label41; case 'M': case 'm': maxDirectMemory *= 1048576L; default: break label41; } } } } catch (Throwable var9) { logger.error(var9.getMessage()); } if (maxDirectMemory <= 0L) { maxDirectMemory = Runtime.getRuntime().maxMemory(); //System.out.println("maxDirectMemory: {} bytes (maybe)" + Long.valueOf(maxDirectMemory)); } else { //System.out.println("maxDirectMemory: {} bytes" + Long.valueOf(maxDirectMemory)); } return maxDirectMemory; } } public static long getMaxDirectMemory() { return MAX_DIRECT_MEMORY; } public static long getMaxHeapMemory() { return Runtime.getRuntime().maxMemory(); } /** * @return true when running JVM is having sun's Unsafe package available in it and underlying * system having unaligned-access capability. */ public static boolean unaligned() { return unaligned; } public static int getInt(Object object, long offset) { return _UNSAFE.getInt(object, offset); } public static void putInt(Object object, long offset, int value) { _UNSAFE.putInt(object, offset, value); } public static boolean getBoolean(Object object, long offset) { return _UNSAFE.getBoolean(object, offset); } public static void putBoolean(Object object, long offset, boolean value) { _UNSAFE.putBoolean(object, offset, value); } public static byte getByte(Object object, long offset) { return _UNSAFE.getByte(object, offset); } public static void putByte(Object object, long offset, byte value) { _UNSAFE.putByte(object, offset, value); } public static short getShort(Object object, long offset) { return _UNSAFE.getShort(object, offset); } public static void putShort(Object object, long offset, short value) { _UNSAFE.putShort(object, offset, value); } public static long getLong(Object object, long offset) { return _UNSAFE.getLong(object, offset); } public static void putLong(Object object, long offset, long value) { _UNSAFE.putLong(object, offset, value); } public static float getFloat(Object object, long offset) { return _UNSAFE.getFloat(object, offset); } public static void putFloat(Object object, long offset, float value) { _UNSAFE.putFloat(object, offset, value); } public static double getDouble(Object object, long offset) { return _UNSAFE.getDouble(object, offset); } public static void putDouble(Object object, long offset, double value) { _UNSAFE.putDouble(object, offset, value); } public static Object getObjectVolatile(Object object, long offset) { return _UNSAFE.getObjectVolatile(object, offset); } public static void putObjectVolatile(Object object, long offset, Object value) { _UNSAFE.putObjectVolatile(object, offset, value); } public static long allocateMemory(long size) { return _UNSAFE.allocateMemory(size); } public static void freeMemory(long address) { _UNSAFE.freeMemory(address); } public static long reallocateMemory(long address, long oldSize, long newSize) { long newMemory = _UNSAFE.allocateMemory(newSize); copyMemory(null, address, null, newMemory, oldSize); freeMemory(address); return newMemory; } /** * Uses internal JDK APIs to allocate a DirectByteBuffer while ignoring the JVM's * MaxDirectMemorySize limit (the default limit is too low and we do not want to require users * to increase it). */ @SuppressWarnings("unchecked") public static ByteBuffer allocateDirectBuffer(int size) { try { Class cls = Class.forName("java.nio.DirectByteBuffer"); Constructor constructor = cls.getDeclaredConstructor(Long.TYPE, Integer.TYPE); constructor.setAccessible(true); Field cleanerField = cls.getDeclaredField("cleaner"); cleanerField.setAccessible(true); final long memory = allocateMemory(size); ByteBuffer buffer = (ByteBuffer) constructor.newInstance(memory, size); Cleaner cleaner = Cleaner.create(buffer, new Runnable() { @Override public void run() { freeMemory(memory); } }); cleanerField.set(buffer, cleaner); return buffer; } catch (Exception e) { throwException(e); } throw new IllegalStateException("unreachable"); } public static void setMemory(long address, byte value, long size) { _UNSAFE.setMemory(address, size, value); } public static void copyMemory( Object src, long srcOffset, Object dst, long dstOffset, long length) { // Check if dstOffset is before or after srcOffset to determine if we should copy // forward or backwards. This is necessary in case src and dst overlap. if (dstOffset < srcOffset) { while (length > 0) { long size = Math.min(length, UNSAFE_COPY_THRESHOLD); _UNSAFE.copyMemory(src, srcOffset, dst, dstOffset, size); length -= size; srcOffset += size; dstOffset += size; } } else { srcOffset += length; dstOffset += length; while (length > 0) { // if backend db run time out ,this will be endless loop long size = Math.min(length, UNSAFE_COPY_THRESHOLD); long lbegin = System.currentTimeMillis(); srcOffset -= size; dstOffset -= size; _UNSAFE.copyMemory(src, srcOffset, dst, dstOffset, size); length -= size; long l = System.currentTimeMillis() - lbegin; if(l > sqlTimeout) { // when sql run timeout,break the loop logger.error("copyMemory timeout.loop(seconds):" + String.valueOf(l / 1000)); break; } } } } /** * Raises an exception bypassing compiler checks for checked exceptions. */ public static void throwException(Throwable t) { _UNSAFE.throwException(t); } /** * Limits the number of bytes to copy per {@link Unsafe#copyMemory(long, long, long)} to * allow safepoint polling during a large copy. */ private static final long UNSAFE_COPY_THRESHOLD = 1024L * 1024L; static { Unsafe unsafe; try { Field unsafeField = Unsafe.class.getDeclaredField("theUnsafe"); unsafeField.setAccessible(true); unsafe = (Unsafe) unsafeField.get(null); } catch (Throwable cause) { unsafe = null; } _UNSAFE = unsafe; if (_UNSAFE != null) { BYTE_ARRAY_OFFSET = _UNSAFE.arrayBaseOffset(byte[].class); SHORT_ARRAY_OFFSET = _UNSAFE.arrayBaseOffset(short[].class); INT_ARRAY_OFFSET = _UNSAFE.arrayBaseOffset(int[].class); LONG_ARRAY_OFFSET = _UNSAFE.arrayBaseOffset(long[].class); FLOAT_ARRAY_OFFSET = _UNSAFE.arrayBaseOffset(float[].class); DOUBLE_ARRAY_OFFSET = _UNSAFE.arrayBaseOffset(double[].class); } else { BYTE_ARRAY_OFFSET = 0; SHORT_ARRAY_OFFSET = 0; INT_ARRAY_OFFSET = 0; LONG_ARRAY_OFFSET = 0; FLOAT_ARRAY_OFFSET = 0; DOUBLE_ARRAY_OFFSET = 0; } } public static long objectFieldOffset(Field field) { return _UNSAFE.objectFieldOffset(field); } public static void putOrderedLong(Object object, long valueOffset, long initialValue) { _UNSAFE.putOrderedLong(object, valueOffset, initialValue); } public static void putLongVolatile(Object object, long valueOffset, long value) { _UNSAFE.putLongVolatile(object, valueOffset, value); } public static boolean compareAndSwapLong(Object object, long valueOffset, long expectedValue, long newValue) { return _UNSAFE.compareAndSwapLong(object, valueOffset, expectedValue, newValue); } public static int arrayBaseOffset(Class aClass) { return _UNSAFE.arrayBaseOffset(aClass); } public static int arrayIndexScale(Class aClass) { return _UNSAFE.arrayIndexScale(aClass); } public static void putOrderedInt(Object availableBuffer, long bufferAddress, int flag) { _UNSAFE.putOrderedInt(availableBuffer, bufferAddress, flag); } public static int getIntVolatile(Object availableBuffer, long bufferAddress) { return _UNSAFE.getIntVolatile(availableBuffer, bufferAddress); } public static Object getObject(Object entries, long l) { return _UNSAFE.getObject(entries, l); } public static char getChar(Object baseObj, long l) { return _UNSAFE.getChar(baseObj, l); } public static void putChar(Object baseObj, long l, char value) { _UNSAFE.putChar(baseObj, l, value); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/array/ByteArrayMethods.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.array; import io.mycat.memory.unsafe.Platform; public class ByteArrayMethods { private ByteArrayMethods() { // Private constructor, since this class only contains static methods. } /** Returns the next number greater or equal num that is power of 2. */ public static long nextPowerOf2(long num) { final long highBit = Long.highestOneBit(num); return (highBit == num) ? num : highBit << 1; } public static int roundNumberOfBytesToNearestWord(int numBytes) { int remainder = numBytes & 0x07; // This is equivalent to `numBytes % 8` if (remainder == 0) { return numBytes; } else { return numBytes + (8 - remainder); } } /** * Optimized byte array equality check for byte arrays. * @return true if the arrays are equal, false otherwise */ public static boolean arrayEquals( Object leftBase, long leftOffset, Object rightBase, long rightOffset, final long length) { int i = 0; while (i <= length - 8) { if (Platform.getLong(leftBase, leftOffset + i) != Platform.getLong(rightBase, rightOffset + i)) { return false; } i += 8; } while (i < length) { if (Platform.getByte(leftBase, leftOffset + i) != Platform.getByte(rightBase, rightOffset + i)) { return false; } i += 1; } return true; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/array/CharArray.java ================================================ package io.mycat.memory.unsafe.array; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.memory.MemoryBlock; import io.mycat.memory.unsafe.memory.mm.MemoryConsumer; /** * @author Hash Zhang * @version 1.0.0 * @date 2016/8/8 */ public class CharArray { private static final long WIDTH = 2; private final MemoryConsumer memoryConsumer; private final MemoryBlock memory; private final Object baseObj; private final long baseOffset; private final long length; public CharArray(MemoryBlock memory,MemoryConsumer memoryConsumer) { assert memory.size() < (long) Integer.MAX_VALUE * 2 : "Array size > 4 billion elements"; this.memory = memory; this.baseObj = memory.getBaseObject(); this.baseOffset = memory.getBaseOffset(); this.length = memory.size() / WIDTH; this.memoryConsumer = memoryConsumer; } public MemoryBlock memoryBlock() { return memory; } public Object getBaseObject() { return baseObj; } public long getBaseOffset() { return baseOffset; } /** * Returns the number of elements this array can hold. */ public long size() { return length; } /** * Fill this all with 0L. */ public void zeroOut() { for (long off = baseOffset; off < baseOffset + length * WIDTH; off += WIDTH) { Platform.putLong(baseObj, off, 0); } } /** * Sets the value at position {@code index}. */ public void set(int index, char value) { assert index >= 0 : "index (" + index + ") should >= 0"; assert index < length : "index (" + index + ") should < length (" + length + ")"; Platform.putChar(baseObj, baseOffset + index * WIDTH, value); } /** * Returns the value at position {@code index}. */ public char get(int index) { assert index >= 0 : "index (" + index + ") should >= 0"; assert index < length : "index (" + index + ") should < length (" + length + ")"; return Platform.getChar(baseObj, baseOffset + index * WIDTH); } public String toString() { StringBuilder stringBuilder = new StringBuilder((int) this.length); for (int i = 0; i < this.length; i++) { stringBuilder.append(get(i)); } return stringBuilder.toString(); } //todo:实现from string，使字符串数组可变 } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/array/LongArray.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.array; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.memory.MemoryBlock; /** * An array of long values. Compared with native JVM arrays, this: *
    *
  • supports using both in-heap and off-heap memory
    • *
  • has no bound checking, and thus can crash the JVM process when assert is turned off
    • *
*/ public final class LongArray { // This is a long so that we perform long multiplications when computing offsets. private static final long WIDTH = 8; private final MemoryBlock memory; private final Object baseObj; private final long baseOffset; private final long length; public LongArray(MemoryBlock memory) { assert memory.size() < (long) Integer.MAX_VALUE * 8: "Array size > 4 billion elements"; this.memory = memory; this.baseObj = memory.getBaseObject(); this.baseOffset = memory.getBaseOffset(); this.length = memory.size() / WIDTH; } public MemoryBlock memoryBlock() { return memory; } public Object getBaseObject() { return baseObj; } public long getBaseOffset() { return baseOffset; } /** * Returns the number of elements this array can hold. */ public long size() { return length; } /** * Fill this all with 0L. */ public void zeroOut() { for (long off = baseOffset; off < baseOffset + length * WIDTH; off += WIDTH) { Platform.putLong(baseObj, off, 0); } } /** * Sets the value at position {@code index}. */ public void set(int index, long value) { assert index >= 0 : "index (" + index + ") should >= 0"; assert index < length : "index (" + index + ") should < length (" + length + ")"; Platform.putLong(baseObj, baseOffset + index * WIDTH, value); } /** * Returns the value at position {@code index}. */ public long get(int index) { assert index >= 0 : "index (" + index + ") should >= 0"; assert index < length : "index (" + index + ") should < length (" + length + ")"; return Platform.getLong(baseObj, baseOffset + index * WIDTH); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/bitset/BitSetMethods.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.bitset; import io.mycat.memory.unsafe.Platform; /** * Methods for working with fixed-size uncompressed bitsets. * * We assume that the bitset data is word-aligned (that is, a multiple of 8 bytes in length). * * Each bit occupies exactly one bit of storage. */ public final class BitSetMethods { private static final long WORD_SIZE = 8; private BitSetMethods() { // Make the default constructor private, since this only holds static methods. } /** * Sets the bit at the specified index to {@code true}. */ public static void set(Object baseObject, long baseOffset, int index) { assert index >= 0 : "index (" + index + ") should >= 0"; final long mask = 1L << (index & 0x3f); // mod 64 and shift final long wordOffset = baseOffset + (index >> 6) * WORD_SIZE; final long word = Platform.getLong(baseObject, wordOffset); Platform.putLong(baseObject, wordOffset, word | mask); } /** * Sets the bit at the specified index to {@code false}. */ public static void unset(Object baseObject, long baseOffset, int index) { assert index >= 0 : "index (" + index + ") should >= 0"; final long mask = 1L << (index & 0x3f); // mod 64 and shift final long wordOffset = baseOffset + (index >> 6) * WORD_SIZE; final long word = Platform.getLong(baseObject, wordOffset); Platform.putLong(baseObject, wordOffset, word & ~mask); } /** * Returns {@code true} if the bit is set at the specified index. */ public static boolean isSet(Object baseObject, long baseOffset, int index) { assert index >= 0 : "index (" + index + ") should >= 0"; final long mask = 1L << (index & 0x3f); // mod 64 and shift final long wordOffset = baseOffset + (index >> 6) * WORD_SIZE; final long word = Platform.getLong(baseObject, wordOffset); return (word & mask) != 0; } /** * Returns {@code true} if any bit is set. */ public static boolean anySet(Object baseObject, long baseOffset, long bitSetWidthInWords) { long addr = baseOffset; for (int i = 0; i < bitSetWidthInWords; i++, addr += WORD_SIZE) { if (Platform.getLong(baseObject, addr) != 0) { return true; } } return false; } /** * Returns the index of the first bit that is set to true that occurs on or after the * specified starting index. If no such bit exists then {@code -1} is returned. *

* To iterate over the true bits in a BitSet, use the following loop: * 

   * 
   *  for (long i = bs.nextSetBit(0, sizeInWords); i >= 0;
   *    i = bs.nextSetBit(i + 1, sizeInWords)) {
   *    // operate on index i here
   *  }
   * 
   *
* * @param fromIndex the index to start checking from (inclusive) * @param bitsetSizeInWords the size of the bitset, measured in 8-byte words * @return the index of the next set bit, or -1 if there is no such bit */ public static int nextSetBit( Object baseObject, long baseOffset, int fromIndex, int bitsetSizeInWords) { int wi = fromIndex >> 6; if (wi >= bitsetSizeInWords) { return -1; } // Try to find the next set bit in the current word final int subIndex = fromIndex & 0x3f; long word = Platform.getLong(baseObject, baseOffset + wi * WORD_SIZE) >> subIndex; if (word != 0) { return (wi << 6) + subIndex + Long.numberOfTrailingZeros(word); } // Find the next set bit in the rest of the words wi += 1; while (wi < bitsetSizeInWords) { word = Platform.getLong(baseObject, baseOffset + wi * WORD_SIZE); if (word != 0) { return (wi << 6) + Long.numberOfTrailingZeros(word); } wi += 1; } return -1; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/hash/Murmur3_x86_32.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.hash; import io.mycat.memory.unsafe.Platform; /** * 32-bit Murmur3 hasher. This is based on Guava's Murmur3_32HashFunction. */ public final class Murmur3_x86_32 { private static final int C1 = 0xcc9e2d51; private static final int C2 = 0x1b873593; private final int seed; public Murmur3_x86_32(int seed) { this.seed = seed; } @Override public String toString() { return "Murmur3_32(seed=" + seed + ")"; } public int hashInt(int input) { return hashInt(input, seed); } public static int hashInt(int input, int seed) { int k1 = mixK1(input); int h1 = mixH1(seed, k1); return fmix(h1, 4); } public int hashUnsafeWords(Object base, long offset, int lengthInBytes) { return hashUnsafeWords(base, offset, lengthInBytes, seed); } public static int hashUnsafeWords(Object base, long offset, int lengthInBytes, int seed) { // This is based on Guava's `Murmur32_Hasher.processRemaining(ByteBuffer)` method. assert (lengthInBytes % 8 == 0): "lengthInBytes must be a multiple of 8 (word-aligned)"; int h1 = hashBytesByInt(base, offset, lengthInBytes, seed); return fmix(h1, lengthInBytes); } public static int hashUnsafeBytes(Object base, long offset, int lengthInBytes, int seed) { assert (lengthInBytes >= 0): "lengthInBytes cannot be negative"; int lengthAligned = lengthInBytes - lengthInBytes % 4; int h1 = hashBytesByInt(base, offset, lengthAligned, seed); for (int i = lengthAligned; i < lengthInBytes; i++) { int halfWord = Platform.getByte(base, offset + i); int k1 = mixK1(halfWord); h1 = mixH1(h1, k1); } return fmix(h1, lengthInBytes); } private static int hashBytesByInt(Object base, long offset, int lengthInBytes, int seed) { assert (lengthInBytes % 4 == 0); int h1 = seed; for (int i = 0; i < lengthInBytes; i += 4) { int halfWord = Platform.getInt(base, offset + i); int k1 = mixK1(halfWord); h1 = mixH1(h1, k1); } return h1; } public int hashLong(long input) { return hashLong(input, seed); } public static int hashLong(long input, int seed) { int low = (int) input; int high = (int) (input >>> 32); int k1 = mixK1(low); int h1 = mixH1(seed, k1); k1 = mixK1(high); h1 = mixH1(h1, k1); return fmix(h1, 8); } private static int mixK1(int k1) { k1 *= C1; k1 = Integer.rotateLeft(k1, 15); k1 *= C2; return k1; } private static int mixH1(int h1, int k1) { h1 ^= k1; h1 = Integer.rotateLeft(h1, 13); h1 = h1 * 5 + 0xe6546b64; return h1; } // Finalization mix - force all bits of a hash block to avalanche private static int fmix(int h1, int length) { h1 ^= length; h1 ^= h1 >>> 16; h1 *= 0x85ebca6b; h1 ^= h1 >>> 13; h1 *= 0xc2b2ae35; h1 ^= h1 >>> 16; return h1; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/map/BytesToBytesMap.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.map; import com.google.common.annotations.VisibleForTesting; import com.google.common.io.Closeables; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.array.ByteArrayMethods; import io.mycat.memory.unsafe.array.LongArray; import io.mycat.memory.unsafe.hash.Murmur3_x86_32; import io.mycat.memory.unsafe.memory.MemoryBlock; import io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager; import io.mycat.memory.unsafe.memory.mm.MemoryConsumer; import io.mycat.memory.unsafe.storage.DataNodeDiskManager; import io.mycat.memory.unsafe.storage.SerializerManager; import io.mycat.memory.unsafe.utils.sort.UnsafeSorterSpillReader; import io.mycat.memory.unsafe.utils.sort.UnsafeSorterSpillWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nullable; import java.io.File; import java.io.IOException; import java.util.Iterator; import java.util.LinkedList; /** * An append-only hash map where keys and values are contiguous regions of bytes. * * This is backed by a power-of-2-sized hash table, using quadratic probing with triangular numbers, * which is guaranteed to exhaust the space. * * The map can support up to 2^29 keys. If the key cardinality is higher than this, you should * probably be using sorting instead of hashing for better cache locality. * * The key and values under the hood are stored together, in the following format: * Bytes 0 to 4: len(k) (key length in bytes) + len(v) (value length in bytes) + 4 * Bytes 4 to 8: len(k) * Bytes 8 to 8 + len(k): key data * Bytes 8 + len(k) to 8 + len(k) + len(v): value data * Bytes 8 + len(k) + len(v) to 8 + len(k) + len(v) + 8: pointer to next pair * * This means that the first four bytes store the entire record (key + value) length. This format * is compatible with {@link io.mycat.memory.unsafe.utils.sort.UnsafeExternalSorter}, * so we can pass records from this map directly into the sorter to sort records in place. */ public final class BytesToBytesMap extends MemoryConsumer { private final Logger logger = LoggerFactory.getLogger(BytesToBytesMap.class); private static final HashMapGrowthStrategy growthStrategy = HashMapGrowthStrategy.DOUBLING; private final DataNodeMemoryManager dataNodeMemoryManager; /** * A linked list for tracking all allocated data pages so that we can free all of our memory. */ private final LinkedList dataPages = new LinkedList(); /** * The data page that will be used to store keys and values for new hashtable entries. When this * page becomes full, a new page will be allocated and this pointer will change to point to that * new page. */ private MemoryBlock currentPage = null; /** * Offset into `currentPage` that points to the location where new data can be inserted into * the page. This does not incorporate the page's base offset. */ private long pageCursor = 0; /** * The maximum number of keys that BytesToBytesMap supports. The hash table has to be * power-of-2-sized and its backing Java array can contain at most (1 << 30) elements, * since that's the largest power-of-2 that's less than Integer.MAX_VALUE. We need two long array * entries per key, giving us a maximum capacity of (1 << 29). */ @VisibleForTesting public static final int MAX_CAPACITY = (1 << 29); // This choice of page table size and page size means that we can address up to 500 gigabytes // of memory. /** * A single array to store the key and value. * * Position {@code 2 * i} in the array is used to track a pointer to the key at index {@code i}, * while position {@code 2 * i + 1} in the array holds key's full 32-bit hashcode. */ @Nullable private LongArray longArray; // TODO: we're wasting 32 bits of space here; we can probably store fewer bits of the hashcode // and exploit word-alignment to use fewer bits to hold the address. This might let us store // only one long per map entry, increasing the chance that this array will fit in cache at the // expense of maybe performing more lookups if we have hash collisions. Say that we stored only // 27 bits of the hashcode and 37 bits of the address. 37 bits is enough to address 1 terabyte // of RAM given word-alignment. If we use 13 bits of this for our page table, that gives us a // maximum page size of 2^24 * 8 = ~134 megabytes per page. This change will require us to store // full base addresses in the page table for off-heap mode so that we can reconstruct the full // absolute memory addresses. /** * Whether or not the longArray can grow. We will not insert more elements if it's false. */ private boolean canGrowArray = true; private final double loadFactor; /** * The size of the data pages that hold key and value data. Map entries cannot span multiple * pages, so this limits the maximum entry size. */ private final long pageSizeBytes; /** * Number of keys defined in the map. */ private int numKeys; /** * Number of values defined in the map. A key could have multiple values. */ private int numValues; /** * The map will be expanded once the number of keys exceeds this threshold. */ private int growthThreshold; /** * Mask for truncating hashcodes so that they do not exceed the long array's size. * This is a strength reduction optimization; we're essentially performing a modulus operation, * but doing so with a bitmask because this is a power-of-2-sized hash map. */ private int mask; /** * Return value of {@link BytesToBytesMap#lookup(Object, long, int)}. */ private final Location loc; private final boolean enablePerfMetrics; private long timeSpentResizingNs = 0; private long numProbes = 0; private long numKeyLookups = 0; private long numHashCollisions = 0; private long peakMemoryUsedBytes = 0L; private final DataNodeDiskManager blockManager; private final SerializerManager serializerManager; private volatile MapIterator destructiveIterator = null; private LinkedList spillWriters = new LinkedList(); public BytesToBytesMap( DataNodeMemoryManager dataNodeMemoryManager, DataNodeDiskManager blockManager, SerializerManager serializerManager, int initialCapacity, double loadFactor, long pageSizeBytes, boolean enablePerfMetrics) { super(dataNodeMemoryManager, pageSizeBytes); this.dataNodeMemoryManager = dataNodeMemoryManager; this.blockManager = blockManager; this.serializerManager = serializerManager; this.loadFactor = loadFactor; this.loc = new Location(); this.pageSizeBytes = pageSizeBytes; this.enablePerfMetrics = enablePerfMetrics; if (initialCapacity <= 0) { throw new IllegalArgumentException("Initial capacity must be greater than 0"); } if (initialCapacity > MAX_CAPACITY) { throw new IllegalArgumentException( "Initial capacity " + initialCapacity + " exceeds maximum capacity of " + MAX_CAPACITY); } if (pageSizeBytes > DataNodeMemoryManager.MAXIMUM_PAGE_SIZE_BYTES) { throw new IllegalArgumentException("Page size " + pageSizeBytes + " cannot exceed " + DataNodeMemoryManager.MAXIMUM_PAGE_SIZE_BYTES); } allocate(initialCapacity); } public BytesToBytesMap( DataNodeMemoryManager dataNodeMemoryManager, int initialCapacity, long pageSizeBytes) { this(dataNodeMemoryManager, initialCapacity, pageSizeBytes, false); } public BytesToBytesMap( DataNodeMemoryManager dataNodeMemoryManager, int initialCapacity, long pageSizeBytes, boolean enablePerfMetrics) { this( dataNodeMemoryManager, null, null, initialCapacity, 0.70, pageSizeBytes, enablePerfMetrics); } /** * Returns the number of keys defined in the map. */ public int numKeys() { return numKeys; } /** * Returns the number of values defined in the map. A key could have multiple values. */ public int numValues() { return numValues; } public final class MapIterator implements Iterator { private int numRecords; private final Location loc; private MemoryBlock currentPage = null; private int recordsInPage = 0; private Object pageBaseObject; private long offsetInPage; // If this iterator destructive or not. When it is true, it frees each page as it moves onto // next one. private boolean destructive = false; private UnsafeSorterSpillReader reader = null; private MapIterator(int numRecords, Location loc, boolean destructive) { this.numRecords = numRecords; this.loc = loc; this.destructive = destructive; if (destructive) { destructiveIterator = this; } } private void advanceToNextPage() { synchronized (this) { int nextIdx = dataPages.indexOf(currentPage) + 1; if (destructive && currentPage != null) { dataPages.remove(currentPage); freePage(currentPage); nextIdx --; } if (dataPages.size() > nextIdx) { currentPage = dataPages.get(nextIdx); pageBaseObject = currentPage.getBaseObject(); offsetInPage = currentPage.getBaseOffset(); recordsInPage = Platform.getInt(pageBaseObject, offsetInPage); offsetInPage += 4; } else { currentPage = null; if (reader != null) { // remove the spill file from disk File file = spillWriters.removeFirst().getFile(); if (file != null && file.exists()) { if (!file.delete()) { logger.error("Was unable to delete spill file {}", file.getAbsolutePath()); } } } try { Closeables.close(reader, /* swallowIOException = */ false); if(spillWriters.size()>0) { reader = spillWriters.getFirst().getReader(serializerManager); } recordsInPage = -1; } catch (IOException e) { // Scala iterator does not handle exception Platform.throwException(e); } } } } @Override public boolean hasNext() { if (numRecords == 0) { if (reader != null) { // remove the spill file from disk File file = spillWriters.removeFirst().getFile(); if (file != null && file.exists()) { if (!file.delete()) { logger.error("Was unable to delete spill file {}", file.getAbsolutePath()); } } } } return numRecords > 0; } @Override public Location next() { if (recordsInPage == 0) { advanceToNextPage(); } numRecords--; if (currentPage != null) { int totalLength = Platform.getInt(pageBaseObject, offsetInPage); loc.with(currentPage, offsetInPage); // [total size] [key size] [key] [value] [pointer to next] offsetInPage += 4 + totalLength + 8; recordsInPage --; return loc; } else { assert(reader != null); // if(reader == null) // return null; if (!reader.hasNext()) { advanceToNextPage(); } try { reader.loadNext(); } catch (IOException e) { try { reader.close(); } catch(IOException e2) { logger.error("Error while closing spill reader", e2); } // Scala iterator does not handle exception Platform.throwException(e); } loc.with(reader.getBaseObject(), reader.getBaseOffset(), reader.getRecordLength()); return loc; } } public long spill(long numBytes) throws IOException { synchronized (this) { if (!destructive || dataPages.size() == 1) { return 0L; } long released = 0L; while (dataPages.size() > 0) { MemoryBlock block = dataPages.getLast(); // The currentPage is used, cannot be released if (block == currentPage) { break; } Object base = block.getBaseObject(); long offset = block.getBaseOffset(); int numRecords = Platform.getInt(base, offset); offset += 4; final UnsafeSorterSpillWriter writer = new UnsafeSorterSpillWriter(blockManager, 32 * 1024, numRecords); while (numRecords > 0) { int length = Platform.getInt(base, offset); writer.write(base, offset + 4, length, 0); offset += 4 + length + 8; numRecords--; } writer.close(); spillWriters.add(writer); dataPages.removeLast(); released += block.size(); freePage(block); if (released >= numBytes) { break; } } return released; } } @Override public void remove() { throw new UnsupportedOperationException(); } } /** * Returns an iterator for iterating over the entries of this map. * * For efficiency, all calls to `next()` will return the same {@link Location} object. * * If any other lookups or operations are performed on this map while iterating over it, including * `lookup()`, the behavior of the returned iterator is undefined. */ public MapIterator iterator() { return new MapIterator(numValues, loc, false); } /** * Returns a destructive iterator for iterating over the entries of this map. It frees each page * as it moves onto next one. Notice: it is illegal to call any method on the map after * `destructiveIterator()` has been called. * * For efficiency, all calls to `next()` will return the same {@link Location} object. * * If any other lookups or operations are performed on this map while iterating over it, including * `lookup()`, the behavior of the returned iterator is undefined. */ public MapIterator destructiveIterator() { return new MapIterator(numValues, loc, true); } /** * Looks up a key, and return a {@link Location} handle that can be used to map existence * and read/write values. * * This function always return the same {@link Location} instance to avoid object allocation. */ public Location lookup(Object keyBase, long keyOffset, int keyLength) { safeLookup(keyBase, keyOffset, keyLength, loc, Murmur3_x86_32.hashUnsafeWords(keyBase, keyOffset, keyLength, 42)); return loc; } /** * Looks up a key, and return a {@link Location} handle that can be used to map existence * and read/write values. * * This function always return the same {@link Location} instance to avoid object allocation. */ public Location lookup(Object keyBase, long keyOffset, int keyLength, int hash) { safeLookup(keyBase, keyOffset, keyLength, loc, hash); return loc; } /** * Looks up a key, and saves the result in provided `loc`. * * This is a thread-safe version of `lookup`, could be used by multiple threads. */ public void safeLookup(Object keyBase, long keyOffset, int keyLength, Location loc, int hash) { assert(longArray != null); if (enablePerfMetrics) { numKeyLookups++; } int pos = hash & mask; int step = 1; while (true) { if (enablePerfMetrics) { numProbes++; } if (longArray.get(pos * 2) == 0) { // This is a new key. loc.with(pos, hash, false); return; } else { long stored = longArray.get(pos * 2 + 1); /** * hash相等 */ if ((int) (stored) == hash) { // Full hash code matches.Let's compare the keys for equality. loc.with(pos,hash,true); /** * 比较key的值 */ if (loc.getKeyLength() == keyLength) { final boolean areEqual = ByteArrayMethods.arrayEquals( keyBase, keyOffset, loc.getKeyBase(), loc.getKeyOffset(), keyLength ); if (areEqual) { return; } else { if (enablePerfMetrics) { numHashCollisions++; } } } } } pos = (pos + step) & mask; step++; } } /** * Handle returned by {@link BytesToBytesMap#lookup(Object, long, int)} function. */ public final class Location { /** An index into the hash map's Long array */ private int pos; /** True if this location points to a position where a key is defined, false otherwise */ private boolean isDefined; /** * The hashcode of the most recent key passed to * {@link BytesToBytesMap#lookup(Object, long, int, int)}. Caching this hashcode here allows us * to avoid re-hashing the key when storing a value for that key. */ private int keyHashcode; private Object baseObject; // the base object for key and value private long keyOffset; private int keyLength; private long valueOffset; private int valueLength; /** * Memory page containing the record. Only set if created by {@link BytesToBytesMap#iterator()}. */ @Nullable private MemoryBlock memoryPage; private void updateAddressesAndSizes(long fullKeyAddress) { updateAddressesAndSizes( dataNodeMemoryManager.getPage(fullKeyAddress), dataNodeMemoryManager.getOffsetInPage(fullKeyAddress)); } private void updateAddressesAndSizes(final Object base, long offset) { baseObject = base; final int totalLength = Platform.getInt(base, offset); offset += 4; keyLength = Platform.getInt(base, offset); offset += 4; keyOffset = offset; valueOffset = offset + keyLength; valueLength = totalLength - keyLength - 4; } private Location with(int pos, int keyHashcode, boolean isDefined) { assert(longArray != null); this.pos = pos; this.isDefined = isDefined; this.keyHashcode = keyHashcode; if (isDefined) { final long fullKeyAddress = longArray.get(pos * 2); updateAddressesAndSizes(fullKeyAddress); } return this; } private Location with(MemoryBlock page, long offsetInPage) { this.isDefined = true; this.memoryPage = page; updateAddressesAndSizes(page.getBaseObject(), offsetInPage); return this; } /** * This is only used for spilling */ private Location with(Object base, long offset, int length) { this.isDefined = true; this.memoryPage = null; baseObject = base; keyOffset = offset + 4; keyLength = Platform.getInt(base, offset); valueOffset = offset + 4 + keyLength; valueLength = length - 4 - keyLength; return this; } /** * Find the next pair that has the same key as current one. */ public boolean nextValue() { assert isDefined; long nextAddr = Platform.getLong(baseObject, valueOffset + valueLength); if (nextAddr == 0) { return false; } else { updateAddressesAndSizes(nextAddr); return true; } } /** * Returns the memory page that contains the current record. * This is only valid if this is returned by {@link BytesToBytesMap#iterator()}. */ public MemoryBlock getMemoryPage() { return this.memoryPage; } /** * Returns true if the key is defined at this position, and false otherwise. */ public boolean isDefined() { return isDefined; } /** * Returns the base object for key. */ public Object getKeyBase() { assert (isDefined); return baseObject; } /** * Returns the offset for key. */ public long getKeyOffset() { assert (isDefined); return keyOffset; } /** * Returns the base object for value. */ public Object getValueBase() { assert (isDefined); return baseObject; } /** * Returns the offset for value. */ public long getValueOffset() { assert (isDefined); return valueOffset; } /** * Returns the length of the key defined at this position. * Unspecified behavior if the key is not defined. */ public int getKeyLength() { assert (isDefined); return keyLength; } /** * Returns the length of the value defined at this position. * Unspecified behavior if the key is not defined. */ public int getValueLength() { assert (isDefined); return valueLength; } /** * Append a new value for the key. This method could be called multiple times for a given key. * The return value indicates whether the put succeeded or whether it failed because additional * memory could not be acquired. *

* It is only valid to call this method immediately after calling `lookup()` using the same key. *

*

* The key and value must be word-aligned (that is, their sizes must multiples of 8). *

*

* After calling this method, calls to `get[Key|Value]Address()` and `get[Key|Value]Length` * will return information on the data stored by this `append` call. *

*

* As an example usage, here's the proper way to store a new key: *

* 
     *   Location loc = map.lookup(keyBase, keyOffset, keyLength);
     *   if (!loc.isDefined()) {
     *     if (!loc.append(keyBase, keyOffset, keyLength, ...)) {
     *       // handle failure to grow map (by spilling, for example)
     *     }
     *   }
     *
*

* Unspecified behavior if the key is not defined. *

* * @return true if the put() was successful and false if the put() failed because memory could * not be acquired. */ public boolean append(Object kbase, long koff, int klen, Object vbase, long voff, int vlen) { assert (klen % 8 == 0); assert (vlen % 8 == 0); assert (longArray != null); if (numKeys == MAX_CAPACITY // The map could be reused from last spill (because of no enough memory to grow), // then we don't try to grow again if hit the `growthThreshold`. || !canGrowArray && numKeys > growthThreshold) { return false; } // Here, we'll copy the data into our data pages. Because we only store a relative offset from // the key address instead of storing the absolute address of the value, the key and value // must be stored in the same memory page. // (8 byte key length) (key) (value) (8 byte pointer to next value) final long recordLength = 8 + klen + vlen + 8; if (currentPage == null || currentPage.size() - pageCursor < recordLength) { if (!acquireNewPage(recordLength + 4L)) { return false; } } // --- Append the key and value data to the current data page -------------------------------- final Object base = currentPage.getBaseObject(); long offset = currentPage.getBaseOffset() + pageCursor; final long recordOffset = offset; Platform.putInt(base, offset, klen + vlen + 4); Platform.putInt(base, offset + 4, klen); offset += 8; Platform.copyMemory(kbase, koff, base, offset, klen); offset += klen; Platform.copyMemory(vbase, voff, base, offset, vlen); offset += vlen; // put this value at the beginning of the list Platform.putLong(base, offset, isDefined ? longArray.get(pos * 2) : 0); // --- Update bookkeeping data structures ---------------------------------------------------- offset = currentPage.getBaseOffset(); Platform.putInt(base, offset, Platform.getInt(base, offset) + 1); pageCursor += recordLength; final long storedKeyAddress = dataNodeMemoryManager.encodePageNumberAndOffset( currentPage, recordOffset); longArray.set(pos * 2, storedKeyAddress); updateAddressesAndSizes(storedKeyAddress); numValues++; if (!isDefined) { numKeys++; longArray.set(pos * 2 + 1, keyHashcode); isDefined = true; if (numKeys > growthThreshold && longArray.size() < MAX_CAPACITY) { try { growAndRehash(); } catch (OutOfMemoryError oom) { canGrowArray = false; } } } return true; } } /** * Acquire a new page from the memory manager. * @return whether there is enough space to allocate the new page. */ private boolean acquireNewPage(long required) { try { currentPage = allocatePage(required); } catch (OutOfMemoryError e) { return false; } dataPages.add(currentPage); Platform.putInt(currentPage.getBaseObject(), currentPage.getBaseOffset(), 0); pageCursor = 4; return true; } @Override public long spill(long size, MemoryConsumer trigger) throws IOException { if (trigger != this && destructiveIterator != null) { return destructiveIterator.spill(size); } return 0L; } /** * Allocate new data structures for this map. When calling this outside of the constructor, * make sure to keep references to the old data structures so that you can free them. * * @param capacity the new map capacity */ private void allocate(int capacity) { assert (capacity >= 0); capacity = Math.max((int) Math.min(MAX_CAPACITY, ByteArrayMethods.nextPowerOf2(capacity)), 64); assert (capacity <= MAX_CAPACITY); longArray = allocateLongArray(capacity * 2); longArray.zeroOut(); this.growthThreshold = (int) (capacity * loadFactor); this.mask = capacity - 1; } /** * Free all allocated memory associated with this map, including the storage for keys and values * as well as the hash map array itself. * * This method is idempotent and can be called multiple times. */ public void free() { updatePeakMemoryUsed(); if (longArray != null) { freeLongArray(longArray); longArray = null; } Iterator dataPagesIterator = dataPages.iterator(); while (dataPagesIterator.hasNext()) { MemoryBlock dataPage = dataPagesIterator.next(); dataPagesIterator.remove(); freePage(dataPage); } assert(dataPages.isEmpty()); while (!spillWriters.isEmpty()) { File file = spillWriters.removeFirst().getFile(); if (file != null && file.exists()) { if (!file.delete()) { logger.error("Was unable to delete spill file {}", file.getAbsolutePath()); } } } } public DataNodeMemoryManager getDataNodeMemoryManager() { return dataNodeMemoryManager; } public long getPageSizeBytes() { return pageSizeBytes; } /** * Returns the total amount of memory, in bytes, consumed by this map's managed structures. */ public long getTotalMemoryConsumption() { long totalDataPagesSize = 0L; for (MemoryBlock dataPage : dataPages) { totalDataPagesSize += dataPage.size(); } return totalDataPagesSize + ((longArray != null) ? longArray.memoryBlock().size() : 0L); } private void updatePeakMemoryUsed() { long mem = getTotalMemoryConsumption(); if (mem > peakMemoryUsedBytes) { peakMemoryUsedBytes = mem; } } /** * Return the peak memory used so far, in bytes. */ public long getPeakMemoryUsedBytes() { updatePeakMemoryUsed(); return peakMemoryUsedBytes; } /** * Returns the total amount of time spent resizing this map (in nanoseconds). */ public long getTimeSpentResizingNs() { if (!enablePerfMetrics) { throw new IllegalStateException(); } return timeSpentResizingNs; } /** * Returns the average number of probes per key lookup. */ public double getAverageProbesPerLookup() { if (!enablePerfMetrics) { throw new IllegalStateException(); } return (1.0 * numProbes) / numKeyLookups; } public long getNumHashCollisions() { if (!enablePerfMetrics) { throw new IllegalStateException(); } return numHashCollisions; } @VisibleForTesting public int getNumDataPages() { return dataPages.size(); } /** * Returns the underline long[] of longArray. */ public LongArray getArray() { assert(longArray != null); return longArray; } /** * Reset this map to initialized state. */ public void reset() { numKeys = 0; numValues = 0; longArray.zeroOut(); while (dataPages.size() > 0) { MemoryBlock dataPage = dataPages.removeLast(); freePage(dataPage); } currentPage = null; pageCursor = 0; } /** * Grows the size of the hash table and re-hash everything. */ @VisibleForTesting void growAndRehash() { assert(longArray != null); long resizeStartTime = -1; if (enablePerfMetrics) { resizeStartTime = System.nanoTime(); } // Store references to the old data structures to be used when we re-hash final LongArray oldLongArray = longArray; final int oldCapacity = (int) oldLongArray.size() / 2; // Allocate the new data structures allocate(Math.min(growthStrategy.nextCapacity(oldCapacity), MAX_CAPACITY)); // Re-mask (we don't recompute the hashcode because we stored all 32 bits of it) for (int i = 0; i < oldLongArray.size(); i += 2) { final long keyPointer = oldLongArray.get(i); if (keyPointer == 0) { continue; } final int hashcode = (int) oldLongArray.get(i + 1); int newPos = hashcode & mask; int step = 1; while (longArray.get(newPos * 2) != 0) { newPos = (newPos + step) & mask; step++; } longArray.set(newPos * 2, keyPointer); longArray.set(newPos * 2 + 1, hashcode); } freeLongArray(oldLongArray); if (enablePerfMetrics) { timeSpentResizingNs += System.nanoTime() - resizeStartTime; } } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/map/HashMapGrowthStrategy.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.map; /** * Interface that defines how we can grow the size of a hash map when it is over a threshold. */ public interface HashMapGrowthStrategy { int nextCapacity(int currentCapacity); /** * Double the size of the hash map every time. */ HashMapGrowthStrategy DOUBLING = new Doubling(); class Doubling implements HashMapGrowthStrategy { @Override public int nextCapacity(int currentCapacity) { assert (currentCapacity > 0); // Guard against overflow return (currentCapacity * 2 > 0) ? (currentCapacity * 2) : Integer.MAX_VALUE; } } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/map/UnsafeFixedWidthAggregationMap.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.map; import io.mycat.MycatServer; import io.mycat.memory.unsafe.KVIterator; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.hash.Murmur3_x86_32; import io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager; import io.mycat.memory.unsafe.row.StructType; import io.mycat.memory.unsafe.row.UnsafeRow; import io.mycat.memory.unsafe.utils.sort.UnsafeKVExternalSorter; import org.apache.log4j.Logger; import java.io.IOException; /** * Modify by zagnix ,add put find func * Unsafe-based HashMap for performing aggregations where the aggregated values are fixed-width. * This map supports a maximum of 2 billion keys. */ public final class UnsafeFixedWidthAggregationMap { private static Logger LOGGER = Logger.getLogger(UnsafeFixedWidthAggregationMap.class); /** * An empty aggregation buffer, encoded in UnsafeRow format. When inserting a new key into the * map, we copy this buffer and use it as the value. */ private final byte[] emptyAggregationBuffer; private final StructType aggregationBufferSchema; private final StructType groupingKeySchema; /** * A hashmap which maps from opaque bytearray keys to bytearray values. */ private final BytesToBytesMap map; /** * Re-used pointer to the current aggregation buffer */ private final UnsafeRow currentAggregationBuffer; private final boolean enablePerfMetrics; private final static int SEED = 42; /** * @return true if UnsafeFixedWidthAggregationMap supports aggregation buffers with the given * schema, false otherwise. */ public static boolean supportsAggregationBufferSchema(StructType schema) { return true; } /** * Create a new UnsafeFixedWidthAggregationMap. * * @param emptyAggregationBuffer the default value for new keys (a "zero" of the agg. function) * @param aggregationBufferSchema the schema of the aggregation buffer, used for row conversion. * @param groupingKeySchema the schema of the grouping key, used for row conversion. * @param dataNodeMemoryManager the memory manager used to allocate our Unsafe memory structures. * @param initialCapacity the initial capacity of the map (a sizing hint to avoid re-hashing). * @param pageSizeBytes the data page size, in bytes; limits the maximum record size. * @param enablePerfMetrics if true, performance metrics will be recorded (has minor perf impact) */ public UnsafeFixedWidthAggregationMap( UnsafeRow emptyAggregationBuffer, StructType aggregationBufferSchema, StructType groupingKeySchema, DataNodeMemoryManager dataNodeMemoryManager, int initialCapacity, long pageSizeBytes, boolean enablePerfMetrics) { this.aggregationBufferSchema = aggregationBufferSchema; this.currentAggregationBuffer = new UnsafeRow(aggregationBufferSchema.length()); this.groupingKeySchema = groupingKeySchema; this.map = new BytesToBytesMap(dataNodeMemoryManager,initialCapacity, pageSizeBytes, enablePerfMetrics); this.enablePerfMetrics = enablePerfMetrics; this.emptyAggregationBuffer = emptyAggregationBuffer.getBytes() ; } /** * Return the aggregation buffer for the current group. For efficiency, all calls to this method * return the same object. If additional memory could not be allocated, then this method will * signal an error by returning null. */ public UnsafeRow getAggregationBuffer(UnsafeRow groupingKey) { return getAggregationBufferFromUnsafeRow(groupingKey); } public UnsafeRow getAggregationBufferFromUnsafeRow(UnsafeRow key) { return getAggregationBufferFromUnsafeRow(key, Murmur3_x86_32.hashUnsafeWords(key.getBaseObject(),key.getBaseOffset(), key.getSizeInBytes(),SEED)); } public boolean put(UnsafeRow key, UnsafeRow value){ int hash = Murmur3_x86_32.hashUnsafeWords(key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(),SEED); // Probe our map using the serialized key final BytesToBytesMap.Location loc = map.lookup( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), hash); if (!loc.isDefined()) { // This is the first time that we've seen this grouping key, so we'll insert a copy of the // empty aggregation buffer into the map: boolean putSucceeded = loc.append( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), value.getBaseObject(), value.getBaseOffset(), value.getSizeInBytes()); if (!putSucceeded) { return false; } } return true; } public boolean find(UnsafeRow key){ int hash = Murmur3_x86_32.hashUnsafeWords(key.getBaseObject(),key.getBaseOffset(), key.getSizeInBytes(),42); // Probe our map using the serialized key final BytesToBytesMap.Location loc = map.lookup(key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), hash); if (!loc.isDefined()) { return false; } return true; } public UnsafeRow getAggregationBufferFromUnsafeRow(UnsafeRow key, int hash) { // Probe our map using the serialized key final BytesToBytesMap.Location loc = map.lookup( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), hash); if (!loc.isDefined()) { // This is the first time that we've seen this grouping key, so we'll insert a copy of the // empty aggregation buffer into the map: boolean putSucceeded = loc.append( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), emptyAggregationBuffer, Platform.BYTE_ARRAY_OFFSET, emptyAggregationBuffer.length ); if (!putSucceeded) { return null; } } // Reset the pointer to point to the value that we just stored or looked up: currentAggregationBuffer.pointTo( loc.getValueBase(), loc.getValueOffset(), loc.getValueLength() ); return currentAggregationBuffer; } /** * Returns an iterator over the keys and values in this map. This uses destructive iterator of * BytesToBytesMap. So it is illegal to call any other method on this map after `iterator()` has * been called. * * For efficiency, each call returns the same object. */ public KVIterator iterator() { return new KVIterator() { private final BytesToBytesMap.MapIterator mapLocationIterator = map.iterator(); private final UnsafeRow key = new UnsafeRow(groupingKeySchema.length()); private final UnsafeRow value = new UnsafeRow(aggregationBufferSchema.length()); @Override public boolean next() { if (mapLocationIterator.hasNext()) { final BytesToBytesMap.Location loc = mapLocationIterator.next(); if (loc == null) return false; key.pointTo( loc.getKeyBase(), loc.getKeyOffset(), loc.getKeyLength() ); value.pointTo( loc.getValueBase(), loc.getValueOffset(), loc.getValueLength() ); return true; } else { return false; } } @Override public UnsafeRow getKey() { return key; } @Override public UnsafeRow getValue() { return value; } @Override public void close() { } }; } /** * Return the peak memory used so far, in bytes. */ public long getPeakMemoryUsedBytes() { return map.getPeakMemoryUsedBytes(); } /** * Free the memory associated with this map. This is idempotent and can be called multiple times. */ public void free() { map.free(); } @SuppressWarnings("UseOfSystemOutOrSystemErr") public void printPerfMetrics() { if (!enablePerfMetrics) { throw new IllegalStateException("Perf metrics not enabled"); } System.out.println("Average probes per lookup: " + map.getAverageProbesPerLookup()); System.out.println("Number of hash collisions: " + map.getNumHashCollisions()); System.out.println("Time spent resizing (ns): " + map.getTimeSpentResizingNs()); System.out.println("Total memory consumption (bytes): " + map.getTotalMemoryConsumption()); } /** * Sorts the map's records in place, spill them to disk, and returns an [[UnsafeKVExternalSorter]] * * Note that the map will be reset for inserting new records, and the returned sorter can NOT be * used to insert records. */ public UnsafeKVExternalSorter destructAndCreateExternalSorter() throws IOException { return new UnsafeKVExternalSorter( groupingKeySchema, aggregationBufferSchema, MycatServer.getInstance().getMyCatMemory().getBlockManager(), MycatServer.getInstance().getMyCatMemory().getSerializerManager(), map.getPageSizeBytes(), map); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/memory/HeapMemoryAllocator.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.memory; import io.mycat.memory.unsafe.Platform; import javax.annotation.concurrent.GuardedBy; import java.lang.ref.WeakReference; import java.util.HashMap; import java.util.LinkedList; import java.util.Map; /** * A simple {@link MemoryAllocator} that can allocate up to 16GB using a JVM long primitive array. */ public class HeapMemoryAllocator implements MemoryAllocator { @GuardedBy("this") private final Map>> bufferPoolsBySize = new HashMap>>(); private static final int POOLING_THRESHOLD_BYTES = 1024 * 1024; /** * Returns true if allocations of the given size should go through the pooling mechanism and * false otherwise. */ private boolean shouldPool(long size) { // Very small allocations are less likely to benefit from pooling. return size >= POOLING_THRESHOLD_BYTES; } @Override public MemoryBlock allocate(long size) throws OutOfMemoryError { if (shouldPool(size)) { synchronized (this) { final LinkedList> pool = bufferPoolsBySize.get(size); if (pool != null) { while (!pool.isEmpty()) { final WeakReference blockReference = pool.pop(); final MemoryBlock memory = blockReference.get(); if (memory != null) { assert (memory.size() == size); return memory; } } bufferPoolsBySize.remove(size); } } } long[] array = new long[(int) ((size + 7) / 8)]; return new MemoryBlock(array, Platform.LONG_ARRAY_OFFSET, size); } @Override public void free(MemoryBlock memory) { final long size = memory.size(); if (shouldPool(size)) { synchronized (this) { LinkedList> pool = bufferPoolsBySize.get(size); if (pool == null) { pool = new LinkedList>(); bufferPoolsBySize.put(size, pool); } pool.add(new WeakReference(memory)); } } else { // Do nothing } } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/memory/MemoryAllocator.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.memory; public interface MemoryAllocator { /** * Allocates a contiguous block of memory. Note that the allocated memory is not guaranteed * to be zeroed out (call `zero()` on the result if this is necessary). */ MemoryBlock allocate(long size) throws OutOfMemoryError; void free(MemoryBlock memory); MemoryAllocator UNSAFE = new UnsafeMemoryAllocator(); MemoryAllocator HEAP = new HeapMemoryAllocator(); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/memory/MemoryBlock.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.memory; import io.mycat.memory.unsafe.Platform; import javax.annotation.Nullable; /** * A consecutive block of memory, starting at a {@link MemoryLocation} with a fixed size. */ public class MemoryBlock extends MemoryLocation { private final long length; /** * Optional page number; used when this MemoryBlock represents a page allocated by a * DataNodeMemoryManager. This field is public so that it can be modified by the DataNodeMemoryManager, * which lives in a different package. */ public int pageNumber = -1; public MemoryBlock(@Nullable Object obj, long offset, long length) { super(obj, offset); this.length = length; } /** * Returns the size of the memory block. */ public long size() { return length; } /** * Creates a memory block pointing to the memory used by the long array. */ public static MemoryBlock fromLongArray(final long[] array) { return new MemoryBlock(array, Platform.LONG_ARRAY_OFFSET, array.length * 8); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/memory/MemoryLocation.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.memory; import javax.annotation.Nullable; /** * A memory location. Tracked either by a memory address (with off-heap allocation), * or by an offset from a JVM object (in-heap allocation). */ public class MemoryLocation { @Nullable Object obj; long offset; public MemoryLocation(@Nullable Object obj, long offset) { this.obj = obj; this.offset = offset; } public MemoryLocation() { this(null, 0); } public void setObjAndOffset(Object newObj, long newOffset) { this.obj = newObj; this.offset = newOffset; } public final Object getBaseObject() { return obj; } public final long getBaseOffset() { return offset; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/memory/UnsafeMemoryAllocator.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.memory; import io.mycat.memory.unsafe.Platform; /** * A simple {@link MemoryAllocator} that uses {@code Unsafe} to allocate off-heap memory. */ public class UnsafeMemoryAllocator implements MemoryAllocator { @Override public MemoryBlock allocate(long size) throws OutOfMemoryError { long address = Platform.allocateMemory(size); return new MemoryBlock(null, address, size); } @Override public void free(MemoryBlock memory) { assert (memory.obj == null) : "baseObject not null; are you trying to use the off-heap allocator to free on-heap memory?"; Platform.freeMemory(memory.offset); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/memory/mm/DataNodeMemoryManager.java ================================================ package io.mycat.memory.unsafe.memory.mm; import com.google.common.annotations.VisibleForTesting; import io.mycat.memory.unsafe.memory.MemoryBlock; import io.mycat.memory.unsafe.utils.JavaUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.concurrent.GuardedBy; import java.io.IOException; import java.util.Arrays; import java.util.BitSet; import java.util.HashSet; /** * Modify by zagnix * Manages the memory allocated by an individual thread. *

* Most of the complexity in this class deals with encoding of off-heap addresses into 64-bit longs. * In off-heap mode, memory can be directly addressed with 64-bit longs. In on-heap mode, memory is * addressed by the combination of a base Object reference and a 64-bit offset within that object. * This is a problem when we want to store pointers to data structures inside of other structures, * such as record pointers inside hashmaps or sorting buffers. Even if we decided to use 128 bits * to address memory, we can't just store the address of the base object since it's not guaranteed * to remain stable as the heap gets reorganized due to GC. *

* Instead, we use the following approach to encode record pointers in 64-bit longs: for off-heap * mode, just store the raw address, and for on-heap mode use the upper 13 bits of the address to * store a "page number" and the lower 51 bits to store an offset within this page. These page * numbers are used to index into a "page table" array inside of the MemoryManager in order to * retrieve the base object. *

* This allows us to address 8192 pages. In on-heap mode, the maximum page size is limited by the * maximum size of a long[] array, allowing us to address 8192 * 2^32 * 8 bytes, which is * approximately 35 terabytes of memory. */ public class DataNodeMemoryManager { private final Logger logger = LoggerFactory.getLogger(DataNodeMemoryManager.class); /** The number of bits used to address the page table. */ private static final int PAGE_NUMBER_BITS = 13; /** The number of bits used to encode offsets in data pages. */ public static final int OFFSET_BITS = 64 - PAGE_NUMBER_BITS; // 51 /** The number of entries in the page table. */ private static final int PAGE_TABLE_SIZE = 1 << PAGE_NUMBER_BITS; /** * Maximum supported data page size (in bytes). In principle, the maximum addressable page size is * (1L << OFFSET_BITS) bytes, which is 2+ petabytes. However, the on-heap allocator's * maximum page size is limited by the maximum amount of data that can be stored in a long[] * array, which is (2^32 - 1) * 8 bytes (or 16 gigabytes). Therefore, we cap this at 16 gigabytes. */ public static final long MAXIMUM_PAGE_SIZE_BYTES = ((1L << 31) - 1) * 8L; /** Bit mask for the lower 51 bits of a long. */ private static final long MASK_LONG_LOWER_51_BITS = 0x7FFFFFFFFFFFFL; /** Bit mask for the upper 13 bits of a long */ private static final long MASK_LONG_UPPER_13_BITS = ~MASK_LONG_LOWER_51_BITS; /** * Similar to an operating system's page table, this array maps page numbers into base object * pointers, allowing us to translate between the hashtable's internal 64-bit address * representation and the baseObject+offset representation which we use to support both in- and * off-heap addresses. When using an off-heap allocator, every entry in this map will be `null`. * When using an in-heap allocator, the entries in this map will point to pages' base objects. * Entries are added to this map as new data pages are allocated. */ private final MemoryBlock[] pageTable = new MemoryBlock[PAGE_TABLE_SIZE]; /** * Bitmap for tracking free pages. */ private final BitSet allocatedPages = new BitSet(PAGE_TABLE_SIZE); private final MemoryManager memoryManager; private final long connectionAttemptId; /** * Tracks whether we're in-heap or off-heap. For off-heap, we short-circuit most of these methods * without doing any masking or lookups. Since this branching should be well-predicted by the JIT, * this extra layer of indirection / abstraction hopefully shouldn't be too expensive. */ public final MemoryMode tungstenMemoryMode; /** * Tracks spillable memory consumers. */ @GuardedBy("this") private final HashSet consumers; /** * The amount of memory that is acquired but not used. */ private volatile long acquiredButNotUsed = 0L; /** * Construct a new DataNodeMemoryManager. */ public DataNodeMemoryManager(MemoryManager memoryManager, long connectionAttemptId) { this.tungstenMemoryMode = memoryManager.tungstenMemoryMode(); this.memoryManager = memoryManager; this.connectionAttemptId = connectionAttemptId; this.consumers = new HashSet(); } /** * Acquire N bytes of memory for a consumer. If there is no enough memory, it will call * spill() of consumers to release more memory. * * @return number of bytes successfully granted (<= N). */ public long acquireExecutionMemory(long required,MemoryMode mode,MemoryConsumer consumer) throws InterruptedException { assert(required >= 0); // If we are allocating Tungsten pages off-heap and receive a request to allocate on-heap // memory here, then it may not make sense to spill since that would only end up freeing // off-heap memory. This is subject to change, though, so it may be risky to make this // optimization now in case we forget to undo it late when making changes. synchronized (this) { long got = memoryManager.acquireExecutionMemory(required,connectionAttemptId, mode); // Try to release memory from other consumers first, then we can reduce the frequency of // spilling, avoid to have too many spilled files. if (got < required) { // Call spill() on other consumers to release memory for (MemoryConsumer c: consumers) { if (c != consumer && c.getUsed() > 0) { try { /** * 调用spill函数，写数据到磁盘中 */ long released = c.spill(required - got, consumer); if (released > 0 && mode == tungstenMemoryMode) { logger.info("Thread "+connectionAttemptId+" released "+ JavaUtils.bytesToString(released) + " from "+ c +" for" + consumer); got += memoryManager.acquireExecutionMemory(required - got, connectionAttemptId, mode); if (got >= required) { break; } } } catch (IOException e) { logger.error("error while calling spill() on " + c, e); throw new OutOfMemoryError("error while calling spill() on " + c + " : " + e.getMessage()); } } } } // call spill() on itself if (got < required && consumer != null) { try { long released = consumer.spill(required - got, consumer); if (released > 0 && mode == tungstenMemoryMode) { logger.info("Thread " + connectionAttemptId + " released "+ JavaUtils.bytesToString(released) +"from itself ("+consumer+ ")"); got += memoryManager.acquireExecutionMemory(required - got, connectionAttemptId, mode); } } catch (IOException e) { logger.error("error while calling spill() on " + consumer, e); throw new OutOfMemoryError("error while calling spill() on " + consumer + " : " + e.getMessage()); } } if (consumer != null) { consumers.add(consumer); } // logger.info("Thread" + connectionAttemptId + " acquire "+ JavaUtils.bytesToString(got) +" for "+ consumer+""); return got; } } /** * Release N bytes of execution memory for a MemoryConsumer. */ public void releaseExecutionMemory(long size, MemoryMode mode, MemoryConsumer consumer) { logger.debug ("Thread" + connectionAttemptId + " release "+ JavaUtils.bytesToString(size) +" from "+ consumer+""); memoryManager.releaseExecutionMemory(size, connectionAttemptId, mode); } /** * Dump the memory usage of all consumers. */ public void showMemoryUsage() { logger.info("Memory used in Thread " + connectionAttemptId); synchronized (this) { long memoryAccountedForByConsumers = 0; for (MemoryConsumer c: consumers) { long totalMemUsage = c.getUsed(); memoryAccountedForByConsumers += totalMemUsage; if (totalMemUsage > 0) { logger.info("Acquired by " + c + ": " + JavaUtils.bytesToString(totalMemUsage)); } } long memoryNotAccountedFor = memoryManager.getExecutionMemoryUsageForConnection(connectionAttemptId) - memoryAccountedForByConsumers; logger.info( "{} bytes of memory were used by task {} but are not associated with specific consumers", memoryNotAccountedFor, connectionAttemptId); logger.info( "{} bytes of memory are used for execution and {} bytes of memory are used for storage", memoryManager.executionMemoryUsed()); } } /** * Return the page size in bytes. */ public long pageSizeBytes() { return memoryManager.pageSizeBytes(); } /** * Allocate a block of memory that will be tracked in the MemoryManager's page table; this is * intended for allocating large blocks of Tungsten memory that will be shared between operators. * * Returns `null` if there was not enough memory to allocate the page. May return a page that * contains fewer bytes than requested, so callers should verify the size of returned pages. */ public MemoryBlock allocatePage(long size, MemoryConsumer consumer) { if (size > MAXIMUM_PAGE_SIZE_BYTES) { throw new IllegalArgumentException( "Cannot allocate a page with more than " + MAXIMUM_PAGE_SIZE_BYTES + " bytes"); } /** * 这里spill到磁盘中，释放内存空间 */ long acquired = 0; try { acquired = acquireExecutionMemory(size,tungstenMemoryMode, consumer); } catch (InterruptedException e) { logger.error(e.getMessage()); } if (acquired <= 0) { return null; } final int pageNumber; synchronized (this) { pageNumber = allocatedPages.nextClearBit(0); if (pageNumber >= PAGE_TABLE_SIZE) { releaseExecutionMemory(acquired, tungstenMemoryMode, consumer); throw new IllegalStateException( "Have already allocated a maximum of " + PAGE_TABLE_SIZE + " pages"); } allocatedPages.set(pageNumber); } MemoryBlock page = null; try { page = memoryManager.tungstenMemoryAllocator().allocate(acquired); } catch (OutOfMemoryError e) { logger.warn("Failed to allocate a page ({} bytes), try again.", acquired); // there is no enough memory actually, it means the actual free memory is smaller than // MemoryManager thought, we should keep the acquired memory. synchronized (this) { acquiredButNotUsed += acquired; allocatedPages.clear(pageNumber); } // this could trigger spilling to free some pages. return allocatePage(size, consumer); } page.pageNumber = pageNumber; pageTable[pageNumber] = page; // logger.info("Allocate page number " + pageNumber + " ("+ acquired +" bytes)"); return page; } /** * Free a block of memory allocated via {@link DataNodeMemoryManager#allocatePage}. */ public void freePage(MemoryBlock page, MemoryConsumer consumer) { assert (page.pageNumber != -1) : "Called freePage() on memory that wasn't allocated with allocatePage()"; assert(allocatedPages.get(page.pageNumber)); pageTable[page.pageNumber] = null; synchronized (this) { allocatedPages.clear(page.pageNumber); } logger.trace("Freed page number "+ page.pageNumber +" ("+page.size() +" bytes)"); long pageSize = page.size(); memoryManager.tungstenMemoryAllocator().free(page); releaseExecutionMemory(pageSize,tungstenMemoryMode,consumer); } /** * Given a memory page and offset within that page, encode this address into a 64-bit long. * This address will remain valid as long as the corresponding page has not been freed. * * @param page a data page allocated by {@link DataNodeMemoryManager#allocatePage}/ * @param offsetInPage an offset in this page which incorporates the base offset. In other words, * this should be the value that you would pass as the base offset into an * UNSAFE call (e.g. page.baseOffset() + something). * @return an encoded page address. */ public long encodePageNumberAndOffset(MemoryBlock page, long offsetInPage) { if (tungstenMemoryMode == MemoryMode.OFF_HEAP) { // In off-heap mode, an offset is an absolute address that may require a full 64 bits to // encode. Due to our page size limitation, though, we can convert this into an offset that's // relative to the page's base offset; this relative offset will fit in 51 bits. offsetInPage -= page.getBaseOffset(); } return encodePageNumberAndOffset(page.pageNumber, offsetInPage); } @VisibleForTesting public static long encodePageNumberAndOffset(int pageNumber, long offsetInPage) { assert (pageNumber != -1) : "encodePageNumberAndOffset called with invalid page"; return (((long) pageNumber) << OFFSET_BITS) | (offsetInPage & MASK_LONG_LOWER_51_BITS); } @VisibleForTesting public static int decodePageNumber(long pagePlusOffsetAddress) { return (int) (pagePlusOffsetAddress >>> OFFSET_BITS); } private static long decodeOffset(long pagePlusOffsetAddress) { return (pagePlusOffsetAddress & MASK_LONG_LOWER_51_BITS); } /** * Get the page associated with an address encoded by * {@link DataNodeMemoryManager#encodePageNumberAndOffset(MemoryBlock, long)} */ public Object getPage(long pagePlusOffsetAddress) { if (tungstenMemoryMode == MemoryMode.ON_HEAP) { final int pageNumber = decodePageNumber(pagePlusOffsetAddress); assert (pageNumber >= 0 && pageNumber < PAGE_TABLE_SIZE); final MemoryBlock page = pageTable[pageNumber]; assert (page != null); assert (page.getBaseObject() != null); return page.getBaseObject(); } else { return null; } } /** * Get the offset associated with an address encoded by * {@link DataNodeMemoryManager#encodePageNumberAndOffset(MemoryBlock, long)} */ public long getOffsetInPage(long pagePlusOffsetAddress) { final long offsetInPage = decodeOffset(pagePlusOffsetAddress); if (tungstenMemoryMode == MemoryMode.ON_HEAP) { return offsetInPage; } else { // In off-heap mode, an offset is an absolute address. In encodePageNumberAndOffset, we // converted the absolute address into a relative address. Here, we invert that operation: final int pageNumber = decodePageNumber(pagePlusOffsetAddress); assert (pageNumber >= 0 && pageNumber < PAGE_TABLE_SIZE); final MemoryBlock page = pageTable[pageNumber]; assert (page != null); return page.getBaseOffset() + offsetInPage; } } /** * Clean up all allocated memory and pages. Returns the number of bytes freed. A non-zero return * value can be used to detect memory leaks. */ public long cleanUpAllAllocatedMemory() { synchronized (this) { for (MemoryConsumer c: consumers) { if (c != null && c.getUsed() > 0) { // In case of failed task, it's normal to see leaked memory logger.warn("leak " + JavaUtils.bytesToString(c.getUsed()) + " memory from " + c); } } consumers.clear(); for (MemoryBlock page : pageTable) { if (page != null) { logger.warn("leak a page: " + page + " in task " + connectionAttemptId); memoryManager.tungstenMemoryAllocator().free(page); } } Arrays.fill(pageTable, null); } // release the memory that is not used by any consumer. memoryManager.releaseExecutionMemory(acquiredButNotUsed, connectionAttemptId, tungstenMemoryMode); return memoryManager.releaseAllExecutionMemoryForConnection(connectionAttemptId); } /** * Returns the memory consumption, in bytes, for the current task. */ public long getMemoryConsumptionForThisConnection() { return memoryManager.getExecutionMemoryUsageForConnection(connectionAttemptId); } /** * Returns Tungsten memory mode */ public MemoryMode getTungstenMemoryMode() { return tungstenMemoryMode; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/memory/mm/MemoryConsumer.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.memory.mm; import io.mycat.memory.unsafe.array.CharArray; import io.mycat.memory.unsafe.array.LongArray; import io.mycat.memory.unsafe.memory.MemoryBlock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; /** * An memory consumer of DataNodeMemoryManager, which support spilling. * Note: this only supports allocation / spilling of Tungsten memory. */ public abstract class MemoryConsumer { private final Logger logger = LoggerFactory.getLogger(MemoryConsumer.class); protected final DataNodeMemoryManager dataNodeMemoryManager; private final long pageSize; protected long used; protected MemoryConsumer(DataNodeMemoryManager dataNodeMemoryManager, long pageSize) { this.dataNodeMemoryManager = dataNodeMemoryManager; this.pageSize = pageSize; } protected MemoryConsumer(DataNodeMemoryManager dataNodeMemoryManager) { this(dataNodeMemoryManager, dataNodeMemoryManager.pageSizeBytes()); } /** * Returns the size of used memory in bytes. */ public long getUsed() { return used; } /** * Force spill during building. * * For testing. */ public void spill() throws IOException { spill(Long.MAX_VALUE, this); } /** * Spill some data to disk to release memory, which will be called by DataNodeMemoryManager * when there is not enough memory for the task. * * This should be implemented by subclass. * * Note: In order to avoid possible deadlock, should not call acquireMemory() from spill(). * * Note: today, this only frees Tungsten-managed pages. * * @param size the amount of memory should be released * @param trigger the MemoryConsumer that trigger this spilling * @return the amount of released memory in bytes * @throws IOException */ public abstract long spill(long size, MemoryConsumer trigger) throws IOException; /** * Allocates a LongArray of `size`. */ public LongArray allocateLongArray(long size) { long required = size * 8L; MemoryBlock page = dataNodeMemoryManager.allocatePage(required,this); if (page == null || page.size() < required) { long got = 0; if (page != null) { got = page.size(); dataNodeMemoryManager.freePage(page, this); } dataNodeMemoryManager.showMemoryUsage(); throw new OutOfMemoryError("Unable to acquire " + required + " bytes of memory, got " + got); } used += required; return new LongArray(page); } /** * Frees a LongArray. */ public void freeLongArray(LongArray array) { freePage(array.memoryBlock()); } public CharArray allocateCharArray(long size) { long required = size * 2L; MemoryBlock page = dataNodeMemoryManager.allocatePage(required,this); if (page == null || page.size() < required) { long got = 0; if (page != null) { got = page.size(); dataNodeMemoryManager.freePage(page, this); } dataNodeMemoryManager.showMemoryUsage(); throw new OutOfMemoryError("Unable to acquire " + required + " bytes of memory, got " + got); } used += required; return new CharArray(page,this); } /** * Frees a CharArray. */ public void freeCharArray(CharArray array) { freePage(array.memoryBlock()); } /** * Allocate a memory block with at least `required` bytes. * * Throws IOException if there is not enough memory. * * @throws OutOfMemoryError */ protected MemoryBlock allocatePage(long required) { MemoryBlock page = dataNodeMemoryManager.allocatePage(Math.max(pageSize, required), this); if (page == null || page.size() < required) { long got = 0; if (page != null) { got = page.size(); dataNodeMemoryManager.freePage(page,this); } dataNodeMemoryManager.showMemoryUsage(); throw new OutOfMemoryError("Unable to acquire " + required + " bytes of memory, got " + got); } used += page.size(); return page; } /** * Free a memory block. */ protected void freePage(MemoryBlock page) { used -= page.size(); dataNodeMemoryManager.freePage(page, this); } /** * Allocates a heap memory of `size`. */ public long acquireOnHeapMemory(long size) { long granted = 0; try { granted = dataNodeMemoryManager.acquireExecutionMemory(size, MemoryMode.ON_HEAP, this); } catch (InterruptedException e) { logger.error(e.getMessage()); } used += granted; return granted; } /** * Release N bytes of heap memory. */ public void freeOnHeapMemory(long size) { dataNodeMemoryManager.releaseExecutionMemory(size, MemoryMode.ON_HEAP, this); used -= size; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/memory/mm/MemoryManager.java ================================================ package io.mycat.memory.unsafe.memory.mm; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.array.ByteArrayMethods; import io.mycat.memory.unsafe.memory.MemoryAllocator; import io.mycat.memory.unsafe.utils.MycatPropertyConf; import javax.annotation.concurrent.GuardedBy; import java.util.concurrent.ConcurrentHashMap; public abstract class MemoryManager { private MycatPropertyConf conf; @GuardedBy("this") protected ResultSetMemoryPool onHeapExecutionMemoryPool = new ResultSetMemoryPool(this, MemoryMode.ON_HEAP); @GuardedBy("this") protected ResultSetMemoryPool offHeapExecutionMemoryPool = new ResultSetMemoryPool(this, MemoryMode.OFF_HEAP); protected long maxOffHeapMemory = 0L; protected long offHeapExecutionMemory = 0L; private int numCores = 0; public MemoryManager(MycatPropertyConf conf, int numCores, long onHeapExecutionMemory){ this.conf = conf; this.numCores =numCores; maxOffHeapMemory = conf.getSizeAsBytes("mycat.memory.offHeap.size","128m"); offHeapExecutionMemory = maxOffHeapMemory; onHeapExecutionMemoryPool.incrementPoolSize(onHeapExecutionMemory); offHeapExecutionMemoryPool.incrementPoolSize(offHeapExecutionMemory); } protected abstract long acquireExecutionMemory(long numBytes,long taskAttemptId,MemoryMode memoryMode) throws InterruptedException; /** * Release numBytes of execution memory belonging to the given task. */ public void releaseExecutionMemory(long numBytes, long taskAttemptId, MemoryMode memoryMode) { synchronized (this) { switch (memoryMode) { case ON_HEAP: onHeapExecutionMemoryPool.releaseMemory(numBytes, taskAttemptId); break; case OFF_HEAP: offHeapExecutionMemoryPool.releaseMemory(numBytes, taskAttemptId); break; } } } /** * Release all memory for the given task and mark it as inactive (e.g. when a task ends). * @return the number of bytes freed. */ public long releaseAllExecutionMemoryForConnection(long connAttemptId){ synchronized(this) { return (onHeapExecutionMemoryPool.releaseAllMemoryForeConnection(connAttemptId) + offHeapExecutionMemoryPool.releaseAllMemoryForeConnection(connAttemptId)); } } /** * Execution memory currently in use, in bytes. */ public final long executionMemoryUsed() { synchronized(this) { return (onHeapExecutionMemoryPool.memoryUsed() + offHeapExecutionMemoryPool.memoryUsed()); } } /** * Returns the execution memory consumption, in bytes, for the given task. */ public long getExecutionMemoryUsageForConnection(long connAttemptId) { synchronized (this) { assert (connAttemptId >= 0); return (onHeapExecutionMemoryPool.getMemoryUsageConnection(connAttemptId) + offHeapExecutionMemoryPool.getMemoryUsageConnection(connAttemptId)); } } /** * Tracks whether Tungsten memory will be allocated on the JVM heap or off-heap using * sun.misc.Unsafe. */ public final MemoryMode tungstenMemoryMode(){ if (conf.getBoolean("mycat.memory.offHeap.enabled", false)) { assert (conf.getSizeAsBytes("mycat.memory.offHeap.size",0) > 0); assert (Platform.unaligned()); return MemoryMode.OFF_HEAP; } else { return MemoryMode.ON_HEAP; } } /** * The default page size, in bytes. * * If user didn't explicitly set "mycat.buffer.pageSize", we figure out the default value * by looking at the number of cores available to the process, and the total amount of memory, * and then divide it by a factor of safety. */ public long pageSizeBytes() { long minPageSize = 1L * 1024 * 1024 ; // 1MB long maxPageSize = 64L * minPageSize ; // 64MB int cores = 0; if (numCores > 0){ cores = numCores ; } else { cores = Runtime.getRuntime().availableProcessors(); } // Because of rounding to next power of 2, we may have safetyFactor as 8 in worst case int safetyFactor = 16; long maxTungstenMemory = 0L; switch (tungstenMemoryMode()){ case ON_HEAP: maxTungstenMemory = onHeapExecutionMemoryPool.poolSize(); break; case OFF_HEAP: maxTungstenMemory = offHeapExecutionMemoryPool.poolSize(); break; } long size = ByteArrayMethods.nextPowerOf2(maxTungstenMemory / cores / safetyFactor); long defaultSize = Math.min(maxPageSize, Math.max(minPageSize, size)); defaultSize = conf.getSizeAsBytes("mycat.buffer.pageSize", defaultSize); return defaultSize; } /** * Allocates memory for use by Unsafe/Tungsten code. */ public final MemoryAllocator tungstenMemoryAllocator() { switch (tungstenMemoryMode()){ case ON_HEAP: return MemoryAllocator.HEAP; case OFF_HEAP: return MemoryAllocator.UNSAFE; } return null; } /** * Get Direct Memory Usage. */ public final ConcurrentHashMap getDirectMemorUsage() { return offHeapExecutionMemoryPool.getMemoryForConnection(); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/memory/mm/MemoryMode.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.memory.mm; public enum MemoryMode { ON_HEAP, OFF_HEAP } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/memory/mm/MemoryPool.java ================================================ package io.mycat.memory.unsafe.memory.mm; import javax.annotation.concurrent.GuardedBy; /** * Manages bookkeeping for an adjustable-sized region of memory. This class is internal to * the [[MemoryManager]]. See subclasses for more details. * */ public abstract class MemoryPool { /** * lock [[MemoryManager]] instance, used for synchronization. We purposely erase the type * to `Object` to avoid programming errors, since this object should only be used for * synchronization purposes. */ protected final Object lock; public MemoryPool(Object lock){ this.lock = lock; } @GuardedBy("lock") private long _poolSize = 0; /** * Returns the current size of the pool, in bytes. */ public final long poolSize() { synchronized(lock) { return _poolSize; } } /** * Returns the amount of free memory in the pool, in bytes. */ public long memoryFree() { synchronized(lock) { return (_poolSize - memoryUsed()); } } /** * Expands the pool by `delta` bytes. */ public final void incrementPoolSize(long delta) { assert (delta >= 0); synchronized(lock) { _poolSize += delta; } } /** * Shrinks the pool by `delta` bytes. */ public final void decrementPoolSize(long delta){ synchronized(lock) { assert (delta >= 0); assert (delta <= _poolSize); assert (_poolSize - delta >= memoryUsed()); _poolSize -= delta; } } /** * Returns the amount of used memory in this pool (in bytes). */ protected abstract long memoryUsed(); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/memory/mm/ResultMergeMemoryManager.java ================================================ package io.mycat.memory.unsafe.memory.mm; import io.mycat.memory.unsafe.utils.MycatPropertyConf; /** * Created by zagnix on 2016/6/7. */ public class ResultMergeMemoryManager extends MemoryManager { private long maxOnHeapExecutionMemory; private int numCores; private MycatPropertyConf conf; public ResultMergeMemoryManager(MycatPropertyConf conf, int numCores, long onHeapExecutionMemory){ super(conf,numCores,onHeapExecutionMemory); this.conf = conf; this.numCores = numCores; this.maxOnHeapExecutionMemory = onHeapExecutionMemory; } @Override protected synchronized long acquireExecutionMemory(long numBytes,long taskAttemptId,MemoryMode memoryMode) throws InterruptedException { switch (memoryMode) { case ON_HEAP: return onHeapExecutionMemoryPool.acquireMemory(numBytes,taskAttemptId); case OFF_HEAP: return offHeapExecutionMemoryPool.acquireMemory(numBytes,taskAttemptId); } return 0L; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/memory/mm/ResultSetMemoryPool.java ================================================ package io.mycat.memory.unsafe.memory.mm; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.concurrent.GuardedBy; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; /** * Created by zagnix on 2016/6/6. */ public class ResultSetMemoryPool extends MemoryPool { private static final Logger LOG = LoggerFactory.getLogger(ResultSetMemoryPool.class); private MemoryMode memoryMode ; /** * @param lock a [[MemoryManager]] instance to synchronize on * @param memoryMode the type of memory tracked by this pool (on- or off-heap) */ public ResultSetMemoryPool(Object lock, MemoryMode memoryMode) { super(lock); this.memoryMode = memoryMode; } private String poolName(){ switch (memoryMode){ case ON_HEAP: return "on-heap memory"; case OFF_HEAP: return "off-heap memory"; } return "off-heap memory"; } public ConcurrentHashMap getMemoryForConnection() { return memoryForConnection; } /** * Map from taskAttemptId -> memory consumption in bytes */ private ConcurrentHashMap memoryForConnection = new ConcurrentHashMap(); @Override protected long memoryUsed() { synchronized (lock) { long used =0; for (Map.Entry entry : memoryForConnection.entrySet()) { used += entry.getValue(); } return used; } } /** * Returns the memory consumption, in bytes, for the given task. */ public long getMemoryUsageConnection(long taskAttemptId) { synchronized (lock) { if (!memoryForConnection.containsKey(taskAttemptId)) { memoryForConnection.put(taskAttemptId, 0L); } return memoryForConnection.get(taskAttemptId); } } /** * Try to acquire up to `numBytes` of memory for the given task and return the number of bytes * obtained, or 0 if none can be allocated. * * This call may block until there is enough free memory in some situations, to make sure each * task has a chance to ramp up to at least 1 / 8N of the total memory pool (where N is the # of * active tasks) before it is forced to spill. This can happen if the number of tasks increase * but an older task had a lot of memory already. * * @param numBytes number of bytes to acquire * @param connAttemptId the task attempt acquiring memory * @return the number of bytes granted to the task. */ public long acquireMemory(long numBytes, long connAttemptId) throws InterruptedException { synchronized (lock) { assert (numBytes > 0); // Add this connection to the taskMemory map just so we can keep an accurate count of the number // of active tasks, to let other tasks ramp down their memory in calls to `acquireMemory` if (!memoryForConnection.containsKey(connAttemptId)) { memoryForConnection.put(connAttemptId, 0L); // This will later cause waiting tasks to wake up and check numTasks again lock.notifyAll(); } while (true) { long numActiveConns = memoryForConnection.size(); long curMem = memoryForConnection.get(connAttemptId); long maxPoolSize = poolSize(); long maxMemoryPerTask = maxPoolSize / numActiveConns; long minMemoryPerTask = poolSize() / (8 * numActiveConns); // How much we can grant this connection; keep its share within 0 <= X <= 1 / numActiveConns long maxToGrant = Math.min(numBytes, Math.max(0, maxMemoryPerTask - curMem)); // Only give it as much memory as is free, which might be none if it reached 1 / numActiveConns long toGrant = Math.min(maxToGrant, memoryFree()); // We want to let each connection get at least 1 / (8 * numActiveConns) before blocking; // if we can't give it this much now, wait for other tasks to free up memory // (this happens if older tasks allocated lots of memory before N grew) if (toGrant < numBytes && curMem + toGrant < minMemoryPerTask) { LOG.info("Thread " + connAttemptId + " waiting for at least 1/8N of " + poolName() + " pool to be free"); lock.wait(); } else { long temp = memoryForConnection.get(connAttemptId); memoryForConnection.put(connAttemptId, (temp + toGrant)); return toGrant; } } } } /** * Release `numBytes` of memory acquired by the given task. */ public void releaseMemory(long numBytes, long connAttemptId) { synchronized (lock) { long curMem = memoryForConnection.get(connAttemptId); long memoryToFree = 0L; if (curMem < numBytes) { LOG.error( "Internal error: release called on $numBytes bytes but task only has $curMem bytes " + "of memory from the " + poolName() + " pool"); memoryToFree = curMem; } else { memoryToFree = numBytes; } if (memoryForConnection.containsKey(connAttemptId)) { long temp = memoryForConnection.get(connAttemptId); memoryForConnection.put(connAttemptId, (temp - memoryToFree)); if (memoryForConnection.get(connAttemptId) <= 0) { memoryForConnection.remove(connAttemptId); } } // Notify waiters in acquireMemory() that memory has been freed lock.notifyAll(); } } /** * Release all memory for the given task and mark it as inactive (e.g. when a task ends). * @return the number of bytes freed. */ public long releaseAllMemoryForeConnection(long connAttemptId) { synchronized (lock){ long numBytesToFree = getMemoryUsageConnection(connAttemptId); releaseMemory(numBytesToFree,connAttemptId); return numBytesToFree; } } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/RingBuffer.java ================================================ package io.mycat.memory.unsafe.ringbuffer; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.memory.mm.MemoryConsumer; import io.mycat.memory.unsafe.ringbuffer.common.Cursored; import io.mycat.memory.unsafe.ringbuffer.common.event.*; import io.mycat.memory.unsafe.ringbuffer.exception.InsufficientCapacityException; import io.mycat.memory.unsafe.ringbuffer.producer.Sequencer; /** * 环形buffer 待实现， */ public class RingBuffer implements Cursored, EventSequencer, EventSink { //Buffer数组填充 private static final int BUFFER_PAD; //Buffer数组起始基址 private static final long REF_ARRAY_BASE; //2^n=每个数组对象引用所占空间，这个n就是REF_ELEMENT_SHIFT private static final int REF_ELEMENT_SHIFT; static { final int scale = Platform.arrayIndexScale(Object[].class); //Object数组引用长度，32位为4字节，64位为8字节 if (4 == scale) { REF_ELEMENT_SHIFT = 2; } else if (8 == scale) { REF_ELEMENT_SHIFT = 3; } else { throw new IllegalStateException("Unknown pointer size"); } //需要填充128字节，缓存行长度一般是128字节 BUFFER_PAD = 128 / scale; REF_ARRAY_BASE = Platform.arrayBaseOffset(Object[].class) + (BUFFER_PAD << REF_ELEMENT_SHIFT); } private final long indexMask; private final Object[] entries; protected final int bufferSize; protected final Sequencer sequencer; public RingBuffer(EventFactory eventFactory, Sequencer sequencer) { this.sequencer = sequencer; this.bufferSize = sequencer.getBufferSize(); //保证buffer大小不小于1 if (bufferSize < 1) { throw new IllegalArgumentException("bufferSize must not be less than 1"); } //保证buffer大小为2的n次方 if (Integer.bitCount(bufferSize) != 1) { throw new IllegalArgumentException("bufferSize must be a power of 2"); } //m % 2^n <=> m & (2^n - 1) this.indexMask = bufferSize - 1; /** * 结构：缓存行填充，避免频繁访问的任一entry与另一被修改的无关变量写入同一缓存行 * -------------- * * 数组头 * BASE * * Padding * 128字节 * * reference1 * SCALE * * reference2 * SCALE * * reference3 * SCALE * .......... * * Padding * 128字节 * -------------- */ this.entries = new Object[sequencer.getBufferSize() + 2 * BUFFER_PAD]; //利用eventFactory初始化RingBuffer的每个槽 fill(eventFactory); } private void fill(EventFactory eventFactory) { for (int i = 0; i < bufferSize; i++) { entries[BUFFER_PAD + i] = eventFactory.newInstance(); } } /** * 根据地址取出一个元素的引用 * * @param sequence * @return */ private E elementAt(long sequence) { return (E) Platform.getObject(entries, REF_ARRAY_BASE + ((sequence & indexMask) << REF_ELEMENT_SHIFT)); } @Override public long getCursor() { return sequencer.getCursor(); } /** * @see io.mycat.memory.unsafe.ringbuffer.common.DataProvider#get(long) */ @Override public E get(long sequence) { return elementAt(sequence); } private void translateAndPublish(EventTranslator translator, long sequence) { try { translator.translateTo(get(sequence), sequence); } finally { sequencer.publish(sequence); } } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public void publishEvent(EventTranslator translator) { final long sequence = sequencer.next(); translateAndPublish(translator, sequence); } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#tryPublishEvent(EventTranslator) */ @Override public boolean tryPublishEvent(EventTranslator translator) { try { final long sequence = sequencer.tryNext(); translateAndPublish(translator, sequence); return true; } catch (InsufficientCapacityException e) { return false; } } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ private void translateAndPublish(EventTranslatorOneArg translator, long sequence, A arg0) { try { translator.translateTo(get(sequence), sequence, arg0); } finally { sequencer.publish(sequence); } } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public void publishEvent(EventTranslatorOneArg translator, A arg0) { final long sequence = sequencer.next(); translateAndPublish(translator, sequence, arg0); } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public boolean tryPublishEvent(EventTranslatorOneArg translator, A arg0) { try { final long sequence = sequencer.tryNext(); translateAndPublish(translator, sequence, arg0); return true; } catch (InsufficientCapacityException e) { return false; } } private void translateAndPublish(EventTranslatorTwoArg translator, long sequence, A arg0, B arg1) { try { translator.translateTo(get(sequence), sequence, arg0, arg1); } finally { sequencer.publish(sequence); } } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public void publishEvent(EventTranslatorTwoArg translator, A arg0, B arg1) { final long sequence = sequencer.next(); translateAndPublish(translator, sequence, arg0, arg1); } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public boolean tryPublishEvent(EventTranslatorTwoArg translator, A arg0, B arg1) { try { final long sequence = sequencer.tryNext(); translateAndPublish(translator, sequence, arg0, arg1); return true; } catch (InsufficientCapacityException e) { return false; } } private void translateAndPublish( EventTranslatorThreeArg translator, long sequence, A arg0, B arg1, C arg2) { try { translator.translateTo(get(sequence), sequence, arg0, arg1, arg2); } finally { sequencer.publish(sequence); } } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public void publishEvent(EventTranslatorThreeArg translator, A arg0, B arg1, C arg2) { final long sequence = sequencer.next(); translateAndPublish(translator, sequence, arg0, arg1, arg2); } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public boolean tryPublishEvent(EventTranslatorThreeArg translator, A arg0, B arg1, C arg2) { try { final long sequence = sequencer.tryNext(); translateAndPublish(translator, sequence, arg0, arg1, arg2); return true; } catch (InsufficientCapacityException e) { return false; } } private void translateAndPublish(EventTranslatorVararg translator, long sequence, Object... args) { try { translator.translateTo(get(sequence), sequence, args); } finally { sequencer.publish(sequence); } } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public void publishEvent(EventTranslatorVararg translator, Object... args) { final long sequence = sequencer.next(); translateAndPublish(translator, sequence, args); } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public boolean tryPublishEvent(EventTranslatorVararg translator, Object... args) { try { final long sequence = sequencer.tryNext(); translateAndPublish(translator, sequence, args); return true; } catch (InsufficientCapacityException e) { return false; } } private void checkBounds(final EventTranslator[] translators, final int batchStartsAt, final int batchSize) { checkBatchSizing(batchStartsAt, batchSize); batchOverRuns(translators, batchStartsAt, batchSize); } private void batchOverRuns(final A[] arg0, final int batchStartsAt, final int batchSize) { if (batchStartsAt + batchSize > arg0.length) { throw new IllegalArgumentException( "A batchSize of: " + batchSize + " with batchStatsAt of: " + batchStartsAt + " will overrun the available number of arguments: " + (arg0.length - batchStartsAt)); } } private void checkBatchSizing(int batchStartsAt, int batchSize) { if (batchStartsAt < 0 || batchSize < 0) { throw new IllegalArgumentException("Both batchStartsAt and batchSize must be positive but got: batchStartsAt " + batchStartsAt + " and batchSize " + batchSize); } else if (batchSize > bufferSize) { throw new IllegalArgumentException("The ring buffer cannot accommodate " + batchSize + " it only has space for " + bufferSize + " entities."); } } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public void publishEvents(EventTranslator[] translators) { publishEvents(translators, 0, translators.length); } private void translateAndPublishBatch( final EventTranslator[] translators, int batchStartsAt, final int batchSize, final long finalSequence) { final long initialSequence = finalSequence - (batchSize - 1); try { long sequence = initialSequence; final int batchEndsAt = batchStartsAt + batchSize; for (int i = batchStartsAt; i < batchEndsAt; i++) { final EventTranslator translator = translators[i]; translator.translateTo(get(sequence), sequence++); } } finally { sequencer.publish(initialSequence, finalSequence); } } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public void publishEvents(EventTranslator[] translators, int batchStartsAt, int batchSize) { checkBounds(translators, batchStartsAt, batchSize); final long finalSequence = sequencer.next(batchSize); translateAndPublishBatch(translators, batchStartsAt, batchSize, finalSequence); } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public boolean tryPublishEvents(EventTranslator[] translators) { return false; } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public boolean tryPublishEvents(EventTranslator[] translators, int batchStartsAt, int batchSize) { checkBounds(translators, batchStartsAt, batchSize); try { final long finalSequence = sequencer.tryNext(batchSize); translateAndPublishBatch(translators, batchStartsAt, batchSize, finalSequence); return true; } catch (InsufficientCapacityException e) { return false; } } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public void publishEvents(EventTranslatorOneArg translator, A[] arg0) { publishEvents(translator, 0, arg0.length, arg0); } private void checkBounds(final A[] arg0, final int batchStartsAt, final int batchSize) { checkBatchSizing(batchStartsAt, batchSize); batchOverRuns(arg0, batchStartsAt, batchSize); } private void translateAndPublishBatch( final EventTranslatorOneArg translator, final A[] arg0, int batchStartsAt, final int batchSize, final long finalSequence) { final long initialSequence = finalSequence - (batchSize - 1); try { long sequence = initialSequence; final int batchEndsAt = batchStartsAt + batchSize; for (int i = batchStartsAt; i < batchEndsAt; i++) { translator.translateTo(get(sequence), sequence++, arg0[i]); } } finally { sequencer.publish(initialSequence, finalSequence); } } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public void publishEvents(EventTranslatorOneArg translator, int batchStartsAt, int batchSize, A[] arg0) { checkBounds(arg0, batchStartsAt, batchSize); final long finalSequence = sequencer.next(batchSize); translateAndPublishBatch(translator, arg0, batchStartsAt, batchSize, finalSequence); } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public boolean tryPublishEvents(EventTranslatorOneArg translator, A[] arg0) { return tryPublishEvents(translator, 0, arg0.length, arg0); } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public boolean tryPublishEvents(EventTranslatorOneArg translator, int batchStartsAt, int batchSize, A[] arg0) { checkBounds(arg0, batchStartsAt, batchSize); try { final long finalSequence = sequencer.tryNext(batchSize); translateAndPublishBatch(translator, arg0, batchStartsAt, batchSize, finalSequence); return true; } catch (InsufficientCapacityException e) { return false; } } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public void publishEvents(EventTranslatorTwoArg translator, A[] arg0, B[] arg1) { publishEvents(translator, 0, arg0.length, arg0, arg1); } private void checkBounds(final A[] arg0, final B[] arg1, final int batchStartsAt, final int batchSize) { checkBatchSizing(batchStartsAt, batchSize); batchOverRuns(arg0, batchStartsAt, batchSize); batchOverRuns(arg1, batchStartsAt, batchSize); } private void translateAndPublishBatch( final EventTranslatorTwoArg translator, final A[] arg0, final B[] arg1, int batchStartsAt, int batchSize, final long finalSequence) { final long initialSequence = finalSequence - (batchSize - 1); try { long sequence = initialSequence; final int batchEndsAt = batchStartsAt + batchSize; for (int i = batchStartsAt; i < batchEndsAt; i++) { translator.translateTo(get(sequence), sequence++, arg0[i], arg1[i]); } } finally { sequencer.publish(initialSequence, finalSequence); } } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public void publishEvents(EventTranslatorTwoArg translator, int batchStartsAt, int batchSize, A[] arg0, B[] arg1) { checkBounds(arg0, arg1, batchStartsAt, batchSize); final long finalSequence = sequencer.next(batchSize); translateAndPublishBatch(translator, arg0, arg1, batchStartsAt, batchSize, finalSequence); } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public boolean tryPublishEvents(EventTranslatorTwoArg translator, A[] arg0, B[] arg1) { return tryPublishEvents(translator, 0, arg0.length, arg0, arg1); } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public boolean tryPublishEvents(EventTranslatorTwoArg translator, int batchStartsAt, int batchSize, A[] arg0, B[] arg1) { checkBounds(arg0, arg1, batchStartsAt, batchSize); try { final long finalSequence = sequencer.tryNext(batchSize); translateAndPublishBatch(translator, arg0, arg1, batchStartsAt, batchSize, finalSequence); return true; } catch (InsufficientCapacityException e) { return false; } } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public void publishEvents(EventTranslatorThreeArg translator, A[] arg0, B[] arg1, C[] arg2) { publishEvents(translator, 0, arg0.length, arg0, arg1, arg2); } private void checkBounds( final A[] arg0, final B[] arg1, final C[] arg2, final int batchStartsAt, final int batchSize) { checkBatchSizing(batchStartsAt, batchSize); batchOverRuns(arg0, batchStartsAt, batchSize); batchOverRuns(arg1, batchStartsAt, batchSize); batchOverRuns(arg2, batchStartsAt, batchSize); } private void translateAndPublishBatch( final EventTranslatorThreeArg translator, final A[] arg0, final B[] arg1, final C[] arg2, int batchStartsAt, final int batchSize, final long finalSequence) { final long initialSequence = finalSequence - (batchSize - 1); try { long sequence = initialSequence; final int batchEndsAt = batchStartsAt + batchSize; for (int i = batchStartsAt; i < batchEndsAt; i++) { translator.translateTo(get(sequence), sequence++, arg0[i], arg1[i], arg2[i]); } } finally { sequencer.publish(initialSequence, finalSequence); } } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public void publishEvents(EventTranslatorThreeArg translator, int batchStartsAt, int batchSize, A[] arg0, B[] arg1, C[] arg2) { checkBounds(arg0, arg1, arg2, batchStartsAt, batchSize); final long finalSequence = sequencer.next(batchSize); translateAndPublishBatch(translator, arg0, arg1, arg2, batchStartsAt, batchSize, finalSequence); } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public boolean tryPublishEvents(EventTranslatorThreeArg translator, A[] arg0, B[] arg1, C[] arg2) { return tryPublishEvents(translator, 0, arg0.length, arg0, arg1, arg2); } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public boolean tryPublishEvents(EventTranslatorThreeArg translator, int batchStartsAt, int batchSize, A[] arg0, B[] arg1, C[] arg2) { checkBounds(arg0, arg1, arg2, batchStartsAt, batchSize); try { final long finalSequence = sequencer.tryNext(batchSize); translateAndPublishBatch(translator, arg0, arg1, arg2, batchStartsAt, batchSize, finalSequence); return true; } catch (InsufficientCapacityException e) { return false; } } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public void publishEvents(EventTranslatorVararg translator, Object[]... args) { publishEvents(translator, 0, args.length, args); } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public void publishEvents(EventTranslatorVararg translator, int batchStartsAt, int batchSize, Object[]... args) { checkBounds(batchStartsAt, batchSize, args); final long finalSequence = sequencer.next(batchSize); translateAndPublishBatch(translator, batchStartsAt, batchSize, finalSequence, args); } private void checkBounds(final int batchStartsAt, final int batchSize, final Object[][] args) { checkBatchSizing(batchStartsAt, batchSize); batchOverRuns(args, batchStartsAt, batchSize); } private void translateAndPublishBatch( final EventTranslatorVararg translator, int batchStartsAt, final int batchSize, final long finalSequence, final Object[][] args) { final long initialSequence = finalSequence - (batchSize - 1); try { long sequence = initialSequence; final int batchEndsAt = batchStartsAt + batchSize; for (int i = batchStartsAt; i < batchEndsAt; i++) { translator.translateTo(get(sequence), sequence++, args[i]); } } finally { sequencer.publish(initialSequence, finalSequence); } } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public boolean tryPublishEvents(EventTranslatorVararg translator, Object[]... args) { return tryPublishEvents(translator, 0, args.length, args); } /** * @see io.mycat.memory.unsafe.ringbuffer.common.event.EventSink#publishEvent(EventTranslator) */ @Override public boolean tryPublishEvents(EventTranslatorVararg translator, int batchStartsAt, int batchSize, Object[]... args) { return false; } @Override public int getBufferSize() { return bufferSize; } @Override public boolean hasAvailableCapacity(int requiredCapacity) { return sequencer.hasAvailableCapacity(requiredCapacity); } @Override public long remainingCapacity() { return sequencer.remainingCapacity(); } @Override public long next() { return sequencer.next(); } @Override public long next(int n) { return sequencer.next(n); } @Override public long tryNext() throws InsufficientCapacityException { return sequencer.tryNext(); } @Override public long tryNext(int n) throws InsufficientCapacityException { return sequencer.tryNext(n); } @Override public void publish(long sequence) { sequencer.publish(sequence); } @Override public void publish(long lo, long hi) { sequencer.publish(lo, hi); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/common/Cursored.java ================================================ package io.mycat.memory.unsafe.ringbuffer.common; /** * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/23 */ public interface Cursored { long getCursor(); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/common/DataProvider.java ================================================ package io.mycat.memory.unsafe.ringbuffer.common; /** * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/29 */ public interface DataProvider { /** * 获取sequence对应的对象 * @param sequence * @return */ public T get(long sequence); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/common/Sequenced.java ================================================ package io.mycat.memory.unsafe.ringbuffer.common; import io.mycat.memory.unsafe.ringbuffer.exception.InsufficientCapacityException; /** * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/23 */ public interface Sequenced { /** * @return ringBuffer的大小 */ int getBufferSize(); /** * @param requiredCapacity 需要的大小 * @return true ringBuffer的剩余空间足够 | false ringBuffer的剩余空间不足 */ boolean hasAvailableCapacity(final int requiredCapacity); /** * @return ringBuffer的剩余空间 */ long remainingCapacity(); /** * 申请下一个sequence(value)作为生产event的位置 * @return sequence的value */ long next(); /** * 申请下n个sequence(value)作为生产多个event的位置 * @param n * @return 最高的sequence的value */ long next(int n); /** * 尝试申请下一个sequence(value)作为生产event的位置 * @return sequence的value * @throws InsufficientCapacityException */ long tryNext() throws InsufficientCapacityException; /** * 尝试申请下n个sequence(value)作为生产多个event的位置 * @param n * @return 最高的sequence的value * @throws InsufficientCapacityException */ long tryNext(int n) throws InsufficientCapacityException; /** * 发布一个Sequence，一般在这个Sequence对应位置的Event被填充后 * @param sequence */ void publish(long sequence); /** * 发布多个Sequence，一般在这些Sequence对应位置的Event被填充后 * @param lo 第一个sequence的value * @param hi 最后一个sequence的value */ void publish(long lo, long hi); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/common/barrier/SequenceBarrier.java ================================================ package io.mycat.memory.unsafe.ringbuffer.common.barrier; import io.mycat.memory.unsafe.ringbuffer.exception.AlertException; import io.mycat.memory.unsafe.ringbuffer.exception.TimeoutException; /** * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/24 */ public interface SequenceBarrier { /** * 等待给定的sequence值可以被消费 * * @param sequence 等待的sequence值 * @return 可以消费的最大sequence值 * @throws AlertException 当Disruptor的状态改变时会抛出 * @throws InterruptedException 唤醒线程 * @throws TimeoutException 超过最大等待时间 */ long waitFor(long sequence) throws AlertException, InterruptedException, TimeoutException; /** * 获取当前可以消费的cursor值 * * @return 当前可以消费的cursor值（已经被publish的） */ long getCursor(); /** * alert状态 * * @return true 如果被alerted */ boolean isAlerted(); /** * 进入alert状态 */ void alert(); /** * 清除当前alert状态 */ void clearAlert(); /** * 检查是否被alerted，如果是，则抛出{@link AlertException} * * @throws AlertException if alert has been raised. */ void checkAlert() throws AlertException; } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/common/event/EventFactory.java ================================================ package io.mycat.memory.unsafe.ringbuffer.common.event; /** * 用户实现，生成Event的接口 * * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/29 */ public interface EventFactory { T newInstance(); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/common/event/EventSequencer.java ================================================ package io.mycat.memory.unsafe.ringbuffer.common.event; import io.mycat.memory.unsafe.ringbuffer.common.DataProvider; import io.mycat.memory.unsafe.ringbuffer.common.Sequenced; /** * EventSequencer接口没有自己的方法，只是为了将Sequencer和DataProvider合起来。 * * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/29 */ public interface EventSequencer extends DataProvider, Sequenced { } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/common/event/EventSink.java ================================================ package io.mycat.memory.unsafe.ringbuffer.common.event; /** * Event槽接口 * * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/29 */ public interface EventSink { /** * 申请下一个Sequence->申请成功则获取对应槽的Event->利用translator初始化并填充对应槽的Event->发布Event * @param translator translator用户实现，用于初始化Event，这里是不带参数Translator */ void publishEvent(EventTranslator translator); /** * 尝试申请下一个Sequence->申请成功则获取对应槽的Event->利用translator初始化并填充对应槽的Event->发布Event * 若空间不足，则立即失败返回 * @param translator translator用户实现，用于初始化Event，这里是不带参数Translator * @return 成功true，失败false */ boolean tryPublishEvent(EventTranslator translator); void publishEvent(EventTranslatorOneArg translator, A arg0); boolean tryPublishEvent(EventTranslatorOneArg translator, A arg0); void publishEvent(EventTranslatorTwoArg translator, A arg0, B arg1); boolean tryPublishEvent(EventTranslatorTwoArg translator, A arg0, B arg1); void publishEvent(EventTranslatorThreeArg translator, A arg0, B arg1, C arg2); boolean tryPublishEvent(EventTranslatorThreeArg translator, A arg0, B arg1, C arg2); void publishEvent(EventTranslatorVararg translator, Object... args); boolean tryPublishEvent(EventTranslatorVararg translator, Object... args); /** * 包括申请多个Sequence->申请成功则获取对应槽的Event->利用每个translator初始化并填充每个对应槽的Event->发布Event * @param translators */ void publishEvents(EventTranslator[] translators); void publishEvents(EventTranslator[] translators, int batchStartsAt, int batchSize); boolean tryPublishEvents(EventTranslator[] translators); boolean tryPublishEvents(EventTranslator[] translators, int batchStartsAt, int batchSize); void publishEvents(EventTranslatorOneArg translator, A[] arg0); void publishEvents(EventTranslatorOneArg translator, int batchStartsAt, int batchSize, A[] arg0); boolean tryPublishEvents(EventTranslatorOneArg translator, A[] arg0); boolean tryPublishEvents(EventTranslatorOneArg translator, int batchStartsAt, int batchSize, A[] arg0); void publishEvents(EventTranslatorTwoArg translator, A[] arg0, B[] arg1); void publishEvents( EventTranslatorTwoArg translator, int batchStartsAt, int batchSize, A[] arg0, B[] arg1); boolean tryPublishEvents(EventTranslatorTwoArg translator, A[] arg0, B[] arg1); boolean tryPublishEvents( EventTranslatorTwoArg translator, int batchStartsAt, int batchSize, A[] arg0, B[] arg1); void publishEvents(EventTranslatorThreeArg translator, A[] arg0, B[] arg1, C[] arg2); void publishEvents( EventTranslatorThreeArg translator, int batchStartsAt, int batchSize, A[] arg0, B[] arg1, C[] arg2); boolean tryPublishEvents(EventTranslatorThreeArg translator, A[] arg0, B[] arg1, C[] arg2); boolean tryPublishEvents( EventTranslatorThreeArg translator, int batchStartsAt, int batchSize, A[] arg0, B[] arg1, C[] arg2); void publishEvents(EventTranslatorVararg translator, Object[]... args); void publishEvents(EventTranslatorVararg translator, int batchStartsAt, int batchSize, Object[]... args); boolean tryPublishEvents(EventTranslatorVararg translator, Object[]... args); boolean tryPublishEvents(EventTranslatorVararg translator, int batchStartsAt, int batchSize, Object[]... args); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/common/event/EventTranslator.java ================================================ package io.mycat.memory.unsafe.ringbuffer.common.event; /** * Event初始化接口，生产者通过实现这个接口，在发布Event时，对应实现的translateTo方法会被调用 * * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/29 */ public interface EventTranslator { void translateTo(final T event, long sequence); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/common/event/EventTranslatorOneArg.java ================================================ package io.mycat.memory.unsafe.ringbuffer.common.event; /** * Event初始化接口，生产者通过实现这个接口，在发布Event时，对应实现的translateTo方法会被调用 * 这里用户可以传一个参数 * * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/29 */ public interface EventTranslatorOneArg { void translateTo(final T event, long sequence, final A arg0); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/common/event/EventTranslatorThreeArg.java ================================================ package io.mycat.memory.unsafe.ringbuffer.common.event; /** * Event初始化接口，生产者通过实现这个接口，在发布Event时，对应实现的translateTo方法会被调用 * 这里用户可以传三个参数 * * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/29 */ public interface EventTranslatorThreeArg { void translateTo(final T event, long sequence, final A arg0, final B arg1, final C arg2); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/common/event/EventTranslatorTwoArg.java ================================================ package io.mycat.memory.unsafe.ringbuffer.common.event; /** * Event初始化接口，生产者通过实现这个接口，在发布Event时，对应实现的translateTo方法会被调用 * 这里用户可以传两个参数 * * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/29 */ public interface EventTranslatorTwoArg { void translateTo(final T event, long sequence, final A arg0, final B arg1); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/common/event/EventTranslatorVararg.java ================================================ package io.mycat.memory.unsafe.ringbuffer.common.event; /** * Event初始化接口，生产者通过实现这个接口，在发布Event时，对应实现的translateTo方法会被调用 * 这里用户可以传多个参数 * * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/29 */ public interface EventTranslatorVararg { void translateTo(final T event, long sequence, final Object... args); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/common/sequence/Sequence.java ================================================ package io.mycat.memory.unsafe.ringbuffer.common.sequence; import io.mycat.memory.unsafe.Platform; /** * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/23 */ class LhsPadding { protected long p1, p2, p3, p4, p5, p6, p7; } class Value extends LhsPadding { protected volatile long value; } class RhsPadding extends Value { protected long p9, p10, p11, p12, p13, p14, p15; } /** *

Concurrent sequence class used for tracking the progress of * the ring buffer and event processors. Support a number * of concurrent operations including CAS and order writes. * *

Also attempts to be more efficient with regards to false * sharing by adding padding around the volatile field. */ public class Sequence extends RhsPadding { public static final long INITIAL_VALUE = -1L; private static final long VALUE_OFFSET; static { try { VALUE_OFFSET = Platform.objectFieldOffset(Value.class.getDeclaredField("value")); } catch (final Exception e) { throw new RuntimeException(e); } } /** * Create a sequence initialised to -1. */ public Sequence() { this(INITIAL_VALUE); } /** * Create a sequence with a specified initial value. * * @param initialValue The initial value for this sequence. */ public Sequence(final long initialValue) { Platform.putOrderedLong(this, VALUE_OFFSET, initialValue); } /** * Perform a volatile read of this sequence's value. * * @return The current value of the sequence. */ public long get() { return value; } /** * Perform an ordered write of this sequence. The intent is * a Store/Store barrier between this write and any previous * store. * * @param value The new value for the sequence. */ public void set(final long value) { Platform.putOrderedLong(this, VALUE_OFFSET, value); } /** * Performs a volatile write of this sequence. The intent is * a Store/Store barrier between this write and any previous * write and a Store/Load barrier between this write and any * subsequent volatile read. * * @param value The new value for the sequence. */ public void setVolatile(final long value) { Platform.putLongVolatile(this, VALUE_OFFSET, value); } /** * Perform a compare and set operation on the sequence. * * @param expectedValue The expected current value. * @param newValue The value to update to. * @return true if the operation succeeds, false otherwise. */ public boolean compareAndSet(final long expectedValue, final long newValue) { return Platform.compareAndSwapLong(this, VALUE_OFFSET, expectedValue, newValue); } /** * Atomically increment the sequence by one. * * @return The value after the increment */ public long incrementAndGet() { return addAndGet(1L); } /** * Atomically add the supplied value. * * @param increment The value to add to the sequence. * @return The value after the increment. */ public long addAndGet(final long increment) { long currentValue; long newValue; do { currentValue = get(); newValue = currentValue + increment; } while (!compareAndSet(currentValue, newValue)); return newValue; } @Override public String toString() { return Long.toString(get()); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/common/sequence/SequenceGroups.java ================================================ package io.mycat.memory.unsafe.ringbuffer.common.sequence; import io.mycat.memory.unsafe.ringbuffer.common.Cursored; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import static java.util.Arrays.copyOf; /** * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/25 */ public class SequenceGroups { /** * 原子添加sequences * * @param holder 原子更新的域所属的类对象 * @param updater 原子更新的域对象 * @param cursor 定位 * @param sequencesToAdd 要添加的sequences * @param */ public static void addSequences( final T holder, final AtomicReferenceFieldUpdater updater, final Cursored cursor, final Sequence... sequencesToAdd) { long cursorSequence; Sequence[] updatedSequences; Sequence[] currentSequences; //在更新成功之前，一直重新读取currentSequences，扩充为添加所有sequence之后的updatedSequences do { currentSequences = updater.get(holder); updatedSequences = copyOf(currentSequences, currentSequences.length + sequencesToAdd.length); cursorSequence = cursor.getCursor(); int index = currentSequences.length; //将新的sequences的值设置为cursorSequence for (Sequence sequence : sequencesToAdd) { sequence.set(cursorSequence); updatedSequences[index++] = sequence; } } while (!updater.compareAndSet(holder, currentSequences, updatedSequences)); cursorSequence = cursor.getCursor(); for (Sequence sequence : sequencesToAdd) { sequence.set(cursorSequence); } } /** * 原子移除某个指定的sequence * * @param holder 原子更新的域所属的类对象 * @param sequenceUpdater 原子更新的域对象 * @param sequence 要移除的sequence * @param * @return */ public static boolean removeSequence( final T holder, final AtomicReferenceFieldUpdater sequenceUpdater, final Sequence sequence) { int numToRemove; Sequence[] oldSequences; Sequence[] newSequences; do { oldSequences = sequenceUpdater.get(holder); numToRemove = countMatching(oldSequences, sequence); if (0 == numToRemove) { break; } final int oldSize = oldSequences.length; newSequences = new Sequence[oldSize - numToRemove]; for (int i = 0, pos = 0; i < oldSize; i++) { final Sequence testSequence = oldSequences[i]; if (sequence != testSequence) { newSequences[pos++] = testSequence; } } } while (!sequenceUpdater.compareAndSet(holder, oldSequences, newSequences)); return numToRemove != 0; } private static int countMatching(T[] values, final T toMatch) { int numToRemove = 0; for (T value : values) { if (value == toMatch) // Specifically uses identity { numToRemove++; } } return numToRemove; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/common/waitStrategy/WaitStrategy.java ================================================ package io.mycat.memory.unsafe.ringbuffer.common.waitStrategy; import io.mycat.memory.unsafe.ringbuffer.common.barrier.SequenceBarrier; import io.mycat.memory.unsafe.ringbuffer.common.sequence.Sequence; import io.mycat.memory.unsafe.ringbuffer.exception.AlertException; import io.mycat.memory.unsafe.ringbuffer.exception.TimeoutException; /** * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/24 */ public interface WaitStrategy { /** * @param sequence 需要等待available的sequence * @param cursor 对应RingBuffer的Cursor * @param dependentSequence 需要等待（依赖）的Sequence * @param barrier 多消费者注册的SequenceBarrier * @return 已经available的sequence * @throws AlertException * @throws InterruptedException * @throws TimeoutException */ long waitFor(long sequence, Sequence cursor, Sequence dependentSequence, SequenceBarrier barrier) throws AlertException, InterruptedException, TimeoutException; /** * 唤醒所有等待的消费者 */ void signalAllWhenBlocking(); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/common/waitStrategy/impl/BlockingWaitStrategy.java ================================================ package io.mycat.memory.unsafe.ringbuffer.common.waitStrategy.impl; import io.mycat.memory.unsafe.ringbuffer.common.barrier.SequenceBarrier; import io.mycat.memory.unsafe.ringbuffer.common.sequence.Sequence; import io.mycat.memory.unsafe.ringbuffer.common.waitStrategy.WaitStrategy; import io.mycat.memory.unsafe.ringbuffer.exception.AlertException; import io.mycat.memory.unsafe.ringbuffer.exception.TimeoutException; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; /** * @author lmax.Disruptor * @version 3.3.5 * @date 2016/8/1 */ public class BlockingWaitStrategy implements WaitStrategy { private final Lock lock = new ReentrantLock(); private final Condition processorNotifyCondition = lock.newCondition(); @Override public long waitFor(long sequence, Sequence cursorSequence, Sequence dependentSequence, SequenceBarrier barrier) throws AlertException, InterruptedException { long availableSequence; if (cursorSequence.get() < sequence) { lock.lock(); try { while (cursorSequence.get() < sequence) { //检查是否Alert，如果Alert，则抛出AlertException //Alert在这里代表对应的消费者被halt停止了 barrier.checkAlert(); //在processorNotifyCondition上等待唤醒 processorNotifyCondition.await(); } } finally { lock.unlock(); } } while ((availableSequence = dependentSequence.get()) < sequence) { barrier.checkAlert(); } return availableSequence; } @Override public void signalAllWhenBlocking() { lock.lock(); try { //生产者生产消息后，会唤醒所有等待的消费者线程 processorNotifyCondition.signalAll(); } finally { lock.unlock(); } } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/common/waitStrategy/impl/BusySpinWaitStrategy.java ================================================ package io.mycat.memory.unsafe.ringbuffer.common.waitStrategy.impl; import io.mycat.memory.unsafe.ringbuffer.common.barrier.SequenceBarrier; import io.mycat.memory.unsafe.ringbuffer.common.sequence.Sequence; import io.mycat.memory.unsafe.ringbuffer.common.waitStrategy.WaitStrategy; import io.mycat.memory.unsafe.ringbuffer.exception.AlertException; /** * @author lmax.Disruptor * @version 3.3.5 * @date 2016/8/1 */ public class BusySpinWaitStrategy implements WaitStrategy { @Override public long waitFor( final long sequence, Sequence cursor, final Sequence dependentSequence, final SequenceBarrier barrier) throws AlertException, InterruptedException { long availableSequence; //一直while自旋检查 while ((availableSequence = dependentSequence.get()) < sequence) { barrier.checkAlert(); } return availableSequence; } @Override public void signalAllWhenBlocking() { } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/common/waitStrategy/impl/SleepingWaitStrategy.java ================================================ package io.mycat.memory.unsafe.ringbuffer.common.waitStrategy.impl; import io.mycat.memory.unsafe.ringbuffer.common.barrier.SequenceBarrier; import io.mycat.memory.unsafe.ringbuffer.common.sequence.Sequence; import io.mycat.memory.unsafe.ringbuffer.common.waitStrategy.WaitStrategy; import io.mycat.memory.unsafe.ringbuffer.exception.AlertException; import java.util.concurrent.locks.LockSupport; /** * @author lmax.Disruptor * @version 3.3.5 * @date 2016/8/1 */ public class SleepingWaitStrategy implements WaitStrategy { //重试200次 private static final int DEFAULT_RETRIES = 200; private final int retries; public SleepingWaitStrategy() { this(DEFAULT_RETRIES); } public SleepingWaitStrategy(int retries) { this.retries = retries; } @Override public long waitFor( final long sequence, Sequence cursor, final Sequence dependentSequence, final SequenceBarrier barrier) throws AlertException, InterruptedException { long availableSequence; int counter = retries; //直接检查dependentSequence.get() < sequence while ((availableSequence = dependentSequence.get()) < sequence) { counter = applyWaitMethod(barrier, counter); } return availableSequence; } @Override public void signalAllWhenBlocking() { } private int applyWaitMethod(final SequenceBarrier barrier, int counter) throws AlertException { //检查是否需要终止 barrier.checkAlert(); //如果在200~100,重试 if (counter > 100) { --counter; } //如果在100~0,调用Thread.yield()让出CPU else if (counter > 0) { --counter; Thread.yield(); } //<0的话，利用LockSupport.parkNanos(1L)来sleep最小时间 else { LockSupport.parkNanos(1L); } return counter; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/exception/AlertException.java ================================================ package io.mycat.memory.unsafe.ringbuffer.exception; /** * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/24 */ public class AlertException extends Exception { public static final AlertException INSTANCE = new AlertException(); private AlertException() { } @Override public Throwable fillInStackTrace() { return this; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/exception/InsufficientCapacityException.java ================================================ package io.mycat.memory.unsafe.ringbuffer.exception; /** * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/23 */ @SuppressWarnings("serial") public final class InsufficientCapacityException extends Exception { public static final InsufficientCapacityException INSTANCE = new InsufficientCapacityException(); private InsufficientCapacityException() { } @Override public synchronized Throwable fillInStackTrace() { return this; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/exception/TimeoutException.java ================================================ package io.mycat.memory.unsafe.ringbuffer.exception; /** * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/24 */ public class TimeoutException extends Exception { public static final TimeoutException INSTANCE = new TimeoutException(); private TimeoutException() { } @Override public synchronized Throwable fillInStackTrace() { return this; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/producer/AbstractSequencer.java ================================================ package io.mycat.memory.unsafe.ringbuffer.producer; import io.mycat.memory.unsafe.ringbuffer.common.sequence.Sequence; import io.mycat.memory.unsafe.ringbuffer.common.barrier.SequenceBarrier; import io.mycat.memory.unsafe.ringbuffer.common.sequence.SequenceGroups; import io.mycat.memory.unsafe.ringbuffer.common.waitStrategy.WaitStrategy; import io.mycat.memory.unsafe.ringbuffer.exception.InsufficientCapacityException; import io.mycat.memory.unsafe.ringbuffer.utils.Util; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; /** * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/24 */ public abstract class AbstractSequencer implements Sequencer { private static final AtomicReferenceFieldUpdater SEQUENCE_UPDATER = AtomicReferenceFieldUpdater.newUpdater(AbstractSequencer.class, Sequence[].class, "gatingSequences"); protected final int bufferSize; protected final WaitStrategy waitStrategy; protected final Sequence cursor = new Sequence(Sequencer.INITIAL_CURSOR_VALUE); protected volatile Sequence[] gatingSequences = new Sequence[0]; public AbstractSequencer(int bufferSize, WaitStrategy waitStrategy) { if (bufferSize < 1) { throw new IllegalArgumentException("bufferSize must not be less than 1"); } if (Integer.bitCount(bufferSize) != 1) { throw new IllegalArgumentException("bufferSize must be a power of 2"); } this.bufferSize = bufferSize; this.waitStrategy = waitStrategy; } @Override public final long getCursor() { return cursor.get(); } @Override public final int getBufferSize() { return bufferSize; } @Override public void addGatingSequences(Sequence... gatingSequences) { SequenceGroups.addSequences(this, SEQUENCE_UPDATER, this, gatingSequences); } @Override public boolean removeGatingSequence(Sequence sequence) { return SequenceGroups.removeSequence(this, SEQUENCE_UPDATER, sequence); } @Override public long getMinimumSequence() { return Util.getMinimumSequence(gatingSequences, cursor.get()); } public SequenceBarrier newBarrier(Sequence... sequencesToTrack) { return null; //TODO 完成SequenceBarrier // return new ProcessingSequenceBarrier(this, waitStrategy, cursor, sequencesToTrack); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/producer/MultiProducerSequencer.java ================================================ package io.mycat.memory.unsafe.ringbuffer.producer; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.ringbuffer.common.sequence.Sequence; import io.mycat.memory.unsafe.ringbuffer.common.waitStrategy.WaitStrategy; import io.mycat.memory.unsafe.ringbuffer.exception.InsufficientCapacityException; import io.mycat.memory.unsafe.ringbuffer.utils.Util; import java.util.concurrent.locks.LockSupport; /** * 多生产者类，线程安全，与单一生产者不同的是，这里的cursor不再是可以消费的标记，而是多线程生产者抢占的标记 * 可以消费的sequence由availableBuffer来判断标识 * * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/24 */ public class MultiProducerSequencer extends AbstractSequencer{ private static final long BASE = Platform.arrayBaseOffset(int[].class); private static final long SCALE = Platform.arrayIndexScale(int[].class); private final Sequence gatingSequenceCache = new Sequence(Sequencer.INITIAL_CURSOR_VALUE); private final int[] availableBuffer; //利用对2^n取模 = 对2^n -1 取与运算原理，indexMask=bufferSize - 1 private final int indexMask; //就是上面的n，用来定位某个sequence到底转了多少圈，用来标识已被发布的sequence。 //为什么不直接将sequence存入availableBuffer，因为这样sequence值会过大，很容易溢出 private final int indexShift; public MultiProducerSequencer(int bufferSize, final WaitStrategy waitStrategy) { super(bufferSize, waitStrategy); availableBuffer = new int[bufferSize]; indexMask = bufferSize - 1; indexShift = Util.log2(bufferSize); initialiseAvailableBuffer(); } /** * 将availableBuffer都初始化为-1 */ private void initialiseAvailableBuffer() { for (int i = availableBuffer.length - 1; i != 0; i--) { setAvailableBufferValue(i, -1); } setAvailableBufferValue(0, -1); } /** * 发布某个sequence之前的都可以被消费了需要将availableBuffer上对应sequence下标的值设置为第几次用到这个槽 * @param sequence */ private void setAvailable(final long sequence) { setAvailableBufferValue(calculateIndex(sequence), calculateAvailabilityFlag(sequence)); } /** * 某个sequence右移indexShift，代表这个Sequence是第几次用到这个ringBuffer的某个槽，也就是这个sequence转了多少圈 * @param sequence * @return */ private int calculateAvailabilityFlag(final long sequence) { return (int) (sequence >>> indexShift); } /** * 定位ringBuffer上某个槽用于生产event，对2^n取模 = 对2^n -1 * @param sequence * @return */ private int calculateIndex(final long sequence) { return ((int) sequence) & indexMask; } /** * 通过Unsafe更新数组非volatile类型的值 * 数组结构 * -------------- * * 数组头 * BASE * * reference1 * SCALE * * reference2 * SCALE * * reference3 * SCALE * -------------- * @param index * @param flag */ private void setAvailableBufferValue(int index, int flag) { long bufferAddress = (index * SCALE) + BASE; Platform.putOrderedInt(availableBuffer, bufferAddress, flag); } @Override public void claim(long sequence) { cursor.set(sequence); } /** * 用同样的方法计算给定的sequence，判断与availableBuffer对应下标的值是否相等，如果相等证明已被发布可以消费 * @param sequence of the buffer to check * @return */ @Override public boolean isAvailable(long sequence) { int index = calculateIndex(sequence); int flag = calculateAvailabilityFlag(sequence); long bufferAddress = (index * SCALE) + BASE; return Platform.getIntVolatile(availableBuffer, bufferAddress) == flag; } /** * 获取最高的可消费sequence，减少获取次数 * @param nextSequence The sequence to start scanning from. * @param availableSequence The sequence to scan to. * @return */ @Override public long getHighestPublishedSequence(long nextSequence, long availableSequence) { for (long sequence = nextSequence; sequence <= availableSequence; sequence++) { if (!isAvailable(sequence)) { return sequence - 1; } } return availableSequence; } @Override public boolean hasAvailableCapacity(final int requiredCapacity) { return hasAvailableCapacity(gatingSequences, requiredCapacity, cursor.get()); } /** * 与单一生产者验证原理类似 * @param gatingSequences * @param requiredCapacity * @param cursorValue * @return */ private boolean hasAvailableCapacity(Sequence[] gatingSequences, final int requiredCapacity, long cursorValue) { //下一位置加上所需容量减去整个bufferSize，如果为正数，那证明至少转了一圈，则需要检查gatingSequences（由消费者更新里面的Sequence值）以保证不覆盖还未被消费的 //由于最多只能生产不大于整个bufferSize的Events。所以减去一个bufferSize与最小sequence相比较即可 long wrapPoint = (cursorValue + requiredCapacity) - bufferSize; //缓存 long cachedGatingSequence = gatingSequenceCache.get(); //缓存失效条件 if (wrapPoint > cachedGatingSequence || cachedGatingSequence > cursorValue) { long minSequence = Util.getMinimumSequence(gatingSequences, cursorValue); gatingSequenceCache.set(minSequence); //空间不足 if (wrapPoint > minSequence) { return false; } } return true; } @Override public long remainingCapacity() { //与单一生产者的计算方法同理，不考虑并发 long consumed = Util.getMinimumSequence(gatingSequences, cursor.get()); long produced = cursor.get(); return getBufferSize() - (produced - consumed); } @Override public long next() { return next(1); } /** * 用于多个生产者抢占n个RingBuffer槽用于生产Event * * @param n * @return */ @Override public long next(int n) { if (n < 1) { throw new IllegalArgumentException("n must be > 0"); } long current; long next; do { //首先通过缓存判断空间是否足够 current = cursor.get(); next = current + n; long wrapPoint = next - bufferSize; long cachedGatingSequence = gatingSequenceCache.get(); //如果缓存不满足 if (wrapPoint > cachedGatingSequence || cachedGatingSequence > current) { //重新获取最小的 long gatingSequence = Util.getMinimumSequence(gatingSequences, current); //如果空间不足，则唤醒消费者消费，并让出CPU if (wrapPoint > gatingSequence) { waitStrategy.signalAllWhenBlocking(); LockSupport.parkNanos(1); // TODO, should we spin based on the wait strategy? continue; } //重新设置缓存 gatingSequenceCache.set(gatingSequence); } //如果空间足够，尝试CAS更新cursor，更新cursor成功代表成功获取n个槽，退出死循环 else if (cursor.compareAndSet(current, next)) { break; } } while (true); //返回最新的cursor值 return next; } @Override public long tryNext() throws InsufficientCapacityException { return tryNext(1); } @Override public long tryNext(int n) throws InsufficientCapacityException { if (n < 1) { throw new IllegalArgumentException("n must be > 0"); } long current; long next; //尝试获取一次，若不成功，则抛InsufficientCapacityException do { current = cursor.get(); next = current + n; if (!hasAvailableCapacity(gatingSequences, n, current)) { throw InsufficientCapacityException.INSTANCE; } } while (!cursor.compareAndSet(current, next)); return next; } @Override public void publish(long sequence) { setAvailable(sequence); waitStrategy.signalAllWhenBlocking(); } @Override public void publish(long lo, long hi) { for (long l = lo; l <= hi; l++) { setAvailable(l); } waitStrategy.signalAllWhenBlocking(); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/producer/Sequencer.java ================================================ package io.mycat.memory.unsafe.ringbuffer.producer; import io.mycat.memory.unsafe.ringbuffer.common.Cursored; import io.mycat.memory.unsafe.ringbuffer.common.sequence.Sequence; import io.mycat.memory.unsafe.ringbuffer.common.Sequenced; import io.mycat.memory.unsafe.ringbuffer.common.barrier.SequenceBarrier; /** * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/23 */ public interface Sequencer extends Cursored,Sequenced{ /** * -1 为 sequence的起始值 */ long INITIAL_CURSOR_VALUE = -1L; /** * 申请一个特殊的Sequence，只有设定特殊起始值的ringBuffer时才会使用 * * @param sequence The sequence to initialise too. */ void claim(long sequence); /** * 非阻塞，验证一个sequence是否已经被published并且可以消费 * * @param sequence of the buffer to check * @return true if the sequence is available for use, false if not */ boolean isAvailable(long sequence); /** * 将这些sequence加入到需要跟踪处理的gatingSequences中 * * @param gatingSequences The sequences to add. */ void addGatingSequences(Sequence... gatingSequences); /** * 移除某个sequence * * @param sequence to be removed. * @return true if this sequence was found, false otherwise. */ boolean removeGatingSequence(Sequence sequence); /** * 给定一串需要跟踪的sequence，创建SequenceBarrier * SequenceBarrier是用来给多消费者确定消费位置是否可以消费用的 * * @param sequencesToTrack * @return A sequence barrier that will track the specified sequences. * @see SequenceBarrier */ SequenceBarrier newBarrier(Sequence... sequencesToTrack); /** * 获取这个ringBuffer的gatingSequences中最小的一个sequence * * @return The minimum gating sequence or the cursor sequence if */ long getMinimumSequence(); /** * 获取最高可以读取的Sequence * * @param nextSequence The sequence to start scanning from. * @param availableSequence The sequence to scan to. * @return The highest value that can be safely read, will be at least nextSequence - 1. */ long getHighestPublishedSequence(long nextSequence, long availableSequence); /** * 并没有什么用，不实现，注释掉 */ // EventPoller newPoller(DataProvider provider, Sequence... gatingSequences); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/producer/SingleProducerSequencer.java ================================================ package io.mycat.memory.unsafe.ringbuffer.producer; import io.mycat.memory.unsafe.ringbuffer.common.sequence.Sequence; import io.mycat.memory.unsafe.ringbuffer.common.waitStrategy.WaitStrategy; import io.mycat.memory.unsafe.ringbuffer.exception.InsufficientCapacityException; import io.mycat.memory.unsafe.ringbuffer.utils.Util; import java.util.concurrent.locks.LockSupport; /** * 单一生产者相关类，非线程安全 * * @author lmax.Disruptor * @version 3.3.5 * @date 2016/7/24 */ abstract class SingleProducerSequencerPad extends AbstractSequencer { protected long p1, p2, p3, p4, p5, p6, p7; public SingleProducerSequencerPad(int bufferSize, WaitStrategy waitStrategy) { super(bufferSize, waitStrategy); } } abstract class SingleProducerSequencerFields extends SingleProducerSequencerPad { public SingleProducerSequencerFields(int bufferSize, WaitStrategy waitStrategy) { super(bufferSize, waitStrategy); } protected long nextValue = Sequence.INITIAL_VALUE; protected long cachedValue = Sequence.INITIAL_VALUE; } public class SingleProducerSequencer extends SingleProducerSequencerFields{ public SingleProducerSequencer(int bufferSize, final WaitStrategy waitStrategy) { super(bufferSize, waitStrategy); } @Override public void claim(long sequence) { nextValue = sequence; } @Override public boolean isAvailable(long sequence) { return sequence <= cursor.get(); } @Override public long getHighestPublishedSequence(long nextSequence, long availableSequence) { return availableSequence; } @Override public boolean hasAvailableCapacity(int requiredCapacity) { //下一个生产Sequence位置 long nextValue = this.nextValue; //下一位置加上所需容量减去整个bufferSize，如果为正数，那证明至少转了一圈，则需要检查gatingSequences（由消费者更新里面的Sequence值）以保证不覆盖还未被消费的 long wrapPoint = (nextValue + requiredCapacity) - bufferSize; //Disruptor经常用缓存，这里缓存之间所有gatingSequences最小的那个，这样不用每次都遍历一遍gatingSequences，影响效率 long cachedGatingSequence = this.cachedValue; //只要wrapPoint大于缓存的所有gatingSequences最小的那个，就重新检查更新缓存 if (wrapPoint > cachedGatingSequence || cachedGatingSequence > nextValue) { long minSequence = Util.getMinimumSequence(gatingSequences, nextValue); this.cachedValue = minSequence; //空间不足返回false if (wrapPoint > minSequence) { return false; } } //若wrapPoint小于缓存的所有gatingSequences最小的那个，证明可以放心生产 return true; } @Override public long remainingCapacity() { //使用的 = 生产的 - 已经消费的 //剩余容量 = 容量 - 使用的 long nextValue = this.nextValue; long consumed = Util.getMinimumSequence(gatingSequences, nextValue); long produced = nextValue; return getBufferSize() - (produced - consumed); } @Override public long next() { return next(1); } @Override public long next(int n) { if (n < 1) { throw new IllegalArgumentException("n must be > 0"); } long nextValue = this.nextValue; //next方法和之前的hasAvailableCapacity同理，只不过这里是相当于阻塞的 long nextSequence = nextValue + n; long wrapPoint = nextSequence - bufferSize; long cachedGatingSequence = this.cachedValue; if (wrapPoint > cachedGatingSequence || cachedGatingSequence > nextValue) { long minSequence; //只要wrapPoint大于最小的gatingSequences，那么不断唤醒消费者去消费，并利用LockSupport让出CPU，直到wrapPoint不大于最小的gatingSequences while (wrapPoint > (minSequence = Util.getMinimumSequence(gatingSequences, nextValue))) { waitStrategy.signalAllWhenBlocking(); LockSupport.parkNanos(1L); // TODO: Use waitStrategy to spin? } //同理，缓存最小的gatingSequences this.cachedValue = minSequence; } this.nextValue = nextSequence; return nextSequence; } @Override public long tryNext() throws InsufficientCapacityException { return tryNext(1); } @Override public long tryNext(int n) throws InsufficientCapacityException { if (n < 1) { throw new IllegalArgumentException("n must be > 0"); } if (!hasAvailableCapacity(n)) { throw InsufficientCapacityException.INSTANCE; } long nextSequence = this.nextValue += n; return nextSequence; } @Override public void publish(long sequence) { //cursor代表可以消费的sequence cursor.set(sequence); waitStrategy.signalAllWhenBlocking(); } @Override public void publish(long lo, long hi) { publish(hi); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/ringbuffer/utils/Util.java ================================================ package io.mycat.memory.unsafe.ringbuffer.utils; import io.mycat.memory.unsafe.ringbuffer.common.sequence.Sequence; /** * Set of common functions used by the Disruptor */ public class Util { /** * 计算下一个不小于x的2的n次方 * 原理：int最长为32位，计算x-1的前面有多少个0，之后用32减去这个值得到n，那么2的n次方就是下一个不小于x的2的n次方 * * @param x Value to round up * @return The next power of 2 from x inclusive */ public static int ceilingNextPowerOfTwo(final int x) { return 1 << (32 - Integer.numberOfLeadingZeros(x - 1)); } /** * 获取Sequence数组中value最小的值 * * @param sequences to compare. * @return the minimum sequence found or Long.MAX_VALUE if the array is empty. */ public static long getMinimumSequence(final Sequence[] sequences) { return getMinimumSequence(sequences, Long.MAX_VALUE); } /** * 获取Sequence数组中value最小的值 * * @param sequences to compare. * @param minimum 如果数组为空，将返回这个值 * @return the smaller of minimum sequence value found in {@code sequences} and {@code minimum}; * {@code minimum} if {@code sequences} is empty */ public static long getMinimumSequence(final Sequence[] sequences, long minimum) { for (int i = 0, n = sequences.length; i < n; i++) { long value = sequences[i].get(); minimum = Math.min(minimum, value); } return minimum; } public static int log2(int i) { int r = 0; while ((i >>= 1) != 0) { ++r; } return r; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/row/BufferHolder.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.row; import io.mycat.memory.unsafe.Platform; /** * A helper class to manage the data buffer for an unsafe row. The data buffer can grow and * automatically re-point the unsafe row to it. * * This class can be used to build a one-pass unsafe row writing program, i.e. data will be written * to the data buffer directly and no extra copy is needed. There should be only one instance of * this class per writing program, so that the memory segment/data buffer can be reused. Note that * for each incoming record, we should call `reset` of BufferHolder instance before write the record * and reuse the data buffer. * * Generally we should call `UnsafeRow.setTotalSize` and pass in `BufferHolder.totalSize` to update * the size of the result row, after writing a record to the buffer. However, we can skip this step * if the fields of row are all fixed-length, as the size of result row is also fixed. */ public class BufferHolder { public byte[] buffer; public int cursor = Platform.BYTE_ARRAY_OFFSET; private final UnsafeRow row; private final int fixedSize; public BufferHolder(UnsafeRow row) { this(row, 64); } public BufferHolder(UnsafeRow row, int initialSize) { this.fixedSize = UnsafeRow.calculateBitSetWidthInBytes(row.numFields()) + 8 * row.numFields(); this.buffer = new byte[fixedSize + initialSize]; this.row = row; this.row.pointTo(buffer, buffer.length); } /** * Grows the buffer by at least neededSize and points the row to the buffer. */ public void grow(int neededSize) { final int length = totalSize() + neededSize; if (buffer.length < length) { // This will not happen frequently, because the buffer is re-used. final byte[] tmp = new byte[length * 2]; Platform.copyMemory( buffer, Platform.BYTE_ARRAY_OFFSET, tmp, Platform.BYTE_ARRAY_OFFSET, totalSize()); buffer = tmp; row.pointTo(buffer, buffer.length); } } public UnsafeRow getRow() { return row; } public void reset() { cursor = Platform.BYTE_ARRAY_OFFSET + fixedSize; } public int totalSize() { return cursor - Platform.BYTE_ARRAY_OFFSET; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/row/StructType.java ================================================ package io.mycat.memory.unsafe.row; import io.mycat.sqlengine.mpp.ColMeta; import io.mycat.sqlengine.mpp.OrderCol; import javax.annotation.Nonnull; import java.util.Map; /** * Created by zagnix on 2016/6/6. */ public class StructType { private final Map columToIndx; private final int fieldCount; private OrderCol[] orderCols = null; public StructType(@Nonnull Map columToIndx,int fieldCount){ assert fieldCount >=0; this.columToIndx = columToIndx; this.fieldCount = fieldCount; } public int length() { return fieldCount; } public Map getColumToIndx() { return columToIndx; } public OrderCol[] getOrderCols() { return orderCols; } public void setOrderCols(OrderCol[] orderCols) { this.orderCols = orderCols; } public long apply(int i) { return 0; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/row/UnsafeRow.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.row; import io.mycat.backend.mysql.BufferUtil; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.array.ByteArrayMethods; import io.mycat.memory.unsafe.bitset.BitSetMethods; import io.mycat.memory.unsafe.hash.Murmur3_x86_32; import io.mycat.memory.unsafe.types.UTF8String; import io.mycat.net.FrontendConnection; import io.mycat.net.mysql.MySQLPacket; import java.io.IOException; import java.io.OutputStream; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; /** * Modify by zagnix * An Unsafe implementation of Row which is backed by raw memory instead of Java objects. * * Each tuple has three parts: [null bit set] [values] [variable length portion] * * The bit set is used for null tracking and is aligned to 8-byte word boundaries. It stores * one bit per field. * * In the `values` region, we store one 8-byte word per field. For fields that hold fixed-length * primitive types, such as long, double, or int, we store the value directly in the word. For * fields with non-primitive or variable-length values, we store a relative offset (w.r.t. the * base address of the row) that points to the beginning of the variable-length field, and length * (they are combined into a long). * * Instances of `UnsafeRow` act as pointers to row data stored in this format. */ public final class UnsafeRow extends MySQLPacket { ////////////////////////////////////////////////////////////////////////////// // Static methods ////////////////////////////////////////////////////////////////////////////// public static int calculateBitSetWidthInBytes(int numFields) { return ((numFields + 63)/ 64) * 8; } public static int calculateFixedPortionByteSize(int numFields) { return 8 * numFields + calculateBitSetWidthInBytes(numFields); } ////////////////////////////////////////////////////////////////////////////// // Private fields and methods ////////////////////////////////////////////////////////////////////////////// private Object baseObject; private long baseOffset; /** The number of fields in this row, used for calculating the bitset width (and in assertions) */ private int numFields; /** The size of this row's backing data, in bytes) */ private int sizeInBytes; /** The width of the null tracking bit set, in bytes */ private int bitSetWidthInBytes; private long getFieldOffset(int ordinal) { return baseOffset + bitSetWidthInBytes + ordinal * 8L; } private void assertIndexIsValid(int index) { assert index >= 0 : "index (" + index + ") should >= 0"; assert index < numFields : "index (" + index + ") should < " + numFields; } ////////////////////////////////////////////////////////////////////////////// // Public methods ////////////////////////////////////////////////////////////////////////////// /** * Construct a new UnsafeRow. The resulting row won't be usable until `pointTo()` has been called, * since the value returned by this constructor is equivalent to a null pointer. * * @param numFields the number of fields in this row */ public UnsafeRow(int numFields) { this.numFields = numFields; this.bitSetWidthInBytes = calculateBitSetWidthInBytes(numFields); } // for serializer public UnsafeRow() {} public Object getBaseObject() { return baseObject; } public long getBaseOffset() { return baseOffset; } public int getSizeInBytes() { return sizeInBytes; } public int numFields() { return numFields; } /** * Update this UnsafeRow to point to different backing data. * * @param baseObject the base object * @param baseOffset the offset within the base object * @param sizeInBytes the size of this row's backing data, in bytes */ public void pointTo(Object baseObject, long baseOffset, int sizeInBytes) { assert numFields >= 0 : "numFields (" + numFields + ") should >= 0"; this.baseObject = baseObject; this.baseOffset = baseOffset; this.sizeInBytes = sizeInBytes; } /** * Update this UnsafeRow to point to the underlying byte array. * * @param buf byte array to point to * @param sizeInBytes the number of bytes valid in the byte array */ public void pointTo(byte[] buf, int sizeInBytes) { pointTo(buf, Platform.BYTE_ARRAY_OFFSET, sizeInBytes); } public void setTotalSize(int sizeInBytes) { this.sizeInBytes = sizeInBytes; } public void setNotNullAt(int i) { assertIndexIsValid(i); BitSetMethods.unset(baseObject, baseOffset, i); } public void setNullAt(int i) { assertIndexIsValid(i); BitSetMethods.set(baseObject, baseOffset, i); // To preserve row equality, zero out the value when setting the column to null. // Since this row does does not currently support updates to variable-length values, we don't // have to worry about zeroing out that data. Platform.putLong(baseObject, getFieldOffset(i), 0); } public void update(int ordinal, Object value) { throw new UnsupportedOperationException(); } public void setInt(int ordinal, int value) { assertIndexIsValid(ordinal); setNotNullAt(ordinal); Platform.putInt(baseObject, getFieldOffset(ordinal), value); } public void setLong(int ordinal, long value) { assertIndexIsValid(ordinal); setNotNullAt(ordinal); Platform.putLong(baseObject, getFieldOffset(ordinal), value); } public void setDouble(int ordinal, double value) { assertIndexIsValid(ordinal); setNotNullAt(ordinal); if (Double.isNaN(value)) { value = Double.NaN; } Platform.putDouble(baseObject, getFieldOffset(ordinal), value); } public void setBoolean(int ordinal, boolean value) { assertIndexIsValid(ordinal); setNotNullAt(ordinal); Platform.putBoolean(baseObject, getFieldOffset(ordinal), value); } public void setShort(int ordinal, short value) { assertIndexIsValid(ordinal); setNotNullAt(ordinal); Platform.putShort(baseObject, getFieldOffset(ordinal), value); } public void setByte(int ordinal, byte value) { assertIndexIsValid(ordinal); setNotNullAt(ordinal); Platform.putByte(baseObject, getFieldOffset(ordinal), value); } public void setFloat(int ordinal, float value) { assertIndexIsValid(ordinal); setNotNullAt(ordinal); if (Float.isNaN(value)) { value = Float.NaN; } Platform.putFloat(baseObject, getFieldOffset(ordinal), value); } public boolean isNullAt(int ordinal) { assertIndexIsValid(ordinal); return BitSetMethods.isSet(baseObject, baseOffset, ordinal); } public boolean getBoolean(int ordinal) { assertIndexIsValid(ordinal); return Platform.getBoolean(baseObject, getFieldOffset(ordinal)); } public byte getByte(int ordinal) { assertIndexIsValid(ordinal); return Platform.getByte(baseObject, getFieldOffset(ordinal)); } public short getShort(int ordinal) { assertIndexIsValid(ordinal); return Platform.getShort(baseObject, getFieldOffset(ordinal)); } public int getInt(int ordinal) { assertIndexIsValid(ordinal); return Platform.getInt(baseObject, getFieldOffset(ordinal)); } public long getLong(int ordinal) { assertIndexIsValid(ordinal); return Platform.getLong(baseObject, getFieldOffset(ordinal)); } public float getFloat(int ordinal) { assertIndexIsValid(ordinal); return Platform.getFloat(baseObject, getFieldOffset(ordinal)); } public double getDouble(int ordinal) { assertIndexIsValid(ordinal); return Platform.getDouble(baseObject, getFieldOffset(ordinal)); } public UTF8String getUTF8String(int ordinal) { if (isNullAt(ordinal)) return null; final long offsetAndSize = getLong(ordinal); final int offset = (int) (offsetAndSize >> 32); final int size = (int) offsetAndSize; return UTF8String.fromAddress(baseObject, baseOffset + offset, size); } public byte[] getBinary(int ordinal) { if (isNullAt(ordinal)) { return null; } else { final long offsetAndSize = getLong(ordinal); final int offset = (int) (offsetAndSize >> 32); final int size = (int) offsetAndSize; final byte[] bytes = new byte[size]; Platform.copyMemory( baseObject, baseOffset + offset, bytes, Platform.BYTE_ARRAY_OFFSET, size ); return bytes; } } /** * Copies this row, returning a self-contained UnsafeRow that stores its data in an internal * byte array rather than referencing data stored in a data page. */ public UnsafeRow copy() { UnsafeRow rowCopy = new UnsafeRow(numFields); final byte[] rowDataCopy = new byte[sizeInBytes]; Platform.copyMemory( baseObject, baseOffset, rowDataCopy, Platform.BYTE_ARRAY_OFFSET, sizeInBytes ); rowCopy.pointTo(rowDataCopy, Platform.BYTE_ARRAY_OFFSET, sizeInBytes); return rowCopy; } /** * Creates an empty UnsafeRow from a byte array with specified numBytes and numFields. * The returned row is invalid until we call copyFrom on it. */ public static UnsafeRow createFromByteArray(int numBytes, int numFields) { final UnsafeRow row = new UnsafeRow(numFields); row.pointTo(new byte[numBytes], numBytes); return row; } /** * Copies the input UnsafeRow to this UnsafeRow, and resize the underlying byte[] when the * input row is larger than this row. */ public void copyFrom(UnsafeRow row) { // copyFrom is only available for UnsafeRow created from byte array. assert (baseObject instanceof byte[]) && baseOffset == Platform.BYTE_ARRAY_OFFSET; if (row.sizeInBytes > this.sizeInBytes) { // resize the underlying byte[] if it's not large enough. this.baseObject = new byte[row.sizeInBytes]; } Platform.copyMemory( row.baseObject, row.baseOffset, this.baseObject, this.baseOffset, row.sizeInBytes); // update the sizeInBytes. this.sizeInBytes = row.sizeInBytes; } /** * Write this UnsafeRow's underlying bytes to the given OutputStream. * * @param out the stream to write to. * @param writeBuffer a byte array for buffering chunks of off-heap data while writing to the * output stream. If this row is backed by an on-heap byte array, then this * buffer will not be used and may be null. */ public void writeToStream(OutputStream out, byte[] writeBuffer) throws IOException { if (baseObject instanceof byte[]) { int offsetInByteArray = (int) (Platform.BYTE_ARRAY_OFFSET - baseOffset); out.write((byte[]) baseObject, offsetInByteArray, sizeInBytes); } else { int dataRemaining = sizeInBytes; long rowReadPosition = baseOffset; while (dataRemaining > 0) { int toTransfer = Math.min(writeBuffer.length, dataRemaining); Platform.copyMemory( baseObject, rowReadPosition, writeBuffer, Platform.BYTE_ARRAY_OFFSET, toTransfer); out.write(writeBuffer, 0, toTransfer); rowReadPosition += toTransfer; dataRemaining -= toTransfer; } } } @Override public int hashCode() { return Murmur3_x86_32.hashUnsafeWords(baseObject, baseOffset, sizeInBytes, 42); } @Override public boolean equals(Object other) { if (other instanceof UnsafeRow) { UnsafeRow o = (UnsafeRow) other; return (sizeInBytes == o.sizeInBytes) && ByteArrayMethods.arrayEquals(baseObject, baseOffset, o.baseObject, o.baseOffset, sizeInBytes); } return false; } /** * Returns the underlying bytes for this UnsafeRow. */ public byte[] getBytes() { if (baseObject instanceof byte[] && baseOffset == Platform.BYTE_ARRAY_OFFSET && (((byte[]) baseObject).length == sizeInBytes)) { return (byte[]) baseObject; } else { byte[] bytes = new byte[sizeInBytes]; Platform.copyMemory(baseObject, baseOffset, bytes, Platform.BYTE_ARRAY_OFFSET, sizeInBytes); return bytes; } } public static final byte NULL_MARK = (byte) 251; public static final byte EMPTY_MARK = (byte) 0; @Override public ByteBuffer write(ByteBuffer bb, FrontendConnection c, boolean writeSocketIfFull) { bb = c.checkWriteBuffer(bb,c.getPacketHeaderSize(),writeSocketIfFull); BufferUtil.writeUB3(bb, calcPacketSize()); bb.put(packetId); for (int i = 0; i < numFields; i++) { if (!isNullAt(i)) { byte[] fv = this.getBinary(i); if (fv.length == 0) { bb = c.checkWriteBuffer(bb, 1, writeSocketIfFull); bb.put(UnsafeRow.EMPTY_MARK); } else { bb = c.checkWriteBuffer(bb, BufferUtil.getLength(fv), writeSocketIfFull); BufferUtil.writeLength(bb, fv.length); /** * 把数据写到Writer Buffer中 */ bb = c.writeToBuffer(fv, bb); } } else { //Col null value bb = c.checkWriteBuffer(bb,1,writeSocketIfFull); bb.put(UnsafeRow.NULL_MARK); } } return bb; } @Override public int calcPacketSize() { int size = 0; for (int i = 0; i < numFields; i++) { byte[] v = this.getBinary(i); size += (v == null || v.length == 0) ? 1 : BufferUtil.getLength(v); } return size; } public BigDecimal getDecimal(int ordinal, int scale) { if (isNullAt(ordinal)) { return null; } byte[] bytes = getBinary(ordinal); BigInteger bigInteger = new BigInteger(bytes); BigDecimal javaDecimal = new BigDecimal(bigInteger, scale); return javaDecimal; } /** * update exist decimal column value to new decimal value * * NOTE: decimal max precision is limit to 38 * @param ordinal * @param value * @param precision */ public void updateDecimal(int ordinal, BigDecimal value) { assertIndexIsValid(ordinal); // fixed length long cursor = getLong(ordinal) >>> 32; assert cursor > 0 : "invalid cursor " + cursor; // zero-out the bytes Platform.putLong(baseObject, baseOffset + cursor, 0L); Platform.putLong(baseObject, baseOffset + cursor + 8, 0L); if (value == null) { setNullAt(ordinal); // keep the offset for future update Platform.putLong(baseObject, getFieldOffset(ordinal), cursor << 32); } else { final BigInteger integer = value.unscaledValue(); byte[] bytes = integer.toByteArray(); assert (bytes.length <= 16); // Write the bytes to the variable length portion. Platform.copyMemory(bytes, Platform.BYTE_ARRAY_OFFSET, baseObject, baseOffset + cursor, bytes.length); setLong(ordinal, (cursor << 32) | ((long) bytes.length)); } } /** public Decimal getDecimal(int ordinal, int precision, int scale) { if (isNullAt(ordinal)) { return null; } if (precision <= Decimal.MAX_LONG_DIGITS()) { return Decimal.createUnsafe(getLong(ordinal), precision, scale); } else { byte[] bytes = getBinary(ordinal); BigInteger bigInteger = new BigInteger(bytes); BigDecimal javaDecimal = new BigDecimal(bigInteger, scale); return Decimal.apply(javaDecimal, precision, scale); } } public void setDecimal(int ordinal, Decimal value, int precision) { assertIndexIsValid(ordinal); if (precision <= Decimal.MAX_LONG_DIGITS()) { // compact format if (value == null) { setNullAt(ordinal); } else { setLong(ordinal, value.toUnscaledLong()); } } else { // fixed length long cursor = getLong(ordinal) >>> 32; assert cursor > 0 : "invalid cursor " + cursor; // zero-out the bytes Platform.putLong(baseObject, baseOffset + cursor, 0L); Platform.putLong(baseObject, baseOffset + cursor + 8, 0L); if (value == null) { setNullAt(ordinal); // keep the offset for future update Platform.putLong(baseObject, getFieldOffset(ordinal), cursor << 32); } else { final BigInteger integer = value.toJavaBigDecimal().unscaledValue(); byte[] bytes = integer.toByteArray(); assert(bytes.length <= 16); // Write the bytes to the variable length portion. Platform.copyMemory( bytes, Platform.BYTE_ARRAY_OFFSET, baseObject, baseOffset + cursor, bytes.length); setLong(ordinal, (cursor << 32) | ((long) bytes.length)); } } } */ @Override protected String getPacketInfo() { return "MySQL RowData Packet"; } // This is for debugging @Override public String toString() { StringBuilder build = new StringBuilder("["); for (int i = 0; i < sizeInBytes; i += 8) { if (i != 0) build.append(','); build.append(Long.toHexString(Platform.getLong(baseObject, baseOffset + i))); } build.append(']'); return build.toString(); } public boolean anyNull() { return BitSetMethods.anySet(baseObject, baseOffset, bitSetWidthInBytes / 8); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/row/UnsafeRowWriter.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.row; import java.math.BigDecimal; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.array.ByteArrayMethods; import io.mycat.memory.unsafe.bitset.BitSetMethods; /** * A helper class to write data into global row buffer using `UnsafeRow` format. * * It will remember the offset of row buffer which it starts to write, and move the cursor of row * buffer while writing. If new data(can be the input record if this is the outermost writer, or * nested struct if this is an inner writer) comes, the starting cursor of row buffer may be * changed, so we need to call `UnsafeRowWriter.reset` before writing, to update the * `startingOffset` and clear out null bits. * * Note that if this is the outermost writer, which means we will always write from the very * beginning of the global row buffer, we don't need to update `startingOffset` and can just call * `zeroOutNullBytes` before writing new data. */ public class UnsafeRowWriter { private final BufferHolder holder; // The offset of the global buffer where we start to write this row. private int startingOffset; private final int nullBitsSize; private final int fixedSize; public UnsafeRowWriter(BufferHolder holder,int numFields) { this.holder = holder; this.nullBitsSize = UnsafeRow.calculateBitSetWidthInBytes(numFields); this.fixedSize = nullBitsSize + 8 * numFields; this.startingOffset = holder.cursor; } /** * Resets the `startingOffset` according to the current cursor of row buffer, and clear out null * bits. This should be called before we write a new nested struct to the row buffer. */ public void reset() { this.startingOffset = holder.cursor; // grow the global buffer to make sure it has enough space to write fixed-length data. holder.grow(fixedSize); holder.cursor += fixedSize; zeroOutNullBytes(); } /** * Clears out null bits. This should be called before we write a new row to row buffer. */ public void zeroOutNullBytes() { for (int i = 0; i < nullBitsSize; i += 8) { Platform.putLong(holder.buffer, startingOffset + i, 0L); } } private void zeroOutPaddingBytes(int numBytes) { if ((numBytes & 0x07) > 0) { Platform.putLong(holder.buffer, holder.cursor + ((numBytes >> 3) << 3), 0L); } } public BufferHolder holder() { return holder; } public boolean isNullAt(int ordinal) { return BitSetMethods.isSet(holder.buffer, startingOffset, ordinal); } public void setNullAt(int ordinal) { BitSetMethods.set(holder.buffer, startingOffset, ordinal); Platform.putLong(holder.buffer, getFieldOffset(ordinal), 0L); } public long getFieldOffset(int ordinal) { return startingOffset + nullBitsSize + 8 * ordinal; } public void setOffsetAndSize(int ordinal, long size) { setOffsetAndSize(ordinal, holder.cursor, size); } public void setOffsetAndSize(int ordinal, long currentCursor, long size) { final long relativeOffset = currentCursor - startingOffset; final long fieldOffset = getFieldOffset(ordinal); final long offsetAndSize = (relativeOffset << 32) | size; Platform.putLong(holder.buffer, fieldOffset, offsetAndSize); } // Do word alignment for this row and grow the row buffer if needed. // todo: remove this after we make unsafe array data word align. public void alignToWords(int numBytes) { final int remainder = numBytes & 0x07; if (remainder > 0) { final int paddingBytes = 8 - remainder; holder.grow(paddingBytes); for (int i = 0; i < paddingBytes; i++) { Platform.putByte(holder.buffer, holder.cursor, (byte) 0); holder.cursor++; } } } public void write(int ordinal, boolean value) { final long offset = getFieldOffset(ordinal); Platform.putLong(holder.buffer, offset, 0L); Platform.putBoolean(holder.buffer, offset, value); } public void write(int ordinal, byte value) { final long offset = getFieldOffset(ordinal); Platform.putLong(holder.buffer, offset, 0L); Platform.putByte(holder.buffer, offset, value); } public void write(int ordinal, short value) { final long offset = getFieldOffset(ordinal); Platform.putLong(holder.buffer, offset, 0L); Platform.putShort(holder.buffer, offset, value); } public void write(int ordinal, int value) { final long offset = getFieldOffset(ordinal); Platform.putLong(holder.buffer, offset, 0L); Platform.putInt(holder.buffer, offset, value); } public void write(int ordinal, long value) { Platform.putLong(holder.buffer, getFieldOffset(ordinal), value); } public void write(int ordinal, float value) { if (Float.isNaN(value)) { value = Float.NaN; } final long offset = getFieldOffset(ordinal); Platform.putLong(holder.buffer, offset, 0L); Platform.putFloat(holder.buffer, offset, value); } public void write(int ordinal, double value) { if (Double.isNaN(value)) { value = Double.NaN; } Platform.putDouble(holder.buffer, getFieldOffset(ordinal), value); } public void write(int ordinal, byte[] input) { if(input == null){ return; } write(ordinal, input, 0, input.length); } public void grow( int numBytes) { final int roundedSize = ByteArrayMethods.roundNumberOfBytesToNearestWord(numBytes); // grow the global buffer before writing data. holder.grow(roundedSize); } public void write(int ordinal, byte[] input, int offset, int numBytes) { final int roundedSize = ByteArrayMethods.roundNumberOfBytesToNearestWord(numBytes); // grow the global buffer before writing data. holder.grow(roundedSize); zeroOutPaddingBytes(numBytes); // Write the bytes to the variable length portion. Platform.copyMemory(input, Platform.BYTE_ARRAY_OFFSET + offset, holder.buffer, holder.cursor, numBytes); setOffsetAndSize(ordinal, numBytes); // move the cursor forward. holder.cursor += roundedSize; } /** * different from Spark, we use java BigDecimal here, * and we limit the max precision to be 38 because the bytes length limit to be 16 * * @param ordinal * @param input */ public void write(int ordinal, BigDecimal input) { // grow the global buffer before writing data. holder.grow(16); // zero-out the bytes Platform.putLong(holder.buffer, holder.cursor, 0L); Platform.putLong(holder.buffer, holder.cursor + 8, 0L); // Make sure Decimal object has the same scale as DecimalType. // Note that we may pass in null Decimal object to set null for it. if (input == null) { BitSetMethods.set(holder.buffer, startingOffset, ordinal); // keep the offset for future update setOffsetAndSize(ordinal, 0L); } else { final byte[] bytes = input.unscaledValue().toByteArray(); assert bytes.length <= 16; // Write the bytes to the variable length portion. Platform.copyMemory(bytes, Platform.BYTE_ARRAY_OFFSET, holder.buffer, holder.cursor, bytes.length); setOffsetAndSize(ordinal, bytes.length); } // move the cursor forward. holder.cursor += 16; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/storage/ConnectionId.java ================================================ package io.mycat.memory.unsafe.storage; /** * * Created by zagnix on 2016/6/6. * */ public abstract class ConnectionId { protected String name; public abstract String getBlockName(); @Override public boolean equals(Object arg0) { return super.equals(arg0); } @Override public int hashCode() { return super.hashCode(); } @Override public String toString() { return super.toString(); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/storage/DataNodeDiskManager.java ================================================ package io.mycat.memory.unsafe.storage; import io.mycat.memory.unsafe.utils.MycatPropertyConf; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; /** * Created by zagnix on 2016/6/3. */ public class DataNodeDiskManager { private MycatPropertyConf conf; private boolean deleteFilesOnStop; private SerializerManager serializerManager; public DataNodeDiskManager(MycatPropertyConf conf, boolean deleteFilesOnStop, SerializerManager serializerManager){ this.conf = conf; this.deleteFilesOnStop = deleteFilesOnStop; this.serializerManager = serializerManager; } public DataNodeFileManager diskBlockManager() throws IOException { return new DataNodeFileManager(conf, deleteFilesOnStop); } /** * A short circuited method to get a block writer that can write data directly to disk. * The Block will be appended to the File specified by filename. Callers should handle error * cases. */ public DiskRowWriter getDiskWriter( ConnectionId blockId, File file, SerializerInstance serializerInstance, int bufferSize) throws IOException { boolean syncWrites = conf.getBoolean("mycat.merge.sync", false); return new DiskRowWriter(file, serializerInstance, bufferSize,new FileOutputStream(file), syncWrites,blockId); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/storage/DataNodeFileManager.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.storage; import io.mycat.memory.unsafe.utils.JavaUtils; import io.mycat.memory.unsafe.utils.MycatPropertyConf; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; /** * Creates and maintains the logical mapping between logical blocks and physical on-disk * locations. One block is mapped to one file with a name given by its BlockId. * * Block files are hashed among the directories listed in mycat.local.dir */ public class DataNodeFileManager { private static final Logger LOG = LoggerFactory.getLogger(DataNodeFileManager.class); private MycatPropertyConf conf; private boolean deleteFilesOnStop; /** * TODO 操作完成之后，需要删除临时文件 */ // The content of subDirs is immutable but the content of subDirs(i) is mutable. And the content // of subDirs(i) is protected by the lock of subDirs(i) // private val shutdownHook ; /* Create one local directory for each path mentioned in spark.local.dir; then, inside this * directory, create multiple subdirectories that we will hash files into, in order to avoid * having really large inodes at the top level. */ private List localDirs ; private int subDirsPerLocalDir; private ConcurrentHashMap> subDirs; public DataNodeFileManager(MycatPropertyConf conf , boolean deleteFilesOnStop) throws IOException { this.conf = conf; this.deleteFilesOnStop = deleteFilesOnStop; subDirsPerLocalDir = conf.getInt("mycat.diskStore.subDirectories", 64); localDirs = createLocalDirs(conf); if (localDirs.isEmpty()) { System.exit(-1); } subDirs = new ConcurrentHashMap>(localDirs.size()); for (int i = 0; i < localDirs.size() ; i++) { ArrayList list = new ArrayList(subDirsPerLocalDir); for (int j = 0; j < subDirsPerLocalDir; j++) { list.add(i,null); } subDirs.put(i,list); } } /** Produces a unique block id and File suitable for storing local intermediate results. */ public TempDataNodeId createTempLocalBlock() throws IOException { TempDataNodeId blockId = new TempDataNodeId(UUID.randomUUID().toString()); while (getFile(blockId).exists()) { blockId = new TempDataNodeId(UUID.randomUUID().toString()); }; return blockId; } /** Looks up a file by hashing it into one of our local subdirectories. */ // This method should be kept in sync with // org.apache.spark.network.shuffle.ExternalShuffleBlockResolver#getFile(). public File getFile(String filename) throws IOException { // Figure out which local directory it hashes to, and which subdirectory in that int hash = JavaUtils.nonNegativeHash(filename); int dirId = hash % localDirs.size(); int subDirId = (hash / localDirs.size()) % subDirsPerLocalDir; synchronized (this) { File file = subDirs.get(dirId).get(subDirId); if (file != null) { } else { file = new File(localDirs.get(dirId), "%02x".format(String.valueOf(subDirId))); if (!file.exists() && !file.mkdir()) { throw new IOException("Failed to create local dir in $newDir."); } subDirs.get(dirId).add(subDirId,file); } } /** *类似二维数组 */ return new File(subDirs.get(dirId).get(subDirId),filename); } public File getFile(ConnectionId connid) throws IOException { return getFile(connid.name); } /**TODO config root * Create local directories for storing block data. These directories are * located inside configured local directories and won't * be deleted on JVM exit when using the external shuffle service. */ private List createLocalDirs(MycatPropertyConf conf) { String rootDirs = conf.getString("mycat.local.dirs","datanode"); String rdir[] = rootDirs.split(","); List dirs = new ArrayList(); for (int i = 0; i 0){ localDir = localDirs.get(i); //System.out.println(localDir); if (localDir.isDirectory() && localDir.exists()) { try { JavaUtils.deleteRecursively(localDir); } catch(Exception e) { LOG.error(e.getMessage()); } } i++; } } } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/storage/DeserializationStream.java ================================================ package io.mycat.memory.unsafe.storage; /** * Created by zagnix on 2016/6/3. */ public abstract class DeserializationStream { /** The most general-purpose method to read an object. */ public abstract T readObject(); /** Reads the object representing the key of a key-value pair. */ public T readKey(){return readObject();} /** Reads the object representing the value of a key-value pair. */ public T readValue(){ return readObject();} public abstract void close(); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/storage/DiskRowWriter.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.storage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; import java.nio.channels.FileChannel; /** * A class for writing JVM objects directly to a file on disk. This class allows data to be appended * to an existing block and can guarantee atomicity in the case of faults as it allows the caller to * revert partial writes. * * This class does not support concurrent writes. Also, once the writer has been opened it cannot be * reopened again. */ public class DiskRowWriter extends OutputStream { /** The file channel, used for repositioning / truncating the file. */ private static final Logger LOG = LoggerFactory.getLogger(DiskRowWriter.class); private FileChannel channel = null; private OutputStream bs = null; private FileOutputStream fos = null; private TimeTrackingOutputStream ts = null; private SerializationStream objOut = null; private boolean initialized = false; private boolean hasBeenClosed = false; private boolean commitAndCloseHasBeenCalled = false; /** * Cursors used to represent positions in the file. * * xxxxxxxx|--------|--- | * ^ ^ ^ * | | finalPosition * | reportedPosition * initialPosition * * initialPosition: Offset in the file where we start writing. Immutable. * reportedPosition: Position at the time of the last update to the write metrics. * finalPosition: Offset where we stopped writing. Set on closeAndCommit() then never changed. * -----: Current writes to the underlying file. * xxxxx: Existing contents of the file. */ private long initialPosition = 0; private long finalPosition = -1; private long reportedPosition = 0; /** * Keep track of number of records written and also use this to periodically * output bytes written since the latter is expensive to do for each record. */ private long numRecordsWritten = 0; private File file; private SerializerInstance serializerInstance; private int bufferSize; private OutputStream compressStream; private boolean syncWrites; // These write metrics concurrently shared with other active DiskBlockObjectWriters who // are themselves performing writes. All updates must be relative. /**ShuffleWriteMetrics writeMetrics,*/ private ConnectionId blockId; public DiskRowWriter( File file, SerializerInstance serializerInstance, int bufferSize, OutputStream compressStream , boolean syncWrites, ConnectionId blockId) throws IOException { this.file = file; this.serializerInstance = serializerInstance; this.bufferSize = bufferSize; this.compressStream = compressStream; this.syncWrites = syncWrites; this.blockId = blockId; initialPosition = file.length(); reportedPosition = initialPosition; } public DiskRowWriter open() throws FileNotFoundException { if (hasBeenClosed) { throw new IllegalStateException("Writer already closed. Cannot be reopened."); } fos = new FileOutputStream(file,true); ts = new TimeTrackingOutputStream(/**writeMetrics,*/ fos); channel = fos.getChannel(); bs = new BufferedOutputStream(ts,bufferSize); objOut = serializerInstance.serializeStream(bs); initialized = true; return this; } @Override public void close() { if (initialized) { try { if (syncWrites) { //Force outstanding writes to disk and track how long it takes objOut.flush(); long start = System.nanoTime(); fos.getFD().sync(); // writeMetrics.incWriteTime(System.nanoTime() - start); } } catch (IOException e) { LOG.error(e.getMessage()); }finally { objOut.close(); } channel = null; bs = null; fos = null; ts = null; objOut = null; initialized = false; hasBeenClosed = true; } } public boolean isOpen(){ return objOut != null; } /** * Flush the partial writes and commit them as a single atomic block. */ public void commitAndClose() throws IOException { if (initialized) { // NOTE: Because Kryo doesn’t flush the underlying stream we explicitly flush both the // serializer stream and the lower level stream. objOut.flush(); bs.flush(); close(); finalPosition = file.length(); // In certain compression codecs, more bytes are written after close() is called //writeMetrics.incBytesWritten(finalPosition - reportedPosition) } else { finalPosition = file.length(); } commitAndCloseHasBeenCalled = true; } /** * Reverts writes that haven’t been flushed yet. Callers should invoke this function * when there are runtime exceptions. This method will not throw, though it may be * unsuccessful in truncating written data. * * @return the file that this DiskRowWriter wrote to. */ public File revertPartialWritesAndClose() throws IOException { // Discard current writes. We do this by flushing the outstanding writes and then // truncating the file to its initial position. try { if (initialized) { // writeMetrics.decBytesWritten(reportedPosition - initialPosition) // writeMetrics.decRecordsWritten(numRecordsWritten) objOut.flush(); bs.flush(); close(); } FileOutputStream truncateStream = new FileOutputStream(file, true); try { truncateStream.getChannel().truncate(initialPosition); return file; } finally { truncateStream.close(); } } catch(Exception e) { LOG.error(e.getMessage()); return file; } } /** * Writes a key-value pair. */ private void write(Object key, Object value) throws IOException { if (!initialized) { open(); } objOut.writeKey(key); objOut.writeValue(value); recordWritten(); } @Override public void write(int b){ throw new UnsupportedOperationException(); } @Override public void write(byte [] kvBytes ,int offs, int len) throws IOException { if (!initialized) { open(); } bs.write(kvBytes,offs, len); } /** * Notify the writer that a record worth of bytes has been written with OutputStream#write. */ public void recordWritten() throws IOException { numRecordsWritten += 1; //writeMetrics.incRecordsWritten(1) // TODO: call updateBytesWritten() less frequently. if (numRecordsWritten % 32 == 0) { updateBytesWritten(); } } /** * Report the number of bytes written in this writer’s shuffle write metrics. * Note that this is only valid before the underlying streams are closed. */ private void updateBytesWritten() throws IOException { long pos = channel.position(); //writeMetrics.incBytesWritten(pos - reportedPosition) reportedPosition = pos; } @Override public void flush() throws IOException { objOut.flush(); bs.flush(); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/storage/DummySerializerInstance.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.storage; import io.mycat.memory.unsafe.Platform; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; /** * Unfortunately, we need a serializer instance in order to construct a DiskRowWriter. * Our shuffle write path doesn't actually use this serializer (since we end up calling the * `write() OutputStream methods), but DiskRowWriter still calls some methods on it. To work * around this, we pass a dummy no-op serializer. */ public final class DummySerializerInstance extends SerializerInstance { public static final DummySerializerInstance INSTANCE = new DummySerializerInstance(); private DummySerializerInstance() { } @Override public SerializationStream serializeStream(final OutputStream s) { return new SerializationStream() { @Override public SerializationStream writeObject(Object o) { return null; } @Override public void flush() { // Need to implement this because DiskObjectWriter uses it to flush the compression stream try { s.flush(); } catch (IOException e) { Platform.throwException(e); } } // public SerializationStream writeObject(T t, T ev1) { // throw new UnsupportedOperationException(); // } @Override public void close() { // Need to implement this because DiskObjectWriter uses it to close the compression stream try { s.close(); } catch (IOException e) { Platform.throwException(e); } } }; } public ByteBuffer serialize(T t, T ev1) { throw new UnsupportedOperationException(); } public DeserializationStream deserializeStream(InputStream s) { throw new UnsupportedOperationException(); } public T deserialize(ByteBuffer bytes, ClassLoader loader, T ev1) { throw new UnsupportedOperationException(); } public T deserialize(ByteBuffer bytes, T ev1) { throw new UnsupportedOperationException(); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/storage/SerializationStream.java ================================================ package io.mycat.memory.unsafe.storage; import java.util.Iterator; /** * Created by zagnix on 2016/6/3. */ public abstract class SerializationStream{ /** The most general-purpose method to write an object. */ public abstract SerializationStream writeObject(T t); /** Writes the object representing the key of a key-value pair. */ public SerializationStream writeKey(T key){ return writeObject(key); } /** Writes the object representing the value of a key-value pair. */ public SerializationStream writeValue(T value){ return writeObject(value); } public abstract void flush(); public abstract void close(); public SerializationStream writeAll(Iterator iter){ while (iter.hasNext()) { writeObject(iter.next()); } return this; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/storage/SerializerInstance.java ================================================ package io.mycat.memory.unsafe.storage; import java.io.InputStream; import java.io.OutputStream; /** * Created by zagnix on 2016/6/3. */ public abstract class SerializerInstance { protected abstract SerializationStream serializeStream(OutputStream s ); protected abstract DeserializationStream deserializeStream(InputStream s); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/storage/SerializerManager.java ================================================ package io.mycat.memory.unsafe.storage; import java.io.InputStream; import java.io.OutputStream; /** * Created by zagnix on 2016/6/3. */ public class SerializerManager { /** * Wrap an output stream for compression if block compression is enabled for its block type */ public OutputStream wrapForCompression(ConnectionId blockId , OutputStream s){ return s; } /** * Wrap an input stream for compression if block compression is enabled for its block type */ public InputStream wrapForCompression(ConnectionId blockId, InputStream s){ return s; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/storage/TempDataNodeId.java ================================================ package io.mycat.memory.unsafe.storage; /** * Created by zagnix on 2016/6/3. */ public class TempDataNodeId extends ConnectionId { private String uuid; public TempDataNodeId(String uuid) { super(); this.name = uuid; this.uuid = uuid; } @Override public String getBlockName() { return "temp_local_" + uuid; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/storage/TimeTrackingOutputStream.java ================================================ package io.mycat.memory.unsafe.storage; import java.io.IOException; import java.io.OutputStream; /** * Intercepts write calls and tracks total time spent writing in order to update shuffle write * metrics. Not thread safe. */ public final class TimeTrackingOutputStream extends OutputStream { /**private final ShuffleWriteMetrics writeMetrics;*/ private final OutputStream outputStream; public TimeTrackingOutputStream(OutputStream outputStream) { this.outputStream = outputStream; } @Override public void write(int b) throws IOException { final long startTime = System.nanoTime(); outputStream.write(b); } @Override public void write(byte[] b) throws IOException { final long startTime = System.nanoTime(); outputStream.write(b); } @Override public void write(byte[] b, int off, int len) throws IOException { final long startTime = System.nanoTime(); outputStream.write(b, off, len); } @Override public void flush() throws IOException { final long startTime = System.nanoTime(); outputStream.flush(); } @Override public void close() throws IOException { final long startTime = System.nanoTime(); outputStream.close(); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/types/ByteArray.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.types; import io.mycat.memory.unsafe.Platform; import java.util.Arrays; public final class ByteArray { public static final byte[] EMPTY_BYTE = new byte[0]; /** * Writes the content of a byte array into a memory address, identified by an object and an * offset. The target memory address must already been allocated, and have enough space to * hold all the bytes in this string. */ public static void writeToMemory(byte[] src, Object target, long targetOffset) { Platform.copyMemory(src, Platform.BYTE_ARRAY_OFFSET, target, targetOffset, src.length); } /** * Returns a 64-bit integer that can be used as the prefix used in sorting. */ public static long getPrefix(byte[] bytes) { if (bytes == null) { return 0L; } else { final int minLen = Math.min(bytes.length, 8); long p = 0; for (int i = 0; i < minLen; ++i) { p |= (128L + Platform.getByte(bytes, Platform.BYTE_ARRAY_OFFSET + i)) << (56 - 8 * i); } return p; } } public static byte[] subStringSQL(byte[] bytes, int pos, int len) { // This pos calculation is according to UTF8String#subStringSQL if (pos > bytes.length) { return EMPTY_BYTE; } int start = 0; int end; if (pos > 0) { start = pos - 1; } else if (pos < 0) { start = bytes.length + pos; } if ((bytes.length - start) < len) { end = bytes.length; } else { end = start + len; } start = Math.max(start, 0); // underflow if (start >= end) { return EMPTY_BYTE; } return Arrays.copyOfRange(bytes, start, end); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/types/CalendarInterval.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.types; import java.io.Serializable; import java.util.regex.Matcher; import java.util.regex.Pattern; /** * The internal representation of interval type. */ public final class CalendarInterval implements Serializable { public static final long MICROS_PER_MILLI = 1000L; public static final long MICROS_PER_SECOND = MICROS_PER_MILLI * 1000; public static final long MICROS_PER_MINUTE = MICROS_PER_SECOND * 60; public static final long MICROS_PER_HOUR = MICROS_PER_MINUTE * 60; public static final long MICROS_PER_DAY = MICROS_PER_HOUR * 24; public static final long MICROS_PER_WEEK = MICROS_PER_DAY * 7; /** * A function to generate regex which matches interval string's unit part like "3 years". * * First, we can leave out some units in interval string, and we only care about the value of * unit, so here we use non-capturing group to wrap the actual regex. * At the beginning of the actual regex, we should match spaces before the unit part. * Next is the number part, starts with an optional "-" to represent negative value. We use * capturing group to wrap this part as we need the value later. * Finally is the unit name, ends with an optional "s". */ private static String unitRegex(String unit) { return "(?:\\s+(-?\\d+)\\s+" + unit + "s?)?"; } private static Pattern p = Pattern.compile("interval" + unitRegex("year") + unitRegex("month") + unitRegex("week") + unitRegex("day") + unitRegex("hour") + unitRegex("minute") + unitRegex("second") + unitRegex("millisecond") + unitRegex("microsecond")); private static Pattern yearMonthPattern = Pattern.compile("^(?:['|\"])?([+|-])?(\\d+)-(\\d+)(?:['|\"])?$"); private static Pattern dayTimePattern = Pattern.compile("^(?:['|\"])?([+|-])?(\\d+) (\\d+):(\\d+):(\\d+)(\\.(\\d+))?(?:['|\"])?$"); private static Pattern quoteTrimPattern = Pattern.compile("^(?:['|\"])?(.*?)(?:['|\"])?$"); private static long toLong(String s) { if (s == null) { return 0; } else { return Long.parseLong(s); } } public static CalendarInterval fromString(String s) { if (s == null) { return null; } s = s.trim(); Matcher m = p.matcher(s); if (!m.matches() || s.equals("interval")) { return null; } else { long months = toLong(m.group(1)) * 12 + toLong(m.group(2)); long microseconds = toLong(m.group(3)) * MICROS_PER_WEEK; microseconds += toLong(m.group(4)) * MICROS_PER_DAY; microseconds += toLong(m.group(5)) * MICROS_PER_HOUR; microseconds += toLong(m.group(6)) * MICROS_PER_MINUTE; microseconds += toLong(m.group(7)) * MICROS_PER_SECOND; microseconds += toLong(m.group(8)) * MICROS_PER_MILLI; microseconds += toLong(m.group(9)); return new CalendarInterval((int) months, microseconds); } } public static long toLongWithRange(String fieldName, String s, long minValue, long maxValue) throws IllegalArgumentException { long result = 0; if (s != null) { result = Long.parseLong(s); if (result < minValue || result > maxValue) { throw new IllegalArgumentException(String.format("%s %d outside range [%d, %d]", fieldName, result, minValue, maxValue)); } } return result; } /** * Parse YearMonth string in form: [-]YYYY-MM * * adapted from HiveIntervalYearMonth.valueOf */ public static CalendarInterval fromYearMonthString(String s) throws IllegalArgumentException { CalendarInterval result = null; if (s == null) { throw new IllegalArgumentException("Interval year-month string was null"); } s = s.trim(); Matcher m = yearMonthPattern.matcher(s); if (!m.matches()) { throw new IllegalArgumentException( "Interval string does not match year-month format of 'y-m': " + s); } else { try { int sign = m.group(1) != null && m.group(1).equals("-") ? -1 : 1; int years = (int) toLongWithRange("year", m.group(2), 0, Integer.MAX_VALUE); int months = (int) toLongWithRange("month", m.group(3), 0, 11); result = new CalendarInterval(sign * (years * 12 + months), 0); } catch (Exception e) { throw new IllegalArgumentException( "Error parsing interval year-month string: " + e.getMessage(), e); } } return result; } /** * Parse dayTime string in form: [-]d HH:mm:ss.nnnnnnnnn * * adapted from HiveIntervalDayTime.valueOf */ public static CalendarInterval fromDayTimeString(String s) throws IllegalArgumentException { CalendarInterval result = null; if (s == null) { throw new IllegalArgumentException("Interval day-time string was null"); } s = s.trim(); Matcher m = dayTimePattern.matcher(s); if (!m.matches()) { throw new IllegalArgumentException( "Interval string does not match day-time format of 'd h:m:s.n': " + s); } else { try { int sign = m.group(1) != null && m.group(1).equals("-") ? -1 : 1; long days = toLongWithRange("day", m.group(2), 0, Integer.MAX_VALUE); long hours = toLongWithRange("hour", m.group(3), 0, 23); long minutes = toLongWithRange("minute", m.group(4), 0, 59); long seconds = toLongWithRange("second", m.group(5), 0, 59); // Hive allow nanosecond precision interval long nanos = toLongWithRange("nanosecond", m.group(7), 0L, 999999999L); result = new CalendarInterval(0, sign * ( days * MICROS_PER_DAY + hours * MICROS_PER_HOUR + minutes * MICROS_PER_MINUTE + seconds * MICROS_PER_SECOND + nanos / 1000L)); } catch (Exception e) { throw new IllegalArgumentException( "Error parsing interval day-time string: " + e.getMessage(), e); } } return result; } public static CalendarInterval fromSingleUnitString(String unit, String s) throws IllegalArgumentException { CalendarInterval result = null; if (s == null) { throw new IllegalArgumentException(String.format("Interval %s string was null", unit)); } s = s.trim(); Matcher m = quoteTrimPattern.matcher(s); if (!m.matches()) { throw new IllegalArgumentException( "Interval string does not match day-time format of 'd h:m:s.n': " + s); } else { try { if (unit.equals("year")) { int year = (int) toLongWithRange("year", m.group(1), Integer.MIN_VALUE / 12, Integer.MAX_VALUE / 12); result = new CalendarInterval(year * 12, 0L); } else if (unit.equals("month")) { int month = (int) toLongWithRange("month", m.group(1), Integer.MIN_VALUE, Integer.MAX_VALUE); result = new CalendarInterval(month, 0L); } else if (unit.equals("week")) { long week = toLongWithRange("week", m.group(1), Long.MIN_VALUE / MICROS_PER_WEEK, Long.MAX_VALUE / MICROS_PER_WEEK); result = new CalendarInterval(0, week * MICROS_PER_WEEK); } else if (unit.equals("day")) { long day = toLongWithRange("day", m.group(1), Long.MIN_VALUE / MICROS_PER_DAY, Long.MAX_VALUE / MICROS_PER_DAY); result = new CalendarInterval(0, day * MICROS_PER_DAY); } else if (unit.equals("hour")) { long hour = toLongWithRange("hour", m.group(1), Long.MIN_VALUE / MICROS_PER_HOUR, Long.MAX_VALUE / MICROS_PER_HOUR); result = new CalendarInterval(0, hour * MICROS_PER_HOUR); } else if (unit.equals("minute")) { long minute = toLongWithRange("minute", m.group(1), Long.MIN_VALUE / MICROS_PER_MINUTE, Long.MAX_VALUE / MICROS_PER_MINUTE); result = new CalendarInterval(0, minute * MICROS_PER_MINUTE); } else if (unit.equals("second")) { long micros = parseSecondNano(m.group(1)); result = new CalendarInterval(0, micros); } else if (unit.equals("millisecond")) { long millisecond = toLongWithRange("millisecond", m.group(1), Long.MIN_VALUE / MICROS_PER_MILLI, Long.MAX_VALUE / MICROS_PER_MILLI); result = new CalendarInterval(0, millisecond * MICROS_PER_MILLI); } else if (unit.equals("microsecond")) { long micros = Long.parseLong(m.group(1)); result = new CalendarInterval(0, micros); } } catch (Exception e) { throw new IllegalArgumentException("Error parsing interval string: " + e.getMessage(), e); } } return result; } /** * Parse second_nano string in ss.nnnnnnnnn format to microseconds */ public static long parseSecondNano(String secondNano) throws IllegalArgumentException { String[] parts = secondNano.split("\\."); if (parts.length == 1) { return toLongWithRange("second", parts[0], Long.MIN_VALUE / MICROS_PER_SECOND, Long.MAX_VALUE / MICROS_PER_SECOND) * MICROS_PER_SECOND; } else if (parts.length == 2) { long seconds = parts[0].equals("") ? 0L : toLongWithRange("second", parts[0], Long.MIN_VALUE / MICROS_PER_SECOND, Long.MAX_VALUE / MICROS_PER_SECOND); long nanos = toLongWithRange("nanosecond", parts[1], 0L, 999999999L); return seconds * MICROS_PER_SECOND + nanos / 1000L; } else { throw new IllegalArgumentException( "Interval string does not match second-nano format of ss.nnnnnnnnn"); } } public final int months; public final long microseconds; public CalendarInterval(int months, long microseconds) { this.months = months; this.microseconds = microseconds; } public CalendarInterval add(CalendarInterval that) { int months = this.months + that.months; long microseconds = this.microseconds + that.microseconds; return new CalendarInterval(months, microseconds); } public CalendarInterval subtract(CalendarInterval that) { int months = this.months - that.months; long microseconds = this.microseconds - that.microseconds; return new CalendarInterval(months, microseconds); } public CalendarInterval negate() { return new CalendarInterval(-this.months, -this.microseconds); } @Override public boolean equals(Object other) { if (this == other) return true; if (other == null || !(other instanceof CalendarInterval)) return false; CalendarInterval o = (CalendarInterval) other; return this.months == o.months && this.microseconds == o.microseconds; } @Override public int hashCode() { return 31 * months + (int) microseconds; } @Override public String toString() { StringBuilder sb = new StringBuilder("interval"); if (months != 0) { appendUnit(sb, months / 12, "year"); appendUnit(sb, months % 12, "month"); } if (microseconds != 0) { long rest = microseconds; appendUnit(sb, rest / MICROS_PER_WEEK, "week"); rest %= MICROS_PER_WEEK; appendUnit(sb, rest / MICROS_PER_DAY, "day"); rest %= MICROS_PER_DAY; appendUnit(sb, rest / MICROS_PER_HOUR, "hour"); rest %= MICROS_PER_HOUR; appendUnit(sb, rest / MICROS_PER_MINUTE, "minute"); rest %= MICROS_PER_MINUTE; appendUnit(sb, rest / MICROS_PER_SECOND, "second"); rest %= MICROS_PER_SECOND; appendUnit(sb, rest / MICROS_PER_MILLI, "millisecond"); rest %= MICROS_PER_MILLI; appendUnit(sb, rest, "microsecond"); } return sb.toString(); } private void appendUnit(StringBuilder sb, long value, String unit) { if (value != 0) { sb.append(' ').append(value).append(' ').append(unit).append('s'); } } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/types/UTF8String.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.types; import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.KryoSerializable; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.array.ByteArrayMethods; import io.mycat.memory.unsafe.hash.Murmur3_x86_32; import javax.annotation.Nonnull; import java.io.Externalizable; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Map; /** * A UTF-8 String for internal Spark use. *

* A String encoded in UTF-8 as an Array[Byte], which can be used for comparison, * search, see http://en.wikipedia.org/wiki/UTF-8 for details. *

* Note: This is not designed for general use cases, should not be used outside SQL. */ public final class UTF8String implements Comparable, Externalizable, KryoSerializable, Cloneable { // These are only updated by readExternal() or read() @Nonnull private Object base; private long offset; private int numBytes; public Object getBaseObject() { return base; } public long getBaseOffset() { return offset; } private static int[] bytesOfCodePointInUTF8 = {2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6}; private static boolean isLittleEndian = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN; private static final UTF8String COMMA_UTF8 = UTF8String.fromString(","); public static final UTF8String EMPTY_UTF8 = UTF8String.fromString(""); /** * Creates an UTF8String from byte array, which should be encoded in UTF-8. * * Note: `bytes` will be hold by returned UTF8String. */ public static UTF8String fromBytes(byte[] bytes) { if (bytes != null) { return new UTF8String(bytes, Platform.BYTE_ARRAY_OFFSET, bytes.length); } else { return null; } } /** * Creates an UTF8String from byte array, which should be encoded in UTF-8. * * Note: `bytes` will be hold by returned UTF8String. */ public static UTF8String fromBytes(byte[] bytes, int offset, int numBytes) { if (bytes != null) { return new UTF8String(bytes, Platform.BYTE_ARRAY_OFFSET + offset, numBytes); } else { return null; } } /** * Creates an UTF8String from given address (base and offset) and length. */ public static UTF8String fromAddress(Object base, long offset, int numBytes) { return new UTF8String(base, offset, numBytes); } /** * Creates an UTF8String from String. */ public static UTF8String fromString(String str) { return str == null ? null : fromBytes(str.getBytes(StandardCharsets.UTF_8)); } /** * Creates an UTF8String that contains `length` spaces. */ public static UTF8String blankString(int length) { byte[] spaces = new byte[length]; Arrays.fill(spaces, (byte) ' '); return fromBytes(spaces); } protected UTF8String(Object base, long offset, int numBytes) { this.base = base; this.offset = offset; this.numBytes = numBytes; } // for serialization public UTF8String() { this(null, 0, 0); } /** * Writes the content of this string into a memory address, identified by an object and an offset. * The target memory address must already been allocated, and have enough space to hold all the * bytes in this string. */ public void writeToMemory(Object target, long targetOffset) { Platform.copyMemory(base, offset, target, targetOffset, numBytes); } public void writeTo(ByteBuffer buffer) { assert(buffer.hasArray()); byte[] target = buffer.array(); int offset = buffer.arrayOffset(); int pos = buffer.position(); writeToMemory(target, Platform.BYTE_ARRAY_OFFSET + offset + pos); buffer.position(pos + numBytes); } /** * Returns the number of bytes for a code point with the first byte as `b` * @param b The first byte of a code point */ private static int numBytesForFirstByte(final byte b) { final int offset = (b & 0xFF) - 192; return (offset >= 0) ? bytesOfCodePointInUTF8[offset] : 1; } /** * Returns the number of bytes */ public int numBytes() { return numBytes; } /** * Returns the number of code points in it. */ public int numChars() { int len = 0; for (int i = 0; i < numBytes; i += numBytesForFirstByte(getByte(i))) { len += 1; } return len; } /** * Returns a 64-bit integer that can be used as the prefix used in sorting. */ public long getPrefix() { // Since JVMs are either 4-byte aligned or 8-byte aligned, we check the size of the string. // If size is 0, just return 0. // If size is between 0 and 4 (inclusive), assume data is 4-byte aligned under the hood and // use a getInt to fetch the prefix. // If size is greater than 4, assume we have at least 8 bytes of data to fetch. // After getting the data, we use a mask to mask out data that is not part of the string. long p; long mask = 0; if (isLittleEndian) { if (numBytes >= 8) { p = Platform.getLong(base, offset); } else if (numBytes > 4) { p = Platform.getLong(base, offset); mask = (1L << (8 - numBytes) * 8) - 1; } else if (numBytes > 0) { p = (long) Platform.getInt(base, offset); mask = (1L << (8 - numBytes) * 8) - 1; } else { p = 0; } p = Long.reverseBytes(p); } else { // byteOrder == ByteOrder.BIG_ENDIAN if (numBytes >= 8) { p = Platform.getLong(base, offset); } else if (numBytes > 4) { p = Platform.getLong(base, offset); mask = (1L << (8 - numBytes) * 8) - 1; } else if (numBytes > 0) { p = ((long) Platform.getInt(base, offset)) << 32; mask = (1L << (8 - numBytes) * 8) - 1; } else { p = 0; } } p &= ~mask; return p; } /** * Returns the underline bytes, will be a copy of it if it's part of another array. */ public byte[] getBytes() { // avoid copy if `base` is `byte[]` if (offset == Platform.BYTE_ARRAY_OFFSET && base instanceof byte[] && ((byte[]) base).length == numBytes) { return (byte[]) base; } else { byte[] bytes = new byte[numBytes]; Platform.copyMemory(base, offset, bytes, Platform.BYTE_ARRAY_OFFSET, numBytes); return bytes; } } /** * Returns a substring of this. * @param start the position of first code point * @param until the position after last code point, exclusive. */ public UTF8String substring(final int start, final int until) { if (until <= start || start >= numBytes) { return EMPTY_UTF8; } int i = 0; int c = 0; while (i < numBytes && c < start) { i += numBytesForFirstByte(getByte(i)); c += 1; } int j = i; while (i < numBytes && c < until) { i += numBytesForFirstByte(getByte(i)); c += 1; } if (i > j) { byte[] bytes = new byte[i - j]; Platform.copyMemory(base, offset + j, bytes, Platform.BYTE_ARRAY_OFFSET, i - j); return fromBytes(bytes); } else { return EMPTY_UTF8; } } public UTF8String substringSQL(int pos, int length) { // Information regarding the pos calculation: // Hive and SQL use one-based indexing for SUBSTR arguments but also accept zero and // negative indices for start positions. If a start index i is greater than 0, it // refers to element i-1 in the sequence. If a start index i is less than 0, it refers // to the -ith element before the end of the sequence. If a start index i is 0, it // refers to the first element. int len = numChars(); int start = (pos > 0) ? pos -1 : ((pos < 0) ? len + pos : 0); int end = (length == Integer.MAX_VALUE) ? len : start + length; return substring(start, end); } /** * Returns whether this contains `substring` or not. */ public boolean contains(final UTF8String substring) { if (substring.numBytes == 0) { return true; } byte first = substring.getByte(0); for (int i = 0; i <= numBytes - substring.numBytes; i++) { if (getByte(i) == first && matchAt(substring, i)) { return true; } } return false; } /** * Returns the byte at position `i`. */ private byte getByte(int i) { return Platform.getByte(base, offset + i); } private boolean matchAt(final UTF8String s, int pos) { if (s.numBytes + pos > numBytes || pos < 0) { return false; } return ByteArrayMethods.arrayEquals(base, offset + pos, s.base, s.offset, s.numBytes); } public boolean startsWith(final UTF8String prefix) { return matchAt(prefix, 0); } public boolean endsWith(final UTF8String suffix) { return matchAt(suffix, numBytes - suffix.numBytes); } /** * Returns the upper case of this string */ public UTF8String toUpperCase() { if (numBytes == 0) { return EMPTY_UTF8; } byte[] bytes = new byte[numBytes]; bytes[0] = (byte) Character.toTitleCase(getByte(0)); for (int i = 0; i < numBytes; i++) { byte b = getByte(i); if (numBytesForFirstByte(b) != 1) { // fallback return toUpperCaseSlow(); } int upper = Character.toUpperCase((int) b); if (upper > 127) { // fallback return toUpperCaseSlow(); } bytes[i] = (byte) upper; } return fromBytes(bytes); } private UTF8String toUpperCaseSlow() { return fromString(toString().toUpperCase()); } /** * Returns the lower case of this string */ public UTF8String toLowerCase() { if (numBytes == 0) { return EMPTY_UTF8; } byte[] bytes = new byte[numBytes]; bytes[0] = (byte) Character.toTitleCase(getByte(0)); for (int i = 0; i < numBytes; i++) { byte b = getByte(i); if (numBytesForFirstByte(b) != 1) { // fallback return toLowerCaseSlow(); } int lower = Character.toLowerCase((int) b); if (lower > 127) { // fallback return toLowerCaseSlow(); } bytes[i] = (byte) lower; } return fromBytes(bytes); } private UTF8String toLowerCaseSlow() { return fromString(toString().toLowerCase()); } /** * Returns the title case of this string, that could be used as title. */ public UTF8String toTitleCase() { if (numBytes == 0) { return EMPTY_UTF8; } byte[] bytes = new byte[numBytes]; for (int i = 0; i < numBytes; i++) { byte b = getByte(i); if (i == 0 || getByte(i - 1) == ' ') { if (numBytesForFirstByte(b) != 1) { // fallback return toTitleCaseSlow(); } int upper = Character.toTitleCase(b); if (upper > 127) { // fallback return toTitleCaseSlow(); } bytes[i] = (byte) upper; } else { bytes[i] = b; } } return fromBytes(bytes); } private UTF8String toTitleCaseSlow() { StringBuffer sb = new StringBuffer(); String s = toString(); sb.append(s); sb.setCharAt(0, Character.toTitleCase(sb.charAt(0))); for (int i = 1; i < s.length(); i++) { if (sb.charAt(i - 1) == ' ') { sb.setCharAt(i, Character.toTitleCase(sb.charAt(i))); } } return fromString(sb.toString()); } /* * Returns the index of the string `match` in this String. This string has to be a comma separated * list. If `match` contains a comma 0 will be returned. If the `match` isn't part of this String, * 0 will be returned, else the index of match (1-based index) */ public int findInSet(UTF8String match) { if (match.contains(COMMA_UTF8)) { return 0; } int n = 1, lastComma = -1; for (int i = 0; i < numBytes; i++) { if (getByte(i) == (byte) ',') { if (i - (lastComma + 1) == match.numBytes && ByteArrayMethods.arrayEquals(base, offset + (lastComma + 1), match.base, match.offset, match.numBytes)) { return n; } lastComma = i; n++; } } if (numBytes - (lastComma + 1) == match.numBytes && ByteArrayMethods.arrayEquals(base, offset + (lastComma + 1), match.base, match.offset, match.numBytes)) { return n; } return 0; } /** * Copy the bytes from the current UTF8String, and make a new UTF8String. * @param start the start position of the current UTF8String in bytes. * @param end the end position of the current UTF8String in bytes. * @return a new UTF8String in the position of [start, end] of current UTF8String bytes. */ private UTF8String copyUTF8String(int start, int end) { int len = end - start + 1; byte[] newBytes = new byte[len]; Platform.copyMemory(base, offset + start, newBytes, Platform.BYTE_ARRAY_OFFSET, len); return UTF8String.fromBytes(newBytes); } public UTF8String trim() { int s = 0; int e = this.numBytes - 1; // skip all of the space (0x20) in the left side while (s < this.numBytes && getByte(s) <= 0x20 && getByte(s) >= 0x00) s++; // skip all of the space (0x20) in the right side while (e >= 0 && getByte(e) <= 0x20 && getByte(e) >= 0x00) e--; if (s > e) { // empty string return UTF8String.fromBytes(new byte[0]); } else { return copyUTF8String(s, e); } } public UTF8String trimLeft() { int s = 0; // skip all of the space (0x20) in the left side while (s < this.numBytes && getByte(s) <= 0x20 && getByte(s) >= 0x00) s++; if (s == this.numBytes) { // empty string return UTF8String.fromBytes(new byte[0]); } else { return copyUTF8String(s, this.numBytes - 1); } } public UTF8String trimRight() { int e = numBytes - 1; // skip all of the space (0x20) in the right side while (e >= 0 && getByte(e) <= 0x20 && getByte(e) >= 0x00) e--; if (e < 0) { // empty string return UTF8String.fromBytes(new byte[0]); } else { return copyUTF8String(0, e); } } public UTF8String reverse() { byte[] result = new byte[this.numBytes]; int i = 0; // position in byte while (i < numBytes) { int len = numBytesForFirstByte(getByte(i)); Platform.copyMemory(this.base, this.offset + i, result, Platform.BYTE_ARRAY_OFFSET + result.length - i - len, len); i += len; } return UTF8String.fromBytes(result); } public UTF8String repeat(int times) { if (times <= 0) { return EMPTY_UTF8; } byte[] newBytes = new byte[numBytes * times]; Platform.copyMemory(this.base, this.offset, newBytes, Platform.BYTE_ARRAY_OFFSET, numBytes); int copied = 1; while (copied < times) { int toCopy = Math.min(copied, times - copied); System.arraycopy(newBytes, 0, newBytes, copied * numBytes, numBytes * toCopy); copied += toCopy; } return UTF8String.fromBytes(newBytes); } /** * Returns the position of the first occurrence of substr in * current string from the specified position (0-based index). * * @param v the string to be searched * @param start the start position of the current string for searching * @return the position of the first occurrence of substr, if not found, -1 returned. */ public int indexOf(UTF8String v, int start) { if (v.numBytes() == 0) { return 0; } // locate to the start position. int i = 0; // position in byte int c = 0; // position in character while (i < numBytes && c < start) { i += numBytesForFirstByte(getByte(i)); c += 1; } do { if (i + v.numBytes > numBytes) { return -1; } if (ByteArrayMethods.arrayEquals(base, offset + i, v.base, v.offset, v.numBytes)) { return c; } i += numBytesForFirstByte(getByte(i)); c += 1; } while (i < numBytes); return -1; } /** * Find the `str` from left to right. */ private int find(UTF8String str, int start) { assert (str.numBytes > 0); while (start <= numBytes - str.numBytes) { if (ByteArrayMethods.arrayEquals(base, offset + start, str.base, str.offset, str.numBytes)) { return start; } start += 1; } return -1; } /** * Find the `str` from right to left. */ private int rfind(UTF8String str, int start) { assert (str.numBytes > 0); while (start >= 0) { if (ByteArrayMethods.arrayEquals(base, offset + start, str.base, str.offset, str.numBytes)) { return start; } start -= 1; } return -1; } /** * Returns the substring from string str before count occurrences of the delimiter delim. * If count is positive, everything the left of the final delimiter (counting from left) is * returned. If count is negative, every to the right of the final delimiter (counting from the * right) is returned. subStringIndex performs a case-sensitive match when searching for delim. */ public UTF8String subStringIndex(UTF8String delim, int count) { if (delim.numBytes == 0 || count == 0) { return EMPTY_UTF8; } if (count > 0) { int idx = -1; while (count > 0) { idx = find(delim, idx + 1); if (idx >= 0) { count --; } else { // can not find enough delim return this; } } if (idx == 0) { return EMPTY_UTF8; } byte[] bytes = new byte[idx]; Platform.copyMemory(base, offset, bytes, Platform.BYTE_ARRAY_OFFSET, idx); return fromBytes(bytes); } else { int idx = numBytes - delim.numBytes + 1; count = -count; while (count > 0) { idx = rfind(delim, idx - 1); if (idx >= 0) { count --; } else { // can not find enough delim return this; } } if (idx + delim.numBytes == numBytes) { return EMPTY_UTF8; } int size = numBytes - delim.numBytes - idx; byte[] bytes = new byte[size]; Platform.copyMemory(base, offset + idx + delim.numBytes, bytes, Platform.BYTE_ARRAY_OFFSET, size); return fromBytes(bytes); } } /** * Returns str, right-padded with pad to a length of len * For example: * ('hi', 5, '??') => 'hi???' * ('hi', 1, '??') => 'h' */ public UTF8String rpad(int len, UTF8String pad) { int spaces = len - this.numChars(); // number of char need to pad if (spaces <= 0 || pad.numBytes() == 0) { // no padding at all, return the substring of the current string return substring(0, len); } else { int padChars = pad.numChars(); int count = spaces / padChars; // how many padding string needed // the partial string of the padding UTF8String remain = pad.substring(0, spaces - padChars * count); byte[] data = new byte[this.numBytes + pad.numBytes * count + remain.numBytes]; Platform.copyMemory(this.base, this.offset, data, Platform.BYTE_ARRAY_OFFSET, this.numBytes); int offset = this.numBytes; int idx = 0; while (idx < count) { Platform.copyMemory(pad.base, pad.offset, data, Platform.BYTE_ARRAY_OFFSET + offset, pad.numBytes); ++ idx; offset += pad.numBytes; } Platform.copyMemory(remain.base, remain.offset, data, Platform.BYTE_ARRAY_OFFSET + offset, remain.numBytes); return UTF8String.fromBytes(data); } } /** * Returns str, left-padded with pad to a length of len. * For example: * ('hi', 5, '??') => '???hi' * ('hi', 1, '??') => 'h' */ public UTF8String lpad(int len, UTF8String pad) { int spaces = len - this.numChars(); // number of char need to pad if (spaces <= 0 || pad.numBytes() == 0) { // no padding at all, return the substring of the current string return substring(0, len); } else { int padChars = pad.numChars(); int count = spaces / padChars; // how many padding string needed // the partial string of the padding UTF8String remain = pad.substring(0, spaces - padChars * count); byte[] data = new byte[this.numBytes + pad.numBytes * count + remain.numBytes]; int offset = 0; int idx = 0; while (idx < count) { Platform.copyMemory(pad.base, pad.offset, data, Platform.BYTE_ARRAY_OFFSET + offset, pad.numBytes); ++ idx; offset += pad.numBytes; } Platform.copyMemory(remain.base, remain.offset, data, Platform.BYTE_ARRAY_OFFSET + offset, remain.numBytes); offset += remain.numBytes; Platform.copyMemory(this.base, this.offset, data, Platform.BYTE_ARRAY_OFFSET + offset, numBytes()); return UTF8String.fromBytes(data); } } /** * Concatenates input strings together into a single string. Returns null if any input is null. */ public static UTF8String concat(UTF8String... inputs) { // Compute the total length of the result. int totalLength = 0; for (int i = 0; i < inputs.length; i++) { if (inputs[i] != null) { totalLength += inputs[i].numBytes; } else { return null; } } // Allocate a new byte array, and copy the inputs one by one into it. final byte[] result = new byte[totalLength]; int offset = 0; for (int i = 0; i < inputs.length; i++) { int len = inputs[i].numBytes; Platform.copyMemory( inputs[i].base, inputs[i].offset, result, Platform.BYTE_ARRAY_OFFSET + offset, len); offset += len; } return fromBytes(result); } /** * Concatenates input strings together into a single string using the separator. * A null input is skipped. For example, concat(",", "a", null, "c") would yield "a,c". */ public static UTF8String concatWs(UTF8String separator, UTF8String... inputs) { if (separator == null) { return null; } int numInputBytes = 0; // total number of bytes from the inputs int numInputs = 0; // number of non-null inputs for (int i = 0; i < inputs.length; i++) { if (inputs[i] != null) { numInputBytes += inputs[i].numBytes; numInputs++; } } if (numInputs == 0) { // Return an empty string if there is no input, or all the inputs are null. return fromBytes(new byte[0]); } // Allocate a new byte array, and copy the inputs one by one into it. // The size of the new array is the size of all inputs, plus the separators. final byte[] result = new byte[numInputBytes + (numInputs - 1) * separator.numBytes]; int offset = 0; for (int i = 0, j = 0; i < inputs.length; i++) { if (inputs[i] != null) { int len = inputs[i].numBytes; Platform.copyMemory( inputs[i].base, inputs[i].offset, result, Platform.BYTE_ARRAY_OFFSET + offset, len); offset += len; j++; // Add separator if this is not the last input. if (j < numInputs) { Platform.copyMemory( separator.base, separator.offset, result, Platform.BYTE_ARRAY_OFFSET + offset, separator.numBytes); offset += separator.numBytes; } } } return fromBytes(result); } public UTF8String[] split(UTF8String pattern, int limit) { String[] splits = toString().split(pattern.toString(), limit); UTF8String[] res = new UTF8String[splits.length]; for (int i = 0; i < res.length; i++) { res[i] = fromString(splits[i]); } return res; } // TODO: Need to use `Code Point` here instead of Char in case the character longer than 2 bytes public UTF8String translate(Map dict) { String srcStr = this.toString(); StringBuilder sb = new StringBuilder(); for(int k = 0; k< srcStr.length(); k++) { if (null == dict.get(srcStr.charAt(k))) { sb.append(srcStr.charAt(k)); } else if ('\0' != dict.get(srcStr.charAt(k))){ sb.append(dict.get(srcStr.charAt(k))); } } return fromString(sb.toString()); } @Override public String toString() { return new String(getBytes(), StandardCharsets.UTF_8); } @Override public UTF8String clone() { return fromBytes(getBytes()); } @Override public int compareTo(@Nonnull final UTF8String other) { int len = Math.min(numBytes, other.numBytes); // TODO: compare 8 bytes as unsigned long for (int i = 0; i < len; i ++) { // In UTF-8, the byte should be unsigned, so we should compare them as unsigned int. int res = (getByte(i) & 0xFF) - (other.getByte(i) & 0xFF); if (res != 0) { return res; } } return numBytes - other.numBytes; } public int compare(final UTF8String other) { return compareTo(other); } @Override public boolean equals(final Object other) { if (other instanceof UTF8String) { UTF8String o = (UTF8String) other; if (numBytes != o.numBytes) { return false; } return ByteArrayMethods.arrayEquals(base, offset, o.base, o.offset, numBytes); } else { return false; } } /** * Levenshtein distance is a metric for measuring the distance of two strings. The distance is * defined by the minimum number of single-character edits (i.e. insertions, deletions or * substitutions) that are required to change one of the strings into the other. */ public int levenshteinDistance(UTF8String other) { // Implementation adopted from org.apache.common.lang3.StringUtils.getLevenshteinDistance int n = numChars(); int m = other.numChars(); if (n == 0) { return m; } else if (m == 0) { return n; } UTF8String s, t; if (n <= m) { s = this; t = other; } else { s = other; t = this; int swap; swap = n; n = m; m = swap; } int[] p = new int[n + 1]; int[] d = new int[n + 1]; int[] swap; int i, i_bytes, j, j_bytes, num_bytes_j, cost; for (i = 0; i <= n; i++) { p[i] = i; } for (j = 0, j_bytes = 0; j < m; j_bytes += num_bytes_j, j++) { num_bytes_j = numBytesForFirstByte(t.getByte(j_bytes)); d[0] = j + 1; for (i = 0, i_bytes = 0; i < n; i_bytes += numBytesForFirstByte(s.getByte(i_bytes)), i++) { if (s.getByte(i_bytes) != t.getByte(j_bytes) || num_bytes_j != numBytesForFirstByte(s.getByte(i_bytes))) { cost = 1; } else { cost = (ByteArrayMethods.arrayEquals(t.base, t.offset + j_bytes, s.base, s.offset + i_bytes, num_bytes_j)) ? 0 : 1; } d[i + 1] = Math.min(Math.min(d[i] + 1, p[i + 1] + 1), p[i] + cost); } swap = p; p = d; d = swap; } return p[n]; } @Override public int hashCode() { return Murmur3_x86_32.hashUnsafeBytes(base, offset, numBytes, 42); } /** * Soundex mapping table */ private static final byte[] US_ENGLISH_MAPPING = {'0', '1', '2', '3', '0', '1', '2', '7', '0', '2', '2', '4', '5', '5', '0', '1', '2', '6', '2', '3', '0', '1', '7', '2', '0', '2'}; /** * Encodes a string into a Soundex value. Soundex is an encoding used to relate similar names, * but can also be used as a general purpose scheme to find word with similar phonemes. * https://en.wikipedia.org/wiki/Soundex */ public UTF8String soundex() { if (numBytes == 0) { return EMPTY_UTF8; } byte b = getByte(0); if ('a' <= b && b <= 'z') { b -= 32; } else if (b < 'A' || 'Z' < b) { // first character must be a letter return this; } byte[] sx = {'0', '0', '0', '0'}; sx[0] = b; int sxi = 1; int idx = b - 'A'; byte lastCode = US_ENGLISH_MAPPING[idx]; for (int i = 1; i < numBytes; i++) { b = getByte(i); if ('a' <= b && b <= 'z') { b -= 32; } else if (b < 'A' || 'Z' < b) { // not a letter, skip it lastCode = '0'; continue; } idx = b - 'A'; byte code = US_ENGLISH_MAPPING[idx]; if (code == '7') { // ignore it } else { if (code != '0' && code != lastCode) { sx[sxi++] = code; if (sxi > 3) break; } lastCode = code; } } return UTF8String.fromBytes(sx); } public void writeExternal(ObjectOutput out) throws IOException { byte[] bytes = getBytes(); out.writeInt(bytes.length); out.write(bytes); } public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { offset = Platform.BYTE_ARRAY_OFFSET; numBytes = in.readInt(); base = new byte[numBytes]; in.readFully((byte[]) base); } @Override public void write(Kryo kryo, Output out) { byte[] bytes = getBytes(); out.writeInt(bytes.length); out.write(bytes); } @Override public void read(Kryo kryo, Input in) { this.offset = Platform.BYTE_ARRAY_OFFSET; this.numBytes = in.readInt(); this.base = new byte[numBytes]; in.read((byte[]) base); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/ByteUnit.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.utils; public enum ByteUnit { BYTE (1), KiB (1024L), MiB ((long) Math.pow(1024L, 2L)), GiB ((long) Math.pow(1024L, 3L)), TiB ((long) Math.pow(1024L, 4L)), PiB ((long) Math.pow(1024L, 5L)), ; ByteUnit(long multiplier) { this.multiplier = multiplier; } // Interpret the provided number (d) with suffix (u) as this unit type. // E.g. KiB.interpret(1, MiB) interprets 1MiB as its KiB representation = 1024k public long convertFrom(long d, ByteUnit u) { return u.convertTo(d, this); } // Convert the provided number (d) interpreted as this unit type to unit type (u). public long convertTo(long d, ByteUnit u) { if (multiplier > u.multiplier) { long ratio = multiplier / u.multiplier; if (Long.MAX_VALUE / ratio < d) { throw new IllegalArgumentException("Conversion of " + d + " exceeds Long.MAX_VALUE in " + name() + ". Try a larger unit (e.g. MiB instead of KiB)"); } return d * ratio; } else { // Perform operations in this order to avoid potential overflow // when computing d * multiplier return d / (u.multiplier / multiplier); } } public double toBytes(long d) { if (d < 0) { throw new IllegalArgumentException("Negative size value. Size must be positive: " + d); } return d * multiplier; } public long toKiB(long d) { return convertTo(d, KiB); } public long toMiB(long d) { return convertTo(d, MiB); } public long toGiB(long d) { return convertTo(d, GiB); } public long toTiB(long d) { return convertTo(d, TiB); } public long toPiB(long d) { return convertTo(d, PiB); } private final long multiplier; } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/BytesTools.java ================================================ package io.mycat.memory.unsafe.utils; /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import com.google.common.annotations.VisibleForTesting; import io.mycat.memory.unsafe.Platform; import sun.misc.Unsafe; import java.io.UnsupportedEncodingException; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.nio.charset.IllegalCharsetNameException; import java.nio.charset.UnsupportedCharsetException; import java.util.concurrent.ConcurrentHashMap; /** * Utility class that handles byte arrays, conversions to/from other types, */ @SuppressWarnings("restriction") public class BytesTools { //HConstants.UTF8_ENCODING should be updated if this changed /** When we encode strings, we always specify UTF8 encoding */ private static final String UTF8_ENCODING = "UTF-8"; //HConstants.UTF8_CHARSET should be updated if this changed /** When we encode strings, we always specify UTF8 encoding */ private static final Charset UTF8_CHARSET = Charset.forName(UTF8_ENCODING); /** * Size of boolean in bytes */ public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE; /** * Size of byte in bytes */ public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN; /** * Size of char in bytes */ public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE; /** * Size of double in bytes */ public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE; /** * Size of float in bytes */ public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE; /** * Size of int in bytes */ public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE; /** * Size of long in bytes */ public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE; /** * Size of short in bytes */ public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE; /** * Convert a byte array to a int value * @param buf * @return int * @throws NumberFormatException */ public static int getInt(byte[] buf) throws NumberFormatException { return getInt(buf, 0, buf.length); } public static int getInt(byte[] buf, int offset, int endPos) throws NumberFormatException { byte base = 10; int s; for(s = offset; s < endPos && Character.isWhitespace((char)buf[s]); ++s) { ; } if(s == endPos) { throw new NumberFormatException(toString(buf)); } else { boolean negative = false; if((char)buf[s] == 45) { negative = true; ++s; } else if((char)buf[s] == 43) { ++s; } int save = s; int cutoff = 2147483647 / base; int cutlim = 2147483647 % base; if(negative) { ++cutlim; } boolean overflow = false; int i; for(i = 0; s < endPos; ++s) { char c = (char)buf[s]; if(Character.isDigit(c)) { c = (char)(c - 48); } else { if(!Character.isLetter(c)) { break; } c = (char)(Character.toUpperCase(c) - 65 + 10); } if(c >= base) { break; } if(i <= cutoff && (i != cutoff || c <= cutlim)) { i *= base; i += c; } else { overflow = true; } } if(s == save) { throw new NumberFormatException(toString(buf)); } else if(overflow) { throw new NumberFormatException(toString(buf)); } else { return negative?-i:i; } } } /** * Convert a byte array to a long value * @param buf * @return * @throws NumberFormatException */ public static long getLong(byte[] buf) throws NumberFormatException { return getLong(buf, 0, buf.length); } public static long getLong(byte[] buf, int offset, int endpos) throws NumberFormatException { byte base = 10; int s; for(s = offset; s < endpos && Character.isWhitespace((char)buf[s]); ++s) { ; } if(s == endpos) { throw new NumberFormatException(toString(buf)); } else { boolean negative = false; if((char)buf[s] == 45) { negative = true; ++s; } else if((char)buf[s] == 43) { ++s; } int save = s; long cutoff = 9223372036854775807L / (long)base; long cutlim = (long)((int)(9223372036854775807L % (long)base)); if(negative) { ++cutlim; } boolean overflow = false; long i; for(i = 0L; s < endpos; ++s) { char c = (char)buf[s]; if(Character.isDigit(c)) { c = (char)(c - 48); } else { if(!Character.isLetter(c)) { break; } c = (char)(Character.toUpperCase(c) - 65 + 10); } if(c >= base) { break; } if(i <= cutoff && (i != cutoff || (long)c <= cutlim)) { i *= (long)base; i += (long)c; } else { overflow = true; } } if(s == save) { throw new NumberFormatException(toString(buf)); } else if(overflow) { throw new NumberFormatException(toString(buf)); } else { return negative?-i:i; } } } /** * Convert a byte array to a short value * @param buf * @return * @throws NumberFormatException */ public static short getShort(byte[] buf) throws NumberFormatException { return getShort(buf, 0, buf.length); } public static short getShort(byte[] buf, int offset, int endpos) throws NumberFormatException { byte base = 10; int s; for(s = offset; s < endpos && Character.isWhitespace((char)buf[s]); ++s) { ; } if(s == endpos) { throw new NumberFormatException(toString(buf)); } else { boolean negative = false; if((char)buf[s] == 45) { negative = true; ++s; } else if((char)buf[s] == 43) { ++s; } int save = s; short cutoff = (short)(32767 / base); short cutlim = (short)(32767 % base); if(negative) { ++cutlim; } boolean overflow = false; short i; for(i = 0; s < endpos; ++s) { char c = (char)buf[s]; if(Character.isDigit(c)) { c = (char)(c - 48); } else { if(!Character.isLetter(c)) { break; } c = (char)(Character.toUpperCase(c) - 65 + 10); } if(c >= base) { break; } if(i <= cutoff && (i != cutoff || c <= cutlim)) { i = (short)(i * base); i = (short)(i + c); } else { overflow = true; } } if(s == save) { throw new NumberFormatException(toString(buf)); } else if(overflow) { throw new NumberFormatException(toString(buf)); } else { return negative?(short)(-i):i; } } } /** * Convert a byte array to a float value * @param src * @return * @throws UnsupportedEncodingException */ public static float getFloat(byte [] src) throws UnsupportedEncodingException { return Float.parseFloat(new String(src,"US-ASCII")); } /** * Convert a byte array to a double value * @param src * @return * @throws UnsupportedEncodingException */ public static double getDouble(byte [] src) throws UnsupportedEncodingException { return Double.parseDouble(new String(src,"US-ASCII")); } /** * Convert a long value to a byte array * @param l * @return * @throws UnsupportedEncodingException */ public static byte[] long2Bytes(long l) throws UnsupportedEncodingException { String lstr = Long.toString(l); return lstr.getBytes("US-ASCII"); } /** * Convert a int value to a byte array * @param i * @return * @throws UnsupportedEncodingException */ public static byte[] int2Bytes(int i) throws UnsupportedEncodingException { String istr = Integer.toString(i); return istr.getBytes("US-ASCII"); } /** * Convert a short value to a byte array * @param i * @return * @throws UnsupportedEncodingException */ public static byte[] short2Bytes(short i) throws UnsupportedEncodingException { String sstr = Short.toString(i); return sstr.getBytes("US-ASCII"); } /** * Convert a float value to a byte array * @param f * @return * @throws UnsupportedEncodingException */ public static byte[] float2Bytes(float f) throws UnsupportedEncodingException { String fstr = Float.toString(f); return fstr.getBytes("US-ASCII"); } /** * Convert a double value to a byte array * @param d * @return * @throws UnsupportedEncodingException */ public static byte[] double2Bytes(double d) throws UnsupportedEncodingException { String dstr = Double.toString(d); return dstr.getBytes("US-ASCII"); } /** * Returns a new byte array, copied from the given {@code buf}, * from the index 0 (inclusive) to the limit (exclusive), * regardless of the current position. * The position and the other index parameters are not changed. * * @param buf a byte buffer * @return the byte array */ public static byte[] toBytes(ByteBuffer buf) { ByteBuffer dup = buf.duplicate(); dup.position(0); return readBytes(dup); } private static byte[] readBytes(ByteBuffer buf) { byte [] result = new byte[buf.remaining()]; buf.get(result); return result; } /** * @param b Presumed UTF-8 encoded byte array. * @return String made from b */ public static String toString(final byte [] b) { if (b == null) { return null; } return toString(b, 0, b.length); } /** * Joins two byte arrays together using a separator. * @param b1 The first byte array. * @param sep The separator to use. * @param b2 The second byte array. */ public static String toString(final byte [] b1, String sep, final byte [] b2) { return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length); } /** * This method will convert utf8 encoded bytes into a string. If * the given byte array is null, this method will return null. * * @param b Presumed UTF-8 encoded byte array. * @param off offset into array * @return String made from b or null */ public static String toString(final byte [] b, int off) { if (b == null) { return null; } int len = b.length - off; if (len <= 0) { return ""; } return new String(b, off, len, UTF8_CHARSET); } /** * This method will convert utf8 encoded bytes into a string. If * the given byte array is null, this method will return null. * * @param b Presumed UTF-8 encoded byte array. * @param off offset into array * @param len length of utf-8 sequence * @return String made from b or null */ public static String toString(final byte [] b, int off, int len) { if (b == null) { return null; } if (len == 0) { return ""; } return new String(b, off, len, UTF8_CHARSET); } /** * Write a printable representation of a byte array. * * @param b byte array * @return string * @see #toStringBinary(byte[], int, int) */ public static String toStringBinary(final byte [] b) { if (b == null) return "null"; return toStringBinary(b, 0, b.length); } /** * Converts the given byte buffer to a printable representation, * from the index 0 (inclusive) to the limit (exclusive), * regardless of the current position. * The position and the other index parameters are not changed. * * @param buf a byte buffer * @return a string representation of the buffer's binary contents * @see #toBytes(ByteBuffer) */ public static String toStringBinary(ByteBuffer buf) { if (buf == null) return "null"; if (buf.hasArray()) { return toStringBinary(buf.array(), buf.arrayOffset(), buf.limit()); } return toStringBinary(toBytes(buf)); } private static final char[] HEX_CHARS_UPPER = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; /** * Write a printable representation of a byte array. Non-printable * characters are hex escaped in the format \\x%02X, eg: * \x00 \x05 etc * * @param b array to write out * @param off offset to start at * @param len length to write * @return string output */ public static String toStringBinary(final byte [] b, int off, int len) { StringBuilder result = new StringBuilder(); // Just in case we are passed a 'len' that is > buffer length... if (off >= b.length) return result.toString(); if (off + len > b.length) len = b.length - off; for (int i = off; i < off + len ; ++i) { int ch = b[i] & 0xFF; if (ch >= ' ' && ch <= '~' && ch != '\\') { result.append((char)ch); } else { result.append("\\x"); result.append(HEX_CHARS_UPPER[ch / 0x10]); result.append(HEX_CHARS_UPPER[ch % 0x10]); } } return result.toString(); } private static boolean isHexDigit(char c) { return (c >= 'A' && c <= 'F') || (c >= '0' && c <= '9'); } /** * Takes a ASCII digit in the range A-F0-9 and returns * the corresponding integer/ordinal value. * @param ch The hex digit. * @return The converted hex value as a byte. */ public static byte toBinaryFromHex(byte ch) { if (ch >= 'A' && ch <= 'F') return (byte) ((byte)10 + (byte) (ch - 'A')); // else return (byte) (ch - '0'); } public static byte [] toBytesBinary(String in) { // this may be bigger than we need, but let's be safe. byte [] b = new byte[in.length()]; int size = 0; for (int i = 0; i < in.length(); ++i) { char ch = in.charAt(i); if (ch == '\\' && in.length() > i+1 && in.charAt(i+1) == 'x') { // ok, take next 2 hex digits. char hd1 = in.charAt(i+2); char hd2 = in.charAt(i+3); // they need to be A-F0-9: if (!isHexDigit(hd1) || !isHexDigit(hd2)) { // bogus escape code, ignore: continue; } // turn hex ASCII digit -> number byte d = (byte) ((toBinaryFromHex((byte)hd1) << 4) + toBinaryFromHex((byte)hd2)); b[size++] = d; i += 3; // skip 3 } else { b[size++] = (byte) ch; } } // resize: byte [] b2 = new byte[size]; System.arraycopy(b, 0, b2, 0, size); return b2; } /** * Converts a string to a UTF-8 byte array. * @param s string * @return the byte array */ public static byte[] toBytes(String s) { return s.getBytes(UTF8_CHARSET); } /** * Convert a boolean to a byte array. True becomes -1 * and false becomes 0. * * @param b value * @return b encoded in a byte array. */ public static byte [] toBytes(final boolean b) { return new byte[] { b ? (byte) -1 : (byte) 0 }; } /** * Reverses {@link #toBytes(boolean)} * @param b array * @return True or false. */ public static boolean toBoolean(final byte [] b) { if (b.length != 1) { throw new IllegalArgumentException("Array has wrong size: " + b.length); } return b[0] != (byte) 0; } /** * Convert a long value to a byte array using big-endian. * * @param val value to convert * @return the byte array */ public static byte[] toBytes(long val) { byte [] b = new byte[8]; for (int i = 7; i > 0; i--) { b[i] = (byte) val; val >>>= 8; } b[0] = (byte) val; return b; } /** * @param left left operand * @param right right operand * @return 0 if equal, < 0 if left is less than right, etc. */ public static int compareTo(final byte [] left, final byte [] right) { return LexicographicalComparerHolder.BEST_COMPARER. compareTo(left, 0, left.length, right, 0, right.length); } /** * Lexicographically compare two arrays. * * @param buffer1 left operand * @param buffer2 right operand * @param offset1 Where to start comparing in the left buffer * @param offset2 Where to start comparing in the right buffer * @param length1 How much to compare from the left buffer * @param length2 How much to compare from the right buffer * @return 0 if equal, < 0 if left is less than right, etc. */ public static int compareTo(byte[] buffer1, int offset1, int length1, byte[] buffer2, int offset2, int length2) { return LexicographicalComparerHolder.BEST_COMPARER. compareTo(buffer1, offset1, length1, buffer2, offset2, length2); } interface Comparer { int compareTo( T buffer1, int offset1, int length1, T buffer2, int offset2, int length2 ); } @VisibleForTesting static Comparer lexicographicalComparerJavaImpl() { return LexicographicalComparerHolder.PureJavaComparer.INSTANCE; } /** * Provides a lexicographical comparer implementation; either a Java * implementation or a faster implementation based on {@link Unsafe}. * *

Uses reflection to gracefully fall back to the Java implementation if * {@code Unsafe} isn't available. */ @VisibleForTesting static class LexicographicalComparerHolder { static final String UNSAFE_COMPARER_NAME = LexicographicalComparerHolder.class.getName() + "$UnsafeComparer"; static final Comparer BEST_COMPARER = getBestComparer(); /** * Returns the Unsafe-using Comparer, or falls back to the pure-Java * implementation if unable to do so. */ static Comparer getBestComparer() { try { Class theClass = Class.forName(UNSAFE_COMPARER_NAME); // yes, UnsafeComparer does implement Comparer @SuppressWarnings("unchecked") Comparer comparer = (Comparer) theClass.getEnumConstants()[0]; return comparer; } catch (Throwable t) { // ensure we really catch *everything* return lexicographicalComparerJavaImpl(); } } enum PureJavaComparer implements Comparer { INSTANCE; @Override public int compareTo(byte[] buffer1, int offset1, int length1, byte[] buffer2, int offset2, int length2) { // Short circuit equal case if (buffer1 == buffer2 && offset1 == offset2 && length1 == length2) { return 0; } // Bring WritableComparator code local int end1 = offset1 + length1; int end2 = offset2 + length2; for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) { int a = (buffer1[i] & 0xff); int b = (buffer2[j] & 0xff); if (a != b) { return a - b; } } return length1 - length2; } } } /** * @param left left operand * @param right right operand * @return True if equal */ public static boolean equals(final byte [] left, final byte [] right) { // Could use Arrays.equals? //noinspection SimplifiableConditionalExpression if (left == right) return true; if (left == null || right == null) return false; if (left.length != right.length) return false; if (left.length == 0) return true; // Since we're often comparing adjacent sorted data, // it's usual to have equal arrays except for the very last byte // so check that first if (left[left.length - 1] != right[right.length - 1]) return false; return compareTo(left, right) == 0; } public static boolean equals(final byte[] left, int leftOffset, int leftLen, final byte[] right, int rightOffset, int rightLen) { // short circuit case if (left == right && leftOffset == rightOffset && leftLen == rightLen) { return true; } // different lengths fast check if (leftLen != rightLen) { return false; } if (leftLen == 0) { return true; } // Since we're often comparing adjacent sorted data, // it's usual to have equal arrays except for the very last byte // so check that first if (left[leftOffset + leftLen - 1] != right[rightOffset + rightLen - 1]) return false; return LexicographicalComparerHolder.BEST_COMPARER. compareTo(left, leftOffset, leftLen, right, rightOffset, rightLen) == 0; } /** * @param a left operand * @param buf right operand * @return True if equal */ public static boolean equals(byte[] a, ByteBuffer buf) { if (a == null) return buf == null; if (buf == null) return false; if (a.length != buf.remaining()) return false; // Thou shalt not modify the original byte buffer in what should be read only operations. ByteBuffer b = buf.duplicate(); for (byte anA : a) { if (anA != b.get()) { return false; } } return true; } /** * Return true if the byte array on the right is a prefix of the byte * array on the left. */ public static boolean startsWith(byte[] bytes, byte[] prefix) { return bytes != null && prefix != null && bytes.length >= prefix.length && LexicographicalComparerHolder.BEST_COMPARER. compareTo(bytes, 0, prefix.length, prefix, 0, prefix.length) == 0; } /** * @param a first third * @param b second third * @param c third third * @return New array made from a, b and c */ public static byte [] add(final byte [] a, final byte [] b, final byte [] c) { byte [] result = new byte[a.length + b.length + c.length]; System.arraycopy(a, 0, result, 0, a.length); System.arraycopy(b, 0, result, a.length, b.length); System.arraycopy(c, 0, result, a.length + b.length, c.length); return result; } /** * @param arrays all the arrays to concatenate together. * @return New array made from the concatenation of the given arrays. */ public static byte [] add(final byte [][] arrays) { int length = 0; for (int i = 0; i < arrays.length; i++) { length += arrays[i].length; } byte [] result = new byte[length]; int index = 0; for (int i = 0; i < arrays.length; i++) { System.arraycopy(arrays[i], 0, result, index, arrays[i].length); index += arrays[i].length; } return result; } /** * Split passed range. Expensive operation relatively. Uses BigInteger math. * Useful splitting ranges for MapReduce jobs. * @param a Beginning of range * @param b End of range * @param num Number of times to split range. Pass 1 if you want to split * the range in two; i.e. one split. * @return Array of dividing values */ /** * @param t operands * @return Array of byte arrays made from passed array of Text */ public static byte [][] toByteArrays(final String [] t) { byte [][] result = new byte[t.length][]; for (int i = 0; i < t.length; i++) { result[i] = BytesTools.toBytes(t[i]); } return result; } /** * @param t operands * @return Array of binary byte arrays made from passed array of binary strings */ public static byte[][] toBinaryByteArrays(final String[] t) { byte[][] result = new byte[t.length][]; for (int i = 0; i < t.length; i++) { result[i] = BytesTools.toBytesBinary(t[i]); } return result; } /** * @param column operand * @return A byte array of a byte array where first and only entry is * column */ public static byte [][] toByteArrays(final String column) { return toByteArrays(toBytes(column)); } /** * @param column operand * @return A byte array of a byte array where first and only entry is * column */ public static byte [][] toByteArrays(final byte [] column) { byte [][] result = new byte[1][]; result[0] = column; return result; } public static byte [] paddingInt(byte [] a){ if(a == null){ return null; } if (a.length==SIZEOF_INT){ return a; } byte [] b = new byte[SIZEOF_INT]; if (Platform.littleEndian){ for (int i = 0; i < SIZEOF_INT-a.length; i++) { b[i] = 0x00; } System.arraycopy(a, 0, b,SIZEOF_INT-a.length, a.length); }else { System.arraycopy(a, 0, b, 0, a.length); for (int i = a.length; i < SIZEOF_INT; i++) { b[i] = 0x00; } } return b; } public static byte [] paddingLong(byte [] a){ if(a == null){ return null; } if (a.length==SIZEOF_LONG){ return a; } byte [] b = new byte[SIZEOF_LONG]; if (Platform.littleEndian){ for (int i = 0; i < SIZEOF_LONG-a.length; i++) { b[i] = 0x00; } System.arraycopy(a, 0, b,SIZEOF_LONG-a.length, a.length); }else { System.arraycopy(a, 0, b, 0, a.length); for (int i = a.length; i < SIZEOF_LONG; i++) { b[i] = 0x00; } } return b; } public static byte [] paddingShort(byte [] a){ if(a == null){ return null; } if (a.length==SIZEOF_SHORT){ return a; } byte [] b = new byte[SIZEOF_SHORT]; if (Platform.littleEndian){ for (int i = 0; i < SIZEOF_SHORT-a.length; i++) { b[i] = 0x00; } System.arraycopy(a, 0, b, SIZEOF_SHORT-a.length, a.length); }else { System.arraycopy(a, 0, b, 0, a.length); for (int i = a.length; i < SIZEOF_SHORT; i++) { b[i] = 0x00; } } return b; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/JavaUtils.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.utils; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.Closeable; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; /** * Utility class, */ public class JavaUtils { private static final Logger logger = LoggerFactory.getLogger(JavaUtils.class); /** * Define a default value for driver memory here since this value is referenced across the code * base and nearly all files already use Utils.scala */ public static final long DEFAULT_DRIVER_MEM_MB = 1024; private static int MAX_DIR_CREATION_ATTEMPTS = 10; /** Closes the given object, ignoring IOExceptions. */ public static void closeQuietly(Closeable closeable) { try { if (closeable != null) { closeable.close(); } } catch (IOException e) { logger.error("IOException should not have been thrown.", e); } } /* * Delete a file or directory and its contents recursively. * Don't follow directories if they are symlinks. * Throws an exception if deletion is unsuccessful. */ public static void deleteRecursively(File file) throws IOException { if (file == null) { return; } if (file.isDirectory() && !isSymlink(file)) { IOException savedIOException = null; for (File child : listFilesSafely(file)) { try { deleteRecursively(child); } catch (IOException e) { // In case of multiple exceptions, only last one will be thrown savedIOException = e; } } if (savedIOException != null) { throw savedIOException; } } boolean deleted = file.delete(); // Delete can also fail if the file simply did not exist. if (!deleted && file.exists()) { throw new IOException("Failed to delete: " + file.getAbsolutePath()); } } private static File[] listFilesSafely(File file) throws IOException { if (file.exists()) { File[] files = file.listFiles(); if (files == null) { throw new IOException("Failed to list files for dir: " + file); } return files; } else { return new File[0]; } } private static boolean isSymlink(File file) throws IOException { Preconditions.checkNotNull(file); File fileInCanonicalDir = null; if (file.getParent() == null) { fileInCanonicalDir = file; } else { fileInCanonicalDir = new File(file.getParentFile().getCanonicalFile(), file.getName()); } return !fileInCanonicalDir.getCanonicalFile().equals(fileInCanonicalDir.getAbsoluteFile()); } private static final ImmutableMap timeSuffixes = ImmutableMap.builder() .put("us", TimeUnit.MICROSECONDS) .put("ms", TimeUnit.MILLISECONDS) .put("s", TimeUnit.SECONDS) .put("m", TimeUnit.MINUTES) .put("min", TimeUnit.MINUTES) .put("h", TimeUnit.HOURS) .put("d", TimeUnit.DAYS) .build(); private static final ImmutableMap byteSuffixes = ImmutableMap.builder() .put("b", ByteUnit.BYTE) .put("k", ByteUnit.KiB) .put("kb", ByteUnit.KiB) .put("m", ByteUnit.MiB) .put("mb", ByteUnit.MiB) .put("g", ByteUnit.GiB) .put("gb", ByteUnit.GiB) .put("t", ByteUnit.TiB) .put("tb", ByteUnit.TiB) .put("p", ByteUnit.PiB) .put("pb", ByteUnit.PiB) .build(); /** * Convert a passed time string (e.g. 50s, 100ms, or 250us) to a time count in the given unit. * The unit is also considered the default if the given string does not specify a unit. */ public static long timeStringAs(String str, TimeUnit unit) { String lower = str.toLowerCase().trim(); try { Matcher m = Pattern.compile("(-?[0-9]+)([a-z]+)?").matcher(lower); if (!m.matches()) { throw new NumberFormatException("Failed to parse time string: " + str); } long val = Long.parseLong(m.group(1)); String suffix = m.group(2); // Check for invalid suffixes if (suffix != null && !timeSuffixes.containsKey(suffix)) { throw new NumberFormatException("Invalid suffix: \"" + suffix + "\""); } // If suffix is valid use that, otherwise none was provided and use the default passed return unit.convert(val, suffix != null ? timeSuffixes.get(suffix) : unit); } catch (NumberFormatException e) { String timeError = "Time must be specified as seconds (s), " + "milliseconds (ms), microseconds (us), minutes (m or min), hour (h), or day (d). " + "E.g. 50s, 100ms, or 250us."; throw new NumberFormatException(timeError + "\n" + e.getMessage()); } } /** * Convert a time parameter such as (50s, 100ms, or 250us) to milliseconds for internal use. If * no suffix is provided, the passed number is assumed to be in ms. */ public static long timeStringAsMs(String str) { return timeStringAs(str, TimeUnit.MILLISECONDS); } /** * Convert a time parameter such as (50s, 100ms, or 250us) to seconds for internal use. If * no suffix is provided, the passed number is assumed to be in seconds. */ public static long timeStringAsSec(String str) { return timeStringAs(str, TimeUnit.SECONDS); } /** * Convert a passed byte string (e.g. 50b, 100kb, or 250mb) to the given. If no suffix is * provided, a direct conversion to the provided unit is attempted. */ public static long byteStringAs(String str, ByteUnit unit) { String lower = str.toLowerCase().trim(); try { Matcher m = Pattern.compile("([0-9]+)([a-z]+)?").matcher(lower); Matcher fractionMatcher = Pattern.compile("([0-9]+\\.[0-9]+)([a-z]+)?").matcher(lower); if (m.matches()) { long val = Long.parseLong(m.group(1)); String suffix = m.group(2); // Check for invalid suffixes if (suffix != null && !byteSuffixes.containsKey(suffix)) { throw new NumberFormatException("Invalid suffix: \"" + suffix + "\""); } // If suffix is valid use that, otherwise none was provided and use the default passed return unit.convertFrom(val, suffix != null ? byteSuffixes.get(suffix) : unit); } else if (fractionMatcher.matches()) { throw new NumberFormatException("Fractional values are not supported. Input was: " + fractionMatcher.group(1)); } else { throw new NumberFormatException("Failed to parse byte string: " + str); } } catch (NumberFormatException e) { String byteError = "Size must be specified as bytes (b), " + "kibibytes (k), mebibytes (m), gibibytes (g), tebibytes (t), or pebibytes(p). " + "E.g. 50b, 100k, or 250m."; throw new NumberFormatException(byteError + "\n" + e.getMessage()); } } /** * Convert a passed byte string (e.g. 50b, 100k, or 250m) to bytes for * internal use. * * If no suffix is provided, the passed number is assumed to be in bytes. */ public static long byteStringAsBytes(String str) { return byteStringAs(str, ByteUnit.BYTE); } /** * Convert a passed byte string (e.g. 50b, 100k, or 250m) to kibibytes for * internal use. * * If no suffix is provided, the passed number is assumed to be in kibibytes. */ public static long byteStringAsKb(String str) { return byteStringAs(str, ByteUnit.KiB); } /** * Convert a passed byte string (e.g. 50b, 100k, or 250m) to mebibytes for * internal use. * * If no suffix is provided, the passed number is assumed to be in mebibytes. */ public static long byteStringAsMb(String str) { return byteStringAs(str, ByteUnit.MiB); } /** * Convert a passed byte string (e.g. 50b, 100k, or 250m) to gibibytes for * internal use. * * If no suffix is provided, the passed number is assumed to be in gibibytes. */ public static long byteStringAsGb(String str) { return byteStringAs(str, ByteUnit.GiB); } public static String bytesToString(long size) { long TB = 1L << 40; long GB = 1L << 30; long MB = 1L << 20; long KB = 1L << 10; double value = 0; String unit = null; if (size >= 2*TB) { value = size/TB; unit = "TB"; } else if (size >= 2*GB) { value = size/GB; unit = "GB"; } else if (size >= 2*MB) { value = size/MB; unit = "MB"; } else if (size >= 2*KB) { value = size/KB; unit = "KB"; } else { value = size; unit = "B"; } return value + " " + unit; } public static String bytesToString2(long size) { long TB = 1L << 40; long GB = 1L << 30; long MB = 1L << 20; long KB = 1L << 10; long value = 0; String unit = null; if (size >= 2*TB) { value = (size/TB); unit = "TB"; } else if (size >= 2*GB) { value = (size/GB); unit = "GB"; } else if (size >= 2*MB) { value = (size/MB); unit = "MB"; } else if (size >= 2*KB) { value = (size/KB); unit = "KB"; } else { value = size; unit = "B"; } return value + unit; } public static File createDirectory(String rootDir, String blockmgr) throws IOException { int attempts = 0; int maxAttempts = MAX_DIR_CREATION_ATTEMPTS; File dir = null; while (dir == null) { attempts += 1; if (attempts > maxAttempts) { throw new IOException("Failed to create a temp directory (under " + rootDir + ") after " + maxAttempts + " attempts!"); } try { dir = new File(rootDir, blockmgr + "-" + UUID.randomUUID().toString()); if (dir.exists() || !dir.mkdirs()) { dir = null; } } catch (Exception e) { logger.error(e.getMessage()); } } return dir.getCanonicalFile(); } /* Calculates 'x' modulo 'mod', takes to consideration sign of x, * i.e. if 'x' is negative, than 'x' % 'mod' is negative too * so function return (x % mod) + mod in that case. */ public static int nonNegativeMod(int x,int mod) { int rawMod = x % mod; int temp; if (rawMod < 0) temp= mod ; else temp =0; return (rawMod + temp); } public static int nonNegativeHash(Object obj) { // Required ? if (obj == null) return 0; int hash = obj.hashCode(); // math.abs fails for Int.MinValue int hashAbs = 0; if (Integer.MAX_VALUE!= hash && Integer.MIN_VALUE != hash) hashAbs = Math.abs(hash); else hashAbs = 0; // Nothing else to guard against ? return hashAbs; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/MycatPropertyConf.java ================================================ package io.mycat.memory.unsafe.utils; import java.util.concurrent.ConcurrentHashMap; /** * Created by zagnix on 2016/6/2. */ public class MycatPropertyConf { private ConcurrentHashMap settings = new ConcurrentHashMap(); public MycatPropertyConf(){ } /** Set a configuration variable. */ public MycatPropertyConf set(String key, String value) { set(key, value, false); return this; } public MycatPropertyConf set(String key, String value, boolean silent){ if (key == null) { throw new NullPointerException("null key"); } if (value == null) { throw new NullPointerException("null value for " + key); } if (!silent) { } settings.put(key, value); return this; } public long getSizeAsBytes(String s, long i) { String value = (String) settings.get(s); if(value !=null){ return byteStringAsBytes(value); } return i; } public long getSizeAsBytes(String s, String defaultValue) { String value = (String) settings.get(s); if(value !=null){ return byteStringAsBytes(value); } return byteStringAsBytes(defaultValue); } public double getDouble(String s, double v) { return v; } public boolean getBoolean(String s, boolean b) { String value = (String) settings.get(s); if(value !=null){ if(value.equals("true")){ return true; }else{ return false; } } return b; } public long getLong(String s, long l) { return l; } public boolean contains(String s) { return true; } public int getInt(String s, int i) { return i; } /** * Convert a passed byte string (e.g. 50b, 100k, or 250m) to bytes for internal use. * * If no suffix is provided, the passed number is assumed to be in bytes. */ public Long byteStringAsBytes(String str) { return JavaUtils.byteStringAsBytes(str); } /** * Convert a passed byte string (e.g. 50b, 100k, or 250m) to kibibytes for internal use. * * If no suffix is provided, the passed number is assumed to be in kibibytes. */ public Long byteStringAsKb(String str){ return JavaUtils.byteStringAsKb(str); } /** * Convert a passed byte string (e.g. 50b, 100k, or 250m) to mebibytes for internal use. * * If no suffix is provided, the passed number is assumed to be in mebibytes. */ public Long byteStringAsMb(String str) { return JavaUtils.byteStringAsMb(str); } /** * Convert a passed byte string (e.g. 50b, 100k, or 250m, 500g) to gibibytes for internal use. * * If no suffix is provided, the passed number is assumed to be in gibibytes. */ public Long byteStringAsGb(String str) { return JavaUtils.byteStringAsGb(str); } /** * Convert a Java memory parameter passed to -Xmx (such as 300m or 1g) to a number of mebibytes. */ public int memoryStringToMb(String str){ // Convert to bytes, rather than directly to MB, because when no units are specified the unit // is assumed to be bytes return (int) (JavaUtils.byteStringAsBytes(str) / 1024 / 1024); } public String getString(String s, String defaultValue) { String value = (String) settings.get(s); if(value !=null){ return value; } return defaultValue; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/AbstractScalaRowIterator.java ================================================ package io.mycat.memory.unsafe.utils.sort; import java.util.Iterator; /** * Created by zagnix 2016/6/6. */ public class AbstractScalaRowIterator implements Iterator { @Override public boolean hasNext() { return false; } @Override public T next() { return null; } @Override public void remove() { } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/PrefixComparator.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.utils.sort; /** * Compares 8-byte key prefixes in prefix sort. Subclasses may implement type-specific * comparisons, such as lexicographic comparison for strings. */ public abstract class PrefixComparator { public abstract int compare(long prefix1, long prefix2); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/PrefixComparators.java ================================================ package io.mycat.memory.unsafe.utils.sort; import com.google.common.primitives.UnsignedLongs; import io.mycat.memory.unsafe.types.ByteArray; public class PrefixComparators { private PrefixComparators() {} public static final PrefixComparator STRING = new UnsignedPrefixComparator(); public static final PrefixComparator STRING_DESC = new UnsignedPrefixComparatorDesc(); public static final PrefixComparator BINARY = new UnsignedPrefixComparator(); public static final PrefixComparator BINARY_DESC = new UnsignedPrefixComparatorDesc(); public static final PrefixComparator LONG = new SignedPrefixComparator(); public static final PrefixComparator LONG_DESC = new SignedPrefixComparatorDesc(); public static final PrefixComparator DOUBLE = new UnsignedPrefixComparator(); public static final PrefixComparator DOUBLE_DESC = new UnsignedPrefixComparatorDesc(); public static final PrefixComparator RadixSortDemo = new RadixSortDemo(); public static final class BinaryPrefixComparator { public static long computePrefix(byte[] bytes) { return ByteArray.getPrefix(bytes); } } public static final class DoublePrefixComparator { /** * Converts the double into a value that compares correctly as an unsigned long. For more * details see http://stereopsis.com/radix.html. */ public static long computePrefix(double value) { // Java's doubleToLongBits already canonicalizes all NaN values to the smallest possible // positive NaN, so there's nothing special we need to do for NaNs. long bits = Double.doubleToLongBits(value); // Negative floats compare backwards due to their sign-magnitude representation, so flip // all the bits in this case. long mask = -(bits >>> 63) | 0x8000000000000000L; return bits ^ mask; } } /** * Provides radix sort parameters. Comparators implementing this also are indicating that the * ordering they define is compatible with radix sort. */ public abstract static class RadixSortSupport extends PrefixComparator { /** @return Whether the sort should be descending in binary sort order. */ public abstract boolean sortDescending(); /** @return Whether the sort should take into account the sign bit. */ public abstract boolean sortSigned(); } public static final class RadixSortDemo extends PrefixComparators.RadixSortSupport{ @Override public boolean sortDescending() { return false; } @Override public boolean sortSigned() { return false; } @Override public int compare(long prefix1, long prefix2) { return PrefixComparators.BINARY.compare(prefix1 & 0xffffff0000L, prefix1 & 0xffffff0000L); } } // // Standard prefix comparator implementations // public static final class UnsignedPrefixComparator extends RadixSortSupport { @Override public boolean sortDescending() { return false; } @Override public boolean sortSigned() { return false; } @Override public int compare(long aPrefix, long bPrefix) { return UnsignedLongs.compare(aPrefix, bPrefix); } } public static final class UnsignedPrefixComparatorDesc extends RadixSortSupport { @Override public boolean sortDescending() { return true; } @Override public boolean sortSigned() { return false; } @Override public int compare(long bPrefix, long aPrefix) { return UnsignedLongs.compare(aPrefix, bPrefix); } } public static final class SignedPrefixComparator extends RadixSortSupport { @Override public boolean sortDescending() { return false; } @Override public boolean sortSigned() { return true; } @Override public int compare(long a, long b) { return (a < b) ? -1 : (a > b) ? 1 : 0; } } public static final class SignedPrefixComparatorDesc extends RadixSortSupport { @Override public boolean sortDescending() { return true; } @Override public boolean sortSigned() { return true; } @Override public int compare(long b, long a) { return (a < b) ? -1 : (a > b) ? 1 : 0; } } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/RadixSort.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.utils.sort; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.array.LongArray; public class RadixSort { /** * Sorts a given array of longs using least-significant-digit radix sort. This routine assumes * you have extra space at the end of the array at least equal to the number of records. The * sort is destructive and may relocate the data positioned within the array. * * @param array array of long elements followed by at least that many empty slots. * @param numRecords number of data records in the array. * @param startByteIndex the first byte (in range [0, 7]) to sort each long by, counting from the * least significant byte. * @param endByteIndex the last byte (in range [0, 7]) to sort each long by, counting from the * least significant byte. Must be greater than startByteIndex. * @param desc whether this is a descending (binary-order) sort. * @param signed whether this is a signed (two's complement) sort. * * @return The starting index of the sorted data within the given array. We return this instead * of always copying the data back to position zero for efficiency. */ public static int sort( LongArray array, int numRecords, int startByteIndex, int endByteIndex, boolean desc, boolean signed) { assert startByteIndex >= 0 : "startByteIndex (" + startByteIndex + ") should >= 0"; assert endByteIndex <= 7 : "endByteIndex (" + endByteIndex + ") should <= 7"; assert endByteIndex > startByteIndex; assert numRecords * 2 <= array.size(); int inIndex = 0; int outIndex = numRecords; if (numRecords > 0) { long[][] counts = getCounts(array, numRecords, startByteIndex, endByteIndex); for (int i = startByteIndex; i <= endByteIndex; i++) { if (counts[i] != null) { sortAtByte( array, numRecords, counts[i], i, inIndex, outIndex, desc, signed && i == endByteIndex); int tmp = inIndex; inIndex = outIndex; outIndex = tmp; } } } return inIndex; } /** * Performs a partial sort by copying data into destination offsets for each byte value at the * specified byte offset. * * @param array array to partially sort. * @param numRecords number of data records in the array. * @param counts counts for each byte value. This routine destructively modifies this array. * @param byteIdx the byte in a long to sort at, counting from the least significant byte. * @param inIndex the starting index in the array where input data is located. * @param outIndex the starting index where sorted output data should be written. * @param desc whether this is a descending (binary-order) sort. * @param signed whether this is a signed (two's complement) sort (only applies to last byte). */ private static void sortAtByte( LongArray array, int numRecords, long[] counts, int byteIdx, int inIndex, int outIndex, boolean desc, boolean signed) { assert counts.length == 256; long[] offsets = transformCountsToOffsets( counts, numRecords, array.getBaseOffset() + outIndex * 8, 8, desc, signed); Object baseObject = array.getBaseObject(); long baseOffset = array.getBaseOffset() + inIndex * 8; long maxOffset = baseOffset + numRecords * 8; for (long offset = baseOffset; offset < maxOffset; offset += 8) { long value = Platform.getLong(baseObject, offset); int bucket = (int)((value >>> (byteIdx * 8)) & 0xff); Platform.putLong(baseObject, offsets[bucket], value); offsets[bucket] += 8; } } /** * Computes a value histogram for each byte in the given array. * * @param array array to count records in. * @param numRecords number of data records in the array. * @param startByteIndex the first byte to compute counts for (the prior are skipped). * @param endByteIndex the last byte to compute counts for. * * @return an array of eight 256-byte count arrays, one for each byte starting from the least * significant byte. If the byte does not need sorting the array will be null. */ private static long[][] getCounts( LongArray array, int numRecords, int startByteIndex, int endByteIndex) { long[][] counts = new long[8][]; // Optimization: do a fast pre-pass to determine which byte indices we can skip for sorting. // If all the byte values at a particular index are the same we don't need to count it. long bitwiseMax = 0; long bitwiseMin = -1L; long maxOffset = array.getBaseOffset() + numRecords * 8; Object baseObject = array.getBaseObject(); for (long offset = array.getBaseOffset(); offset < maxOffset; offset += 8) { long value = Platform.getLong(baseObject, offset); bitwiseMax |= value; bitwiseMin &= value; } long bitsChanged = bitwiseMin ^ bitwiseMax; // Compute counts for each byte index. for (int i = startByteIndex; i <= endByteIndex; i++) { if (((bitsChanged >>> (i * 8)) & 0xff) != 0) { counts[i] = new long[256]; // TODO(ekl) consider computing all the counts in one pass. for (long offset = array.getBaseOffset(); offset < maxOffset; offset += 8) { counts[i][(int)((Platform.getLong(baseObject, offset) >>> (i * 8)) & 0xff)]++; } } } return counts; } /** * Transforms counts into the proper unsafe output offsets for the sort type. * * @param counts counts for each byte value. This routine destructively modifies this array. * @param numRecords number of data records in the original data array. * @param outputOffset output offset in bytes from the base array object. * @param bytesPerRecord size of each record (8 for plain sort, 16 for key-prefix sort). * @param desc whether this is a descending (binary-order) sort. * @param signed whether this is a signed (two's complement) sort. * * @return the input counts array. */ private static long[] transformCountsToOffsets( long[] counts, int numRecords, long outputOffset, int bytesPerRecord, boolean desc, boolean signed) { assert counts.length == 256; int start = signed ? 128 : 0; // output the negative records first (values 129-255). if (desc) { int pos = numRecords; for (int i = start; i < start + 256; i++) { pos -= counts[i & 0xff]; counts[i & 0xff] = outputOffset + pos * bytesPerRecord; } } else { int pos = 0; for (int i = start; i < start + 256; i++) { long tmp = counts[i & 0xff]; counts[i & 0xff] = outputOffset + pos * bytesPerRecord; pos += tmp; } } return counts; } /** * Specialization of sort() for key-prefix arrays. In this type of array, each record consists * of two longs, only the second of which is sorted on. */ public static int sortKeyPrefixArray( LongArray array, int numRecords, int startByteIndex, int endByteIndex, boolean desc, boolean signed) { assert startByteIndex >= 0 : "startByteIndex (" + startByteIndex + ") should >= 0"; assert endByteIndex <= 7 : "endByteIndex (" + endByteIndex + ") should <= 7"; assert endByteIndex > startByteIndex; assert numRecords * 4 <= array.size(); int inIndex = 0; int outIndex = numRecords * 2; if (numRecords > 0) { long[][] counts = getKeyPrefixArrayCounts(array, numRecords, startByteIndex, endByteIndex); for (int i = startByteIndex; i <= endByteIndex; i++) { if (counts[i] != null) { sortKeyPrefixArrayAtByte( array, numRecords, counts[i], i, inIndex, outIndex, desc, signed && i == endByteIndex); int tmp = inIndex; inIndex = outIndex; outIndex = tmp; } } } return inIndex; } /** * Specialization of getCounts() for key-prefix arrays. We could probably combine this with * getCounts with some added parameters but that seems to hurt in benchmarks. */ private static long[][] getKeyPrefixArrayCounts( LongArray array, int numRecords, int startByteIndex, int endByteIndex) { long[][] counts = new long[8][]; long bitwiseMax = 0; long bitwiseMin = -1L; long limit = array.getBaseOffset() + numRecords * 16; Object baseObject = array.getBaseObject(); for (long offset = array.getBaseOffset(); offset < limit; offset += 16) { long value = Platform.getLong(baseObject, offset + 8); bitwiseMax |= value; bitwiseMin &= value; } long bitsChanged = bitwiseMin ^ bitwiseMax; for (int i = startByteIndex; i <= endByteIndex; i++) { if (((bitsChanged >>> (i * 8)) & 0xff) != 0) { counts[i] = new long[256]; for (long offset = array.getBaseOffset(); offset < limit; offset += 16) { counts[i][(int)((Platform.getLong(baseObject, offset + 8) >>> (i * 8)) & 0xff)]++; } } } return counts; } /** * Specialization of sortAtByte() for key-prefix arrays. */ private static void sortKeyPrefixArrayAtByte( LongArray array, int numRecords, long[] counts, int byteIdx, int inIndex, int outIndex, boolean desc, boolean signed) { assert counts.length == 256; long[] offsets = transformCountsToOffsets( counts, numRecords, array.getBaseOffset() + outIndex * 8, 16, desc, signed); Object baseObject = array.getBaseObject(); long baseOffset = array.getBaseOffset() + inIndex * 8; long maxOffset = baseOffset + numRecords * 16; for (long offset = baseOffset; offset < maxOffset; offset += 16) { long key = Platform.getLong(baseObject, offset); long prefix = Platform.getLong(baseObject, offset + 8); int bucket = (int)((prefix >>> (byteIdx * 8)) & 0xff); long dest = offsets[bucket]; Platform.putLong(baseObject, dest, key); Platform.putLong(baseObject, dest + 8, prefix); offsets[bucket] += 16; } } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/RecordComparator.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.utils.sort; /** * Compares records for ordering. In cases where the entire sorting key can fit in the 8-byte * prefix, this may simply return 0. */ public abstract class RecordComparator { /** * Compare two records for order. * * @return a negative integer, zero, or a positive integer as the first record is less than, * equal to, or greater than the second. */ public abstract int compare( Object leftBaseObject, long leftBaseOffset, Object rightBaseObject, long rightBaseOffset); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/RecordPointerAndKeyPrefix.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.utils.sort; public final class RecordPointerAndKeyPrefix { /** * A pointer to a record; see {@link io.mycat.memory.unsafe.memory} for a * description of how these addresses are encoded. */ public long recordPointer; /** * A key prefix, for use in comparisons. */ public long keyPrefix; } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/RowPrefixComputer.java ================================================ package io.mycat.memory.unsafe.utils.sort; import io.mycat.memory.unsafe.row.StructType; import io.mycat.memory.unsafe.row.UnsafeRow; import io.mycat.memory.unsafe.utils.BytesTools; import io.mycat.sqlengine.mpp.ColMeta; import io.mycat.sqlengine.mpp.OrderCol; import javax.annotation.Nonnull; import java.io.UnsupportedEncodingException; /** * Created by zagnix on 2016/6/20. */ public class RowPrefixComputer extends UnsafeExternalRowSorter.PrefixComputer { @Nonnull private final StructType schema; private final ColMeta colMeta; public RowPrefixComputer(StructType schema){ this.schema = schema; /** * 通过计算得到排序关键词的第一个在行的索引下标 */ OrderCol[] orderCols = schema.getOrderCols(); if (orderCols != null && orderCols.length > 0){ this.colMeta = orderCols[0].colMeta; }else { this.colMeta = null; } } protected long computePrefix(UnsafeRow row) throws UnsupportedEncodingException { if(this.colMeta == null){ return 0; } int orderIndexType = colMeta.colType; byte[] rowIndexElem = null; if(!row.isNullAt(colMeta.colIndex)) { rowIndexElem = row.getBinary(colMeta.colIndex); /** * 这里注意一下，order by 排序的第一个字段 */ switch (orderIndexType) { case ColMeta.COL_TYPE_INT: case ColMeta.COL_TYPE_LONG: case ColMeta.COL_TYPE_INT24: return BytesTools.getInt(rowIndexElem); case ColMeta.COL_TYPE_SHORT: return BytesTools.getShort(rowIndexElem); case ColMeta.COL_TYPE_LONGLONG: return BytesTools.getLong(rowIndexElem); case ColMeta.COL_TYPE_FLOAT: return PrefixComparators.DoublePrefixComparator. computePrefix(BytesTools.getFloat(rowIndexElem)); case ColMeta.COL_TYPE_DOUBLE: case ColMeta.COL_TYPE_DECIMAL: case ColMeta.COL_TYPE_NEWDECIMAL: return PrefixComparators.DoublePrefixComparator. computePrefix(BytesTools.getDouble(rowIndexElem)); case ColMeta.COL_TYPE_DATE: case ColMeta.COL_TYPE_TIMSTAMP: case ColMeta.COL_TYPE_TIME: case ColMeta.COL_TYPE_YEAR: case ColMeta.COL_TYPE_DATETIME: case ColMeta.COL_TYPE_NEWDATE: case ColMeta.COL_TYPE_BIT: case ColMeta.COL_TYPE_VAR_STRING: case ColMeta.COL_TYPE_STRING: // ENUM和SET类型都是字符串，按字符串处理 case ColMeta.COL_TYPE_ENUM: case ColMeta.COL_TYPE_SET: return PrefixComparators.BinaryPrefixComparator.computePrefix(rowIndexElem); //BLOB相关类型和GEOMETRY类型不支持排序，略掉 } } else { rowIndexElem = new byte[1]; rowIndexElem[0] = UnsafeRow.NULL_MARK; return PrefixComparators.BinaryPrefixComparator.computePrefix(rowIndexElem); } return 0; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/SortDataFormat.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.utils.sort; /** * Abstraction for sorting an arbitrary input buffer of data. This interface requires determining * the sort key for a given element index, as well as swapping elements and moving data from one * buffer to another. * * Example format: an array of numbers, where each element is also the key. * See [[KVArraySortDataFormat]] for a more exciting format. * * Note: Declaring and instantiating multiple subclasses of this class would prevent JIT inlining * overridden methods and hence decrease the shuffle performance. * * @tparam K Type of the sort key of each element * @tparam Buffer Internal data structure used by a particular format (e.g., Array[Int]). */ // TODO: Making Buffer a real trait would be a better abstraction, but adds some complexity. public abstract class SortDataFormat { /** * Creates a new mutable key for reuse. This should be implemented if you want to override * [[getKey(Buffer, Int, K)]]. */ public abstract K newKey(); /** Return the sort key for the element at the given index. */ protected abstract K getKey(Buffer data, int pos); /** * Returns the sort key for the element at the given index and reuse the input key if possible. * The default implementation ignores the reuse parameter and invokes [[getKey(Buffer, Int]]. * If you want to override this method, you must implement [[newKey()]]. */ protected K getKey(Buffer data, int pos, K reuse) { return getKey(data, pos); } /** Swap two elements. */ protected abstract void swap(Buffer data, int pos0,int pos1); /** Copy a single element from src(srcPos) to dst(dstPos). */ protected abstract void copyElement(Buffer src, int srcPos,Buffer dst ,int dstPos); /** * Copy a range of elements starting at src(srcPos) to dst, starting at dstPos. * Overlapping ranges are allowed. */ protected abstract void copyRange(Buffer src, int srcPos,Buffer dst, int dstPos, int length); /** * Allocates a Buffer that can hold up to 'length' elements. * All elements of the buffer should be considered invalid until data is explicitly copied in. */ protected abstract Buffer allocate(int length); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/SortPrefixUtils.java ================================================ package io.mycat.memory.unsafe.utils.sort; import io.mycat.memory.unsafe.row.StructType; /** * Created by zagnix on 2016/6/6. */ public final class SortPrefixUtils { public static boolean canSortFullyWithPrefix(long apply) { return true; } public static PrefixComparator getPrefixComparator(StructType keySchema) { return null; } public static UnsafeExternalRowSorter.PrefixComputer createPrefixGenerator(StructType keySchema) { return null; } /** * A dummy prefix comparator which always claims that prefixes are equal. This is used in cases * where we don't know how to generate or compare prefixes for a SortOrder. */ private class NoOpPrefixComparator extends PrefixComparator { @Override public int compare(long prefix1, long prefix2) { return 0; } } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/Sorter.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.utils.sort; import java.util.Comparator; /** * A simple wrapper over the Java implementation [[TimSort]]. * * The Java implementation is package private, and hence it cannot be called outside package * org.opencloudb.memory.unsafe.utils.sort. This is a simple wrapper of it that is available to mycat. */ public class Sorter { private TimSort timSort = null; public Sorter(SortDataFormat s){ timSort = new TimSort(s); } /** * Sorts the input buffer within range [lo, hi). */ public void sort(Buffer a, int lo, int hi, Comparator c) { timSort.sort(a, lo, hi, c); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/TestSorter.java ================================================ package io.mycat.memory.unsafe.utils.sort; import io.mycat.memory.MyCatMemory; import io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager; import io.mycat.memory.unsafe.memory.mm.MemoryManager; import io.mycat.memory.unsafe.row.BufferHolder; import io.mycat.memory.unsafe.row.StructType; import io.mycat.memory.unsafe.row.UnsafeRow; import io.mycat.memory.unsafe.row.UnsafeRowWriter; import io.mycat.memory.unsafe.utils.MycatPropertyConf; import io.mycat.sqlengine.mpp.ColMeta; import io.mycat.sqlengine.mpp.OrderCol; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.*; import java.util.concurrent.CountDownLatch; /** * Created by zagnix on 16-7-9. */ public class TestSorter implements Runnable { private static final Logger logger = LoggerFactory.getLogger(TestSorter.class); private static final int TEST_SIZE = 1000000; private static int TASK_SIZE = 100; private static CountDownLatch countDownLatch = new CountDownLatch(100); public void runSorter( MyCatMemory myCatMemory, MemoryManager memoryManager, MycatPropertyConf conf) throws NoSuchFieldException, IllegalAccessException, IOException { DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId()); /** * 1.schema ,模拟一个field字段值 * */ int fieldCount = 3; ColMeta colMeta = null; Map colMetaMap = new HashMap(fieldCount); colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING); colMetaMap.put("id", colMeta); colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING); colMetaMap.put("name", colMeta); colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING); colMetaMap.put("age", colMeta); OrderCol[] orderCols = new OrderCol[1]; OrderCol orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_ASC); orderCols[0] = orderCol; /** * 2 .PrefixComputer */ StructType schema = new StructType(colMetaMap, fieldCount); schema.setOrderCols(orderCols); UnsafeExternalRowSorter.PrefixComputer prefixComputer = new RowPrefixComputer(schema); /** * 3 .PrefixComparator 默认是ASC，可以选择DESC */ final PrefixComparator prefixComparator = PrefixComparators.LONG; UnsafeExternalRowSorter sorter = new UnsafeExternalRowSorter(dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize","1m"), true, /**使用基数排序？true or false*/ true); UnsafeRow unsafeRow; BufferHolder bufferHolder; UnsafeRowWriter unsafeRowWriter; String line = "testUnsafeRow"; final Random rand = new Random(42); for (int i = 0; i < TEST_SIZE; i++) { unsafeRow = new UnsafeRow(3); bufferHolder = new BufferHolder(unsafeRow); unsafeRowWriter = new UnsafeRowWriter(bufferHolder,3); bufferHolder.reset(); String key = getRandomString(rand.nextInt(300)+100); unsafeRowWriter.write(0,key.getBytes()); unsafeRowWriter.write(1, line.getBytes()); unsafeRowWriter.write(2, ("35" + 1).getBytes()); unsafeRow.setTotalSize(bufferHolder.totalSize()); sorter.insertRow(unsafeRow); } Iterator iter = sorter.sort(); UnsafeRow row = null; int indexprint = 0; while (iter.hasNext()) { row = iter.next(); indexprint++; } sorter.cleanupResources(); countDownLatch.countDown(); System.out.println("Thread ID :" + Thread.currentThread().getId() + "Index : " + indexprint); } public static String getRandomString(int length) { //length表示生成字符串的长度 String base = "abcdefghijklmnopqrstuvwxyz0123456789"; Random random = new Random(); StringBuffer sb = new StringBuffer(); for (int i = 0; i < length; i++) { int number = random.nextInt(base.length()); sb.append(base.charAt(number)); } return sb.toString(); } final MyCatMemory myCatMemory ; final MemoryManager memoryManager; final MycatPropertyConf conf; public TestSorter( MyCatMemory myCatMemory, MemoryManager memoryManager,MycatPropertyConf conf) throws NoSuchFieldException, IllegalAccessException { this.myCatMemory = myCatMemory; this.memoryManager = memoryManager; this.conf = conf; } @Override public void run() { try { runSorter(myCatMemory,memoryManager,conf); } catch (NoSuchFieldException e) { logger.error(e.getMessage()); } catch (IllegalAccessException e) { logger.error(e.getMessage()); } catch (IOException e) { logger.error(e.getMessage()); } } public static void main(String[] args) throws Exception { MyCatMemory myCatMemory ; MemoryManager memoryManager; MycatPropertyConf conf; myCatMemory = new MyCatMemory(); memoryManager = myCatMemory.getResultMergeMemoryManager(); conf = myCatMemory.getConf(); for (int i = 0; i < TASK_SIZE; i++) { Thread thread = new Thread(new TestSorter(myCatMemory,memoryManager,conf)); thread.start(); } while (countDownLatch.getCount() != 0){ System.err.println("count ========================>" + countDownLatch.getCount()); Thread.sleep(1000); } System.err.println(TASK_SIZE + " tasks sorter finished ok !!!!!!!!!"); System.exit(1); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/TimSort.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * Based on TimSort.java from the Android Open Source Project * * Copyright (C) 2008 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.utils.sort; import java.util.Comparator; /** * A port of the Android TimSort class, which utilizes a "stable, adaptive, iterative mergesort." * See the method comment on sort() for more details. * * This has been kept in Java with the original style in order to match very closely with the * Android source code, and thus be easy to verify correctness. The class is package private. We put * a simple Scala wrapper {@link io.mycat.memory.unsafe.utils.sort.Sorter}, which is available to * The purpose of the port is to generalize the interface to the sort to accept input data formats * besides simple arrays where every element is sorted individually. For instance, the AppendOnlyMap * uses this to sort an Array with alternating elements of the form [key, value, key, value]. * This generalization comes with minimal overhead -- see SortDataFormat for more information. * * We allow key reuse to prevent creating many key objects -- see SortDataFormat. * * @see io.mycat.memory.unsafe.utils.sort.SortDataFormat * @see io.mycat.memory.unsafe.utils.sort.Sorter */ class TimSort { /** * This is the minimum sized sequence that will be merged. Shorter * sequences will be lengthened by calling binarySort. If the entire * array is less than this length, no merges will be performed. * * This constant should be a power of two. It was 64 in Tim Peter's C * implementation, but 32 was empirically determined to work better in * this implementation. In the unlikely event that you set this constant * to be a number that's not a power of two, you'll need to change the * minRunLength computation. * * If you decrease this constant, you must change the stackLen * computation in the TimSort constructor, or you risk an * ArrayOutOfBounds exception. See listsort.txt for a discussion * of the minimum stack length required as a function of the length * of the array being sorted and the minimum merge sequence length. */ private static final int MIN_MERGE = 32; private final SortDataFormat s; public TimSort(SortDataFormat sortDataFormat) { this.s = sortDataFormat; } /** * A stable, adaptive, iterative mergesort that requires far fewer than * n lg(n) comparisons when running on partially sorted arrays, while * offering performance comparable to a traditional mergesort when run * on random arrays. Like all proper mergesorts, this sort is stable and * runs O(n log n) time (worst case). In the worst case, this sort requires * temporary storage space for n/2 object references; in the best case, * it requires only a small constant amount of space. * * This implementation was adapted from Tim Peters's list sort for * Python, which is described in detail here: * * http://svn.python.org/projects/python/trunk/Objects/listsort.txt * * Tim's C code may be found here: * * http://svn.python.org/projects/python/trunk/Objects/listobject.c * * The underlying techniques are described in this paper (and may have * even earlier origins): * * "Optimistic Sorting and Information Theoretic Complexity" * Peter McIlroy * SODA (Fourth Annual ACM-SIAM Symposium on Discrete Algorithms), * pp 467-474, Austin, Texas, 25-27 January 1993. * * While the API to this class consists solely of static methods, it is * (privately) instantiable; a TimSort instance holds the state of an ongoing * sort, assuming the input array is large enough to warrant the full-blown * TimSort. Small arrays are sorted in place, using a binary insertion sort. * * @author Josh Bloch */ public void sort(Buffer a, int lo, int hi, Comparator c) { assert c != null; int nRemaining = hi - lo; if (nRemaining < 2) return; // Arrays of size 0 and 1 are always sorted // If array is small, do a "mini-TimSort" with no merges if (nRemaining < MIN_MERGE) { int initRunLen = countRunAndMakeAscending(a, lo, hi, c); binarySort(a, lo, hi, lo + initRunLen, c); return; } /** * March over the array once, left to right, finding natural runs, * extending short natural runs to minRun elements, and merging runs * to maintain stack invariant. */ SortState sortState = new SortState(a, c, hi - lo); int minRun = minRunLength(nRemaining); do { // Identify next run int runLen = countRunAndMakeAscending(a, lo, hi, c); // If run is short, extend to min(minRun, nRemaining) if (runLen < minRun) { int force = nRemaining <= minRun ? nRemaining : minRun; binarySort(a, lo, lo + force, lo + runLen, c); runLen = force; } // Push run onto pending-run stack, and maybe merge sortState.pushRun(lo, runLen); sortState.mergeCollapse(); // Advance to find next run lo += runLen; nRemaining -= runLen; } while (nRemaining != 0); // Merge all remaining runs to complete sort assert lo == hi; sortState.mergeForceCollapse(); assert sortState.stackSize == 1; } /** * Sorts the specified portion of the specified array using a binary * insertion sort. This is the best method for sorting small numbers * of elements. It requires O(n log n) compares, but O(n^2) data * movement (worst case). * * If the initial part of the specified range is already sorted, * this method can take advantage of it: the method assumes that the * elements from index {@code lo}, inclusive, to {@code start}, * exclusive are already sorted. * * @param a the array in which a range is to be sorted * @param lo the index of the first element in the range to be sorted * @param hi the index after the last element in the range to be sorted * @param start the index of the first element in the range that is * not already known to be sorted ({@code lo <= start <= hi}) * @param c comparator to used for the sort */ @SuppressWarnings("fallthrough") private void binarySort(Buffer a, int lo, int hi, int start, Comparator c) { assert lo <= start && start <= hi; if (start == lo) start++; K key0 = s.newKey(); K key1 = s.newKey(); Buffer pivotStore = s.allocate(1); for ( ; start < hi; start++) { s.copyElement(a, start, pivotStore, 0); K pivot = s.getKey(pivotStore, 0, key0); // Set left (and right) to the index where a[start] (pivot) belongs int left = lo; int right = start; assert left <= right; /* * Invariants: * pivot >= all in [lo, left). * pivot < all in [right, start). */ while (left < right) { int mid = (left + right) >>> 1; if (c.compare(pivot, s.getKey(a, mid, key1)) < 0) right = mid; else left = mid + 1; } assert left == right; /* * The invariants still hold: pivot >= all in [lo, left) and * pivot < all in [left, start), so pivot belongs at left. Note * that if there are elements equal to pivot, left points to the * first slot after them -- that's why this sort is stable. * Slide elements over to make room for pivot. */ int n = start - left; // The number of elements to move // Switch is just an optimization for arraycopy in default case switch (n) { case 2: s.copyElement(a, left + 1, a, left + 2); case 1: s.copyElement(a, left, a, left + 1); break; default: s.copyRange(a, left, a, left + 1, n); } s.copyElement(pivotStore, 0, a, left); } } /** * Returns the length of the run beginning at the specified position in * the specified array and reverses the run if it is descending (ensuring * that the run will always be ascending when the method returns). * * A run is the longest ascending sequence with: * * a[lo] <= a[lo + 1] <= a[lo + 2] <= ... * * or the longest descending sequence with: * * a[lo] > a[lo + 1] > a[lo + 2] > ... * * For its intended use in a stable mergesort, the strictness of the * definition of "descending" is needed so that the call can safely * reverse a descending sequence without violating stability. * * @param a the array in which a run is to be counted and possibly reversed * @param lo index of the first element in the run * @param hi index after the last element that may be contained in the run. It is required that {@code lo < hi}. * @param c the comparator to used for the sort * @return the length of the run beginning at the specified position in * the specified array */ private int countRunAndMakeAscending(Buffer a, int lo, int hi, Comparator c) { assert lo < hi; int runHi = lo + 1; if (runHi == hi) return 1; K key0 = s.newKey(); K key1 = s.newKey(); // Find end of run, and reverse range if descending if (c.compare(s.getKey(a, runHi++, key0), s.getKey(a, lo, key1)) < 0) { // Descending while (runHi < hi && c.compare(s.getKey(a, runHi, key0), s.getKey(a, runHi - 1, key1)) < 0) runHi++; reverseRange(a, lo, runHi); } else { // Ascending while (runHi < hi && c.compare(s.getKey(a, runHi, key0), s.getKey(a, runHi - 1, key1)) >= 0) runHi++; } return runHi - lo; } /** * Reverse the specified range of the specified array. * * @param a the array in which a range is to be reversed * @param lo the index of the first element in the range to be reversed * @param hi the index after the last element in the range to be reversed */ private void reverseRange(Buffer a, int lo, int hi) { hi--; while (lo < hi) { s.swap(a, lo, hi); lo++; hi--; } } /** * Returns the minimum acceptable run length for an array of the specified * length. Natural runs shorter than this will be extended with * {@link #binarySort}. * * Roughly speaking, the computation is: * * If n < MIN_MERGE, return n (it's too small to bother with fancy stuff). * Else if n is an exact power of 2, return MIN_MERGE/2. * Else return an int k, MIN_MERGE/2 <= k <= MIN_MERGE, such that n/k * is close to, but strictly less than, an exact power of 2. * * For the rationale, see listsort.txt. * * @param n the length of the array to be sorted * @return the length of the minimum run to be merged */ private int minRunLength(int n) { assert n >= 0; int r = 0; // Becomes 1 if any 1 bits are shifted off while (n >= MIN_MERGE) { r |= (n & 1); n >>= 1; } return n + r; } private class SortState { /** * The Buffer being sorted. */ private final Buffer a; /** * Length of the sort Buffer. */ private final int aLength; /** * The comparator for this sort. */ private final Comparator c; /** * When we get into galloping mode, we stay there until both runs win less * often than MIN_GALLOP consecutive times. */ private static final int MIN_GALLOP = 7; /** * This controls when we get *into* galloping mode. It is initialized * to MIN_GALLOP. The mergeLo and mergeHi methods nudge it higher for * random data, and lower for highly structured data. */ private int minGallop = MIN_GALLOP; /** * Maximum initial size of tmp array, which is used for merging. The array * can grow to accommodate demand. * * Unlike Tim's original C version, we do not allocate this much storage * when sorting smaller arrays. This change was required for performance. */ private static final int INITIAL_TMP_STORAGE_LENGTH = 256; /** * Temp storage for merges. */ private Buffer tmp; // Actual runtime type will be Object[], regardless of T /** * Length of the temp storage. */ private int tmpLength = 0; /** * A stack of pending runs yet to be merged. Run i starts at * address base[i] and extends for len[i] elements. It's always * true (so long as the indices are in bounds) that: * * runBase[i] + runLen[i] == runBase[i + 1] * * so we could cut the storage for this, but it's a minor amount, * and keeping all the info explicit simplifies the code. */ private int stackSize = 0; // Number of pending runs on stack private final int[] runBase; private final int[] runLen; /** * Creates a TimSort instance to maintain the state of an ongoing sort. * * @param a the array to be sorted * @param c the comparator to determine the order of the sort */ private SortState(Buffer a, Comparator c, int len) { this.aLength = len; this.a = a; this.c = c; // Allocate temp storage (which may be increased later if necessary) tmpLength = len < 2 * INITIAL_TMP_STORAGE_LENGTH ? len >>> 1 : INITIAL_TMP_STORAGE_LENGTH; tmp = s.allocate(tmpLength); /* * Allocate runs-to-be-merged stack (which cannot be expanded). The * stack length requirements are described in listsort.txt. The C * version always uses the same stack length (85), but this was * measured to be too expensive when sorting "mid-sized" arrays (e.g., * 100 elements) in Java. Therefore, we use smaller (but sufficiently * large) stack lengths for smaller arrays. The "magic numbers" in the * computation below must be changed if MIN_MERGE is decreased. See * the MIN_MERGE declaration above for more information. */ int stackLen = (len < 120 ? 5 : len < 1542 ? 10 : len < 119151 ? 19 : 40); runBase = new int[stackLen]; runLen = new int[stackLen]; } /** * Pushes the specified run onto the pending-run stack. * * @param runBase index of the first element in the run * @param runLen the number of elements in the run */ private void pushRun(int runBase, int runLen) { this.runBase[stackSize] = runBase; this.runLen[stackSize] = runLen; stackSize++; } /** * Examines the stack of runs waiting to be merged and merges adjacent runs * until the stack invariants are reestablished: * * 1. runLen[i - 3] > runLen[i - 2] + runLen[i - 1] * 2. runLen[i - 2] > runLen[i - 1] * * This method is called each time a new run is pushed onto the stack, * so the invariants are guaranteed to hold for i < stackSize upon * entry to the method. */ private void mergeCollapse() { while (stackSize > 1) { int n = stackSize - 2; if ( (n >= 1 && runLen[n-1] <= runLen[n] + runLen[n+1]) || (n >= 2 && runLen[n-2] <= runLen[n] + runLen[n-1])) { if (runLen[n - 1] < runLen[n + 1]) n--; } else if (runLen[n] > runLen[n + 1]) { break; // Invariant is established } mergeAt(n); } } /** * Merges all runs on the stack until only one remains. This method is * called once, to complete the sort. */ private void mergeForceCollapse() { while (stackSize > 1) { int n = stackSize - 2; if (n > 0 && runLen[n - 1] < runLen[n + 1]) n--; mergeAt(n); } } /** * Merges the two runs at stack indices i and i+1. Run i must be * the penultimate or antepenultimate run on the stack. In other words, * i must be equal to stackSize-2 or stackSize-3. * * @param i stack index of the first of the two runs to merge */ private void mergeAt(int i) { assert stackSize >= 2; assert i >= 0; assert i == stackSize - 2 || i == stackSize - 3; int base1 = runBase[i]; int len1 = runLen[i]; int base2 = runBase[i + 1]; int len2 = runLen[i + 1]; assert len1 > 0 && len2 > 0; assert base1 + len1 == base2; /* * Record the length of the combined runs; if i is the 3rd-last * run now, also slide over the last run (which isn't involved * in this merge). The current run (i+1) goes away in any case. */ runLen[i] = len1 + len2; if (i == stackSize - 3) { runBase[i + 1] = runBase[i + 2]; runLen[i + 1] = runLen[i + 2]; } stackSize--; K key0 = s.newKey(); /* * Find where the first element of run2 goes in run1. Prior elements * in run1 can be ignored (because they're already in place). */ int k = gallopRight(s.getKey(a, base2, key0), a, base1, len1, 0, c); assert k >= 0; base1 += k; len1 -= k; if (len1 == 0) return; /* * Find where the last element of run1 goes in run2. Subsequent elements * in run2 can be ignored (because they're already in place). */ len2 = gallopLeft(s.getKey(a, base1 + len1 - 1, key0), a, base2, len2, len2 - 1, c); assert len2 >= 0; if (len2 == 0) return; // Merge remaining runs, using tmp array with min(len1, len2) elements if (len1 <= len2) mergeLo(base1, len1, base2, len2); else mergeHi(base1, len1, base2, len2); } /** * Locates the position at which to insert the specified key into the * specified sorted range; if the range contains an element equal to key, * returns the index of the leftmost equal element. * * @param key the key whose insertion point to search for * @param a the array in which to search * @param base the index of the first element in the range * @param len the length of the range; must be > 0 * @param hint the index at which to begin the search, 0 <= hint < n. * The closer hint is to the result, the faster this method will run. * @param c the comparator used to order the range, and to search * @return the int k, 0 <= k <= n such that a[b + k - 1] < key <= a[b + k], * pretending that a[b - 1] is minus infinity and a[b + n] is infinity. * In other words, key belongs at index b + k; or in other words, * the first k elements of a should precede key, and the last n - k * should follow it. */ private int gallopLeft(K key, Buffer a, int base, int len, int hint, Comparator c) { assert len > 0 && hint >= 0 && hint < len; int lastOfs = 0; int ofs = 1; K key0 = s.newKey(); if (c.compare(key, s.getKey(a, base + hint, key0)) > 0) { // Gallop right until a[base+hint+lastOfs] < key <= a[base+hint+ofs] int maxOfs = len - hint; while (ofs < maxOfs && c.compare(key, s.getKey(a, base + hint + ofs, key0)) > 0) { lastOfs = ofs; ofs = (ofs << 1) + 1; if (ofs <= 0) // int overflow ofs = maxOfs; } if (ofs > maxOfs) ofs = maxOfs; // Make offsets relative to base lastOfs += hint; ofs += hint; } else { // key <= a[base + hint] // Gallop left until a[base+hint-ofs] < key <= a[base+hint-lastOfs] final int maxOfs = hint + 1; while (ofs < maxOfs && c.compare(key, s.getKey(a, base + hint - ofs, key0)) <= 0) { lastOfs = ofs; ofs = (ofs << 1) + 1; if (ofs <= 0) // int overflow ofs = maxOfs; } if (ofs > maxOfs) ofs = maxOfs; // Make offsets relative to base int tmp = lastOfs; lastOfs = hint - ofs; ofs = hint - tmp; } assert -1 <= lastOfs && lastOfs < ofs && ofs <= len; /* * Now a[base+lastOfs] < key <= a[base+ofs], so key belongs somewhere * to the right of lastOfs but no farther right than ofs. Do a binary * search, with invariant a[base + lastOfs - 1] < key <= a[base + ofs]. */ lastOfs++; while (lastOfs < ofs) { int m = lastOfs + ((ofs - lastOfs) >>> 1); if (c.compare(key, s.getKey(a, base + m, key0)) > 0) lastOfs = m + 1; // a[base + m] < key else ofs = m; // key <= a[base + m] } assert lastOfs == ofs; // so a[base + ofs - 1] < key <= a[base + ofs] return ofs; } /** * Like gallopLeft, except that if the range contains an element equal to * key, gallopRight returns the index after the rightmost equal element. * * @param key the key whose insertion point to search for * @param a the array in which to search * @param base the index of the first element in the range * @param len the length of the range; must be > 0 * @param hint the index at which to begin the search, 0 <= hint < n. * The closer hint is to the result, the faster this method will run. * @param c the comparator used to order the range, and to search * @return the int k, 0 <= k <= n such that a[b + k - 1] <= key < a[b + k] */ private int gallopRight(K key, Buffer a, int base, int len, int hint, Comparator c) { assert len > 0 && hint >= 0 && hint < len; int ofs = 1; int lastOfs = 0; K key1 = s.newKey(); if (c.compare(key, s.getKey(a, base + hint, key1)) < 0) { // Gallop left until a[b+hint - ofs] <= key < a[b+hint - lastOfs] int maxOfs = hint + 1; while (ofs < maxOfs && c.compare(key, s.getKey(a, base + hint - ofs, key1)) < 0) { lastOfs = ofs; ofs = (ofs << 1) + 1; if (ofs <= 0) // int overflow ofs = maxOfs; } if (ofs > maxOfs) ofs = maxOfs; // Make offsets relative to b int tmp = lastOfs; lastOfs = hint - ofs; ofs = hint - tmp; } else { // a[b + hint] <= key // Gallop right until a[b+hint + lastOfs] <= key < a[b+hint + ofs] int maxOfs = len - hint; while (ofs < maxOfs && c.compare(key, s.getKey(a, base + hint + ofs, key1)) >= 0) { lastOfs = ofs; ofs = (ofs << 1) + 1; if (ofs <= 0) // int overflow ofs = maxOfs; } if (ofs > maxOfs) ofs = maxOfs; // Make offsets relative to b lastOfs += hint; ofs += hint; } assert -1 <= lastOfs && lastOfs < ofs && ofs <= len; /* * Now a[b + lastOfs] <= key < a[b + ofs], so key belongs somewhere to * the right of lastOfs but no farther right than ofs. Do a binary * search, with invariant a[b + lastOfs - 1] <= key < a[b + ofs]. */ lastOfs++; while (lastOfs < ofs) { int m = lastOfs + ((ofs - lastOfs) >>> 1); if (c.compare(key, s.getKey(a, base + m, key1)) < 0) ofs = m; // key < a[b + m] else lastOfs = m + 1; // a[b + m] <= key } assert lastOfs == ofs; // so a[b + ofs - 1] <= key < a[b + ofs] return ofs; } /** * Merges two adjacent runs in place, in a stable fashion. The first * element of the first run must be greater than the first element of the * second run (a[base1] > a[base2]), and the last element of the first run * (a[base1 + len1-1]) must be greater than all elements of the second run. * * For performance, this method should be called only when len1 <= len2; * its twin, mergeHi should be called if len1 >= len2. (Either method * may be called if len1 == len2.) * * @param base1 index of first element in first run to be merged * @param len1 length of first run to be merged (must be > 0) * @param base2 index of first element in second run to be merged * (must be aBase + aLen) * @param len2 length of second run to be merged (must be > 0) */ private void mergeLo(int base1, int len1, int base2, int len2) { assert len1 > 0 && len2 > 0 && base1 + len1 == base2; // Copy first run into temp array Buffer a = this.a; // For performance Buffer tmp = ensureCapacity(len1); s.copyRange(a, base1, tmp, 0, len1); int cursor1 = 0; // Indexes into tmp array int cursor2 = base2; // Indexes int a int dest = base1; // Indexes int a // Move first element of second run and deal with degenerate cases s.copyElement(a, cursor2++, a, dest++); if (--len2 == 0) { s.copyRange(tmp, cursor1, a, dest, len1); return; } if (len1 == 1) { s.copyRange(a, cursor2, a, dest, len2); s.copyElement(tmp, cursor1, a, dest + len2); // Last elt of run 1 to end of merge return; } K key0 = s.newKey(); K key1 = s.newKey(); Comparator c = this.c; // Use local variable for performance int minGallop = this.minGallop; // " " " " " outer: while (true) { int count1 = 0; // Number of times in a row that first run won int count2 = 0; // Number of times in a row that second run won /* * Do the straightforward thing until (if ever) one run starts * winning consistently. */ do { assert len1 > 1 && len2 > 0; if (c.compare(s.getKey(a, cursor2, key0), s.getKey(tmp, cursor1, key1)) < 0) { s.copyElement(a, cursor2++, a, dest++); count2++; count1 = 0; if (--len2 == 0) break outer; } else { s.copyElement(tmp, cursor1++, a, dest++); count1++; count2 = 0; if (--len1 == 1) break outer; } } while ((count1 | count2) < minGallop); /* * One run is winning so consistently that galloping may be a * huge win. So try that, and continue galloping until (if ever) * neither run appears to be winning consistently anymore. */ do { assert len1 > 1 && len2 > 0; count1 = gallopRight(s.getKey(a, cursor2, key0), tmp, cursor1, len1, 0, c); if (count1 != 0) { s.copyRange(tmp, cursor1, a, dest, count1); dest += count1; cursor1 += count1; len1 -= count1; if (len1 <= 1) // len1 == 1 || len1 == 0 break outer; } s.copyElement(a, cursor2++, a, dest++); if (--len2 == 0) break outer; count2 = gallopLeft(s.getKey(tmp, cursor1, key0), a, cursor2, len2, 0, c); if (count2 != 0) { s.copyRange(a, cursor2, a, dest, count2); dest += count2; cursor2 += count2; len2 -= count2; if (len2 == 0) break outer; } s.copyElement(tmp, cursor1++, a, dest++); if (--len1 == 1) break outer; minGallop--; } while (count1 >= MIN_GALLOP | count2 >= MIN_GALLOP); if (minGallop < 0) minGallop = 0; minGallop += 2; // Penalize for leaving gallop mode } // End of "outer" loop this.minGallop = minGallop < 1 ? 1 : minGallop; // Write back to field if (len1 == 1) { assert len2 > 0; s.copyRange(a, cursor2, a, dest, len2); s.copyElement(tmp, cursor1, a, dest + len2); // Last elt of run 1 to end of merge } else if (len1 == 0) { throw new IllegalArgumentException( "Comparison method violates its general contract!"); } else { assert len2 == 0; assert len1 > 1; s.copyRange(tmp, cursor1, a, dest, len1); } } /** * Like mergeLo, except that this method should be called only if * len1 >= len2; mergeLo should be called if len1 <= len2. (Either method * may be called if len1 == len2.) * * @param base1 index of first element in first run to be merged * @param len1 length of first run to be merged (must be > 0) * @param base2 index of first element in second run to be merged * (must be aBase + aLen) * @param len2 length of second run to be merged (must be > 0) */ private void mergeHi(int base1, int len1, int base2, int len2) { assert len1 > 0 && len2 > 0 && base1 + len1 == base2; // Copy second run into temp array Buffer a = this.a; // For performance Buffer tmp = ensureCapacity(len2); s.copyRange(a, base2, tmp, 0, len2); int cursor1 = base1 + len1 - 1; // Indexes into a int cursor2 = len2 - 1; // Indexes into tmp array int dest = base2 + len2 - 1; // Indexes into a K key0 = s.newKey(); K key1 = s.newKey(); // Move last element of first run and deal with degenerate cases s.copyElement(a, cursor1--, a, dest--); if (--len1 == 0) { s.copyRange(tmp, 0, a, dest - (len2 - 1), len2); return; } if (len2 == 1) { dest -= len1; cursor1 -= len1; s.copyRange(a, cursor1 + 1, a, dest + 1, len1); s.copyElement(tmp, cursor2, a, dest); return; } Comparator c = this.c; // Use local variable for performance int minGallop = this.minGallop; // " " " " " outer: while (true) { int count1 = 0; // Number of times in a row that first run won int count2 = 0; // Number of times in a row that second run won /* * Do the straightforward thing until (if ever) one run * appears to win consistently. */ do { assert len1 > 0 && len2 > 1; if (c.compare(s.getKey(tmp, cursor2, key0), s.getKey(a, cursor1, key1)) < 0) { s.copyElement(a, cursor1--, a, dest--); count1++; count2 = 0; if (--len1 == 0) break outer; } else { s.copyElement(tmp, cursor2--, a, dest--); count2++; count1 = 0; if (--len2 == 1) break outer; } } while ((count1 | count2) < minGallop); /* * One run is winning so consistently that galloping may be a * huge win. So try that, and continue galloping until (if ever) * neither run appears to be winning consistently anymore. */ do { assert len1 > 0 && len2 > 1; count1 = len1 - gallopRight(s.getKey(tmp, cursor2, key0), a, base1, len1, len1 - 1, c); if (count1 != 0) { dest -= count1; cursor1 -= count1; len1 -= count1; s.copyRange(a, cursor1 + 1, a, dest + 1, count1); if (len1 == 0) break outer; } s.copyElement(tmp, cursor2--, a, dest--); if (--len2 == 1) break outer; count2 = len2 - gallopLeft(s.getKey(a, cursor1, key0), tmp, 0, len2, len2 - 1, c); if (count2 != 0) { dest -= count2; cursor2 -= count2; len2 -= count2; s.copyRange(tmp, cursor2 + 1, a, dest + 1, count2); if (len2 <= 1) // len2 == 1 || len2 == 0 break outer; } s.copyElement(a, cursor1--, a, dest--); if (--len1 == 0) break outer; minGallop--; } while (count1 >= MIN_GALLOP | count2 >= MIN_GALLOP); if (minGallop < 0) minGallop = 0; minGallop += 2; // Penalize for leaving gallop mode } // End of "outer" loop this.minGallop = minGallop < 1 ? 1 : minGallop; // Write back to field if (len2 == 1) { assert len1 > 0; dest -= len1; cursor1 -= len1; s.copyRange(a, cursor1 + 1, a, dest + 1, len1); s.copyElement(tmp, cursor2, a, dest); // Move first elt of run2 to front of merge } else if (len2 == 0) { throw new IllegalArgumentException( "Comparison method violates its general contract!"); } else { assert len1 == 0; assert len2 > 0; s.copyRange(tmp, 0, a, dest - (len2 - 1), len2); } } /** * Ensures that the external array tmp has at least the specified * number of elements, increasing its size if necessary. The size * increases exponentially to ensure amortized linear time complexity. * * @param minCapacity the minimum required capacity of the tmp array * @return tmp, whether or not it grew */ private Buffer ensureCapacity(int minCapacity) { if (tmpLength < minCapacity) { // Compute smallest power of 2 > minCapacity int newSize = minCapacity; newSize |= newSize >> 1; newSize |= newSize >> 2; newSize |= newSize >> 4; newSize |= newSize >> 8; newSize |= newSize >> 16; newSize++; if (newSize < 0) // Not bloody likely! newSize = minCapacity; else newSize = Math.min(newSize, aLength >>> 1); tmp = s.allocate(newSize); tmpLength = newSize; } return tmp; } } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/UnsafeExternalRowSorter.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.utils.sort; import io.mycat.memory.MyCatMemory; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager; import io.mycat.memory.unsafe.row.StructType; import io.mycat.memory.unsafe.row.UnsafeRow; import io.mycat.sqlengine.mpp.OrderCol; import io.mycat.sqlengine.mpp.RowDataPacketSorter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.util.Iterator; import java.util.List; public final class UnsafeExternalRowSorter { private final Logger logger = LoggerFactory.getLogger(UnsafeExternalSorter.class); private long numRowsInserted = 0; private final StructType schema; private final PrefixComputer prefixComputer; private final UnsafeExternalSorter sorter; private final PrefixComparator prefixComparator; private final RecordComparator recordComparator; public abstract static class PrefixComputer { protected abstract long computePrefix(UnsafeRow row) throws UnsupportedEncodingException; } public UnsafeExternalRowSorter(DataNodeMemoryManager dataNodeMemoryManager, @Nonnull MyCatMemory myCatMemory, StructType schema, PrefixComparator prefixComparator, PrefixComputer prefixComputer, long pageSizeBytes, boolean canUseRadixSort, boolean enableSort) throws IOException { this.schema = schema; this.prefixComputer = prefixComputer; this.prefixComparator = prefixComparator; this.recordComparator = new RowComparator(schema); sorter = UnsafeExternalSorter.create( dataNodeMemoryManager, myCatMemory.getBlockManager(), myCatMemory.getSerializerManager(), recordComparator, prefixComparator, myCatMemory.getConf().getSizeAsBytes("mycat.pointer.array.len","1K"), pageSizeBytes, canUseRadixSort, enableSort); } public void insertRow(UnsafeRow row) throws IOException { final long prefix = prefixComputer.computePrefix(row); sorter.insertRecord( row.getBaseObject(), row.getBaseOffset(), row.getSizeInBytes(), prefix); numRowsInserted++; } /** * Return total rows */ public long getNumRowsInserted() { return numRowsInserted; } /** * Return the peak memory used so far, in bytes. */ public long getPeakMemoryUsage() { return sorter.getPeakMemoryUsedBytes(); } /** * @return the total amount of time spent sorting data (in-memory only). */ public long getSortTimeNanos() { return sorter.getSortTimeNanos(); } public void cleanupResources() { sorter.cleanupResources(); } public Iterator sort() throws IOException { try { final UnsafeSorterIterator sortedIterator = sorter.getSortedIterator(); if (!sortedIterator.hasNext()) { cleanupResources(); } return new AbstractScalaRowIterator() { private final int numFields = schema.length(); private UnsafeRow row = new UnsafeRow(numFields); @Override public boolean hasNext() { return sortedIterator.hasNext(); } @Override public UnsafeRow next() { try { sortedIterator.loadNext(); row.pointTo(sortedIterator.getBaseObject(), sortedIterator.getBaseOffset(), sortedIterator.getRecordLength()); if (!hasNext()) { UnsafeRow copy = row.copy(); // so that we don't have dangling pointers to freed page row = null; // so that we don't keep references to the base object cleanupResources(); return copy; } else { return row; } } catch (IOException e) { cleanupResources(); // Scala iterators don't declare any checked exceptions, so we need to use this hack // to re-throw the exception: Platform.throwException(e); } throw new RuntimeException("Exception should have been re-thrown in next()"); } @Override public void remove() { } }; } catch (IOException e) { cleanupResources(); throw e; } } public UnsafeSorterIterator getRowUnsafeSorterIterator() throws IOException{ return sorter.getSortedIterator(); } public Iterator mergerSort(List list) throws IOException { UnsafeRowsMerger unsafeRowsMerger = new UnsafeRowsMerger(recordComparator,prefixComparator,list.size()); for (int i = 0; i () { private final int numFields = schema.length(); private UnsafeRow row = new UnsafeRow(numFields); @Override public boolean hasNext() { return sortedIterator.hasNext(); } @Override public UnsafeRow next() { try { sortedIterator.loadNext(); row.pointTo( sortedIterator.getBaseObject(), sortedIterator.getBaseOffset(), sortedIterator.getRecordLength()); if (!hasNext()) { UnsafeRow copy = row.copy(); // so that we don't have dangling pointers to freed page row = null; // so that we don't keep references to the base object cleanupResources(); return copy; } else { return row; } } catch (IOException e) { cleanupResources(); // Scala iterators don't declare any checked exceptions, so we need to use this hack // to re-throw the exception: Platform.throwException(e); } throw new RuntimeException("Exception should have been re-thrown in next()"); } @Override public void remove() { } }; } catch (IOException e) { cleanupResources(); throw e; } } public Iterator sort(Iterator inputIterator) throws IOException { while (inputIterator.hasNext()) { insertRow(inputIterator.next()); } return sort(); } private static final class RowComparator extends RecordComparator { private final int numFields; private final UnsafeRow row1; private final UnsafeRow row2; private final StructType schema; RowComparator(StructType schema) { assert schema.length()>=0; this.schema = schema; this.numFields = schema.length(); this.row1 = new UnsafeRow(numFields); this.row2 = new UnsafeRow(numFields); } @Override public int compare(Object baseObj1, long baseOff1, Object baseObj2, long baseOff2) { OrderCol[] orderCols = schema.getOrderCols(); if(orderCols == null){ return 0; } /**取出一行数据*/ row1.pointTo(baseObj1, baseOff1, -1); row2.pointTo(baseObj2, baseOff2, -1); int cmp = 0; int len = orderCols.length; int type = OrderCol.COL_ORDER_TYPE_ASC; /**升序*/ for (int i = 0; i < len; i++) { int colIndex = orderCols[i].colMeta.colIndex; /**取出一行数据中的列值，进行大小比对*/ byte[] left = null; byte[] right = null; if(!row1.isNullAt(colIndex)) { left = row1.getBinary(colIndex); }else { left = new byte[1]; left[0] = UnsafeRow.NULL_MARK; } if(!row2.isNullAt(colIndex)) { right = row2.getBinary(colIndex); }else { right = new byte[1]; right[0] = UnsafeRow.NULL_MARK; } if (orderCols[i].orderType == type) { cmp = RowDataPacketSorter.compareObject(left, right, orderCols[i]); } else { cmp = RowDataPacketSorter.compareObject(right, left, orderCols[i]); } if (cmp != 0) return cmp; } return cmp; } } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/UnsafeExternalSorter.java ================================================ package io.mycat.memory.unsafe.utils.sort; import com.google.common.annotations.VisibleForTesting; import io.mycat.MycatServer; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.array.LongArray; import io.mycat.memory.unsafe.memory.MemoryBlock; import io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager; import io.mycat.memory.unsafe.memory.mm.MemoryConsumer; import io.mycat.memory.unsafe.storage.DataNodeDiskManager; import io.mycat.memory.unsafe.storage.SerializerManager; import io.mycat.memory.unsafe.utils.JavaUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nullable; import java.io.File; import java.io.IOException; import java.util.LinkedList; import java.util.Queue; /** * External sorter based on {@link UnsafeInMemorySorter}. */ public final class UnsafeExternalSorter extends MemoryConsumer { private final Logger logger = LoggerFactory.getLogger(UnsafeExternalSorter.class); @Nullable private final PrefixComparator prefixComparator; @Nullable private final RecordComparator recordComparator; private final DataNodeMemoryManager dataNodeMemoryManager; private final DataNodeDiskManager blockManager; private final SerializerManager serializerManager; /** The buffer size to use when writing spills using DiskRowWriter */ private final int fileBufferSizeBytes; /** * Memory pages that hold the records being sorted. The pages in this list are freed when * spilling, although in principle we could recycle these pages across spills (on the other hand, * this might not be necessary if we maintained a pool of re-usable pages in the DataNodeMemoryManager * itself). */ private final LinkedList allocatedPages = new LinkedList(); private final LinkedList spillWriters = new LinkedList(); // These variables are reset after spilling: @Nullable private volatile UnsafeInMemorySorter inMemSorter; private MemoryBlock currentPage = null; private long pageCursor = -1; private long peakMemoryUsedBytes = 0; private long totalSpillBytes = 0L; private long totalSortTimeNanos = 0L; private volatile SpillableIterator readingIterator = null; public static UnsafeExternalSorter createWithExistingInMemorySorter( DataNodeMemoryManager dataNodeMemoryManager, DataNodeDiskManager blockManager, SerializerManager serializerManager, RecordComparator recordComparator, PrefixComparator prefixComparator, int initialSize, long pageSizeBytes, UnsafeInMemorySorter inMemorySorter,boolean enableSort) throws IOException { UnsafeExternalSorter sorter = new UnsafeExternalSorter(dataNodeMemoryManager, blockManager, serializerManager,recordComparator, prefixComparator, initialSize, pageSizeBytes, inMemorySorter, false /* ignored */,enableSort); sorter.spill(Long.MAX_VALUE, sorter); // The external sorter will be used to insert records, in-memory sorter is not needed. sorter.inMemSorter = null; return sorter; } public static UnsafeExternalSorter create( DataNodeMemoryManager dataNodeMemoryManager, DataNodeDiskManager blockManager, SerializerManager serializerManager, RecordComparator recordComparator, PrefixComparator prefixComparator, long initialSize, long pageSizeBytes, boolean canUseRadixSort, boolean enableSort) { return new UnsafeExternalSorter(dataNodeMemoryManager, blockManager, serializerManager, recordComparator, prefixComparator, initialSize, pageSizeBytes, null, canUseRadixSort,enableSort); } private UnsafeExternalSorter( DataNodeMemoryManager dataNodeMemoryManager, DataNodeDiskManager blockManager, SerializerManager serializerManager, RecordComparator recordComparator, PrefixComparator prefixComparator, long initialSize, long pageSizeBytes, @Nullable UnsafeInMemorySorter existingInMemorySorter, boolean canUseRadixSort,boolean enableSort) { super(dataNodeMemoryManager, pageSizeBytes); this.dataNodeMemoryManager = dataNodeMemoryManager; this.blockManager = blockManager; this.serializerManager = serializerManager; this.recordComparator = recordComparator; this.prefixComparator = prefixComparator; if(MycatServer.getInstance().getMyCatMemory() != null){ this.fileBufferSizeBytes = (int) MycatServer.getInstance(). getMyCatMemory().getConf().getSizeAsBytes("mycat.merge.file.buffer", "32k"); }else{ this.fileBufferSizeBytes = 32*1024; } if (existingInMemorySorter == null) { this.inMemSorter = new UnsafeInMemorySorter( this, dataNodeMemoryManager, recordComparator, prefixComparator, initialSize, canUseRadixSort,enableSort); } else { this.inMemSorter = existingInMemorySorter; } this.peakMemoryUsedBytes = getMemoryUsage(); } /** * Marks the current page as no-more-space-available, and as a result, either allocate a * new page or spill when we see the next record. */ @VisibleForTesting public void closeCurrentPage() { if (currentPage != null) { pageCursor = currentPage.getBaseOffset() + currentPage.size(); } } /** * Sort and spill the current records in response to memory pressure. */ @Override public long spill(long size, MemoryConsumer trigger) throws IOException { if (trigger != this) { if (readingIterator != null) { return readingIterator.spill(); } return 0L; // this should throw exception } if (inMemSorter == null || inMemSorter.numRecords() <= 0) { return 0L; } logger.info("Thread" + Thread.currentThread().getId() +" spilling sort data of "+ JavaUtils.bytesToString(getMemoryUsage()) +" to disk ("+ spillWriters.size()+" times so far)"); // We only write out contents of the inMemSorter if it is not empty. if (inMemSorter.numRecords() > 0) { /** * 创建一个写外存的SpillWriter，当前内存数据已经排序了，直接写到磁盘中. */ final UnsafeSorterSpillWriter spillWriter = new UnsafeSorterSpillWriter(blockManager, fileBufferSizeBytes,/**writeMetrics,*/ inMemSorter.numRecords()); /** * 添加到SpillWriter列表中，标志了有多少 spillWriters.size()写到磁盘中了。 */ spillWriters.add(spillWriter); /** * 获取本次内存中排序的迭代器，这个函数执行In Memory Sort use time sorter 或者 radix sorter */ final UnsafeSorterIterator sortedRecords = inMemSorter.getSortedIterator(); /** * 一条一条记录写入磁盘 */ while (sortedRecords.hasNext()) { /** * */ sortedRecords.loadNext(); /** * 获取当前记录的起始对象实例，on-heap为obj，off-heap为null */ final Object baseObject = sortedRecords.getBaseObject(); /** * 获取当前记录的相对起始对象实例地址偏移量 */ final long baseOffset = sortedRecords.getBaseOffset(); /** * 当前记录的长度 */ final int recordLength = sortedRecords.getRecordLength(); /** * 把数据写入磁盘写入器中 Write a record to a spill file. */ spillWriter.write(baseObject, baseOffset, recordLength, sortedRecords.getKeyPrefix()); } /** * 关闭spillWriter */ spillWriter.close(); } /** * 释放当前sorter所占的内存数据 */ final long spillSize = freeMemory(); // Note that this is more-or-less going to be a multiple of the page size, so wasted space in // pages will currently be counted as memory spilled even though that space isn't actually // written to disk. This also counts the space needed to store the sorter's pointer array. inMemSorter.reset(); // Reset the in-memory sorter's pointer array only after freeing up the memory pages holding the // records. Otherwise, if the task is over allocated memory, then without freeing the memory // pages, we might not be able to get memory for the pointer array. totalSpillBytes += spillSize; return spillSize; } /** * Return the total memory usage of this sorter, including the data pages and the sorter's pointer * array. */ private long getMemoryUsage() { long totalPageSize = 0; for (MemoryBlock page : allocatedPages) { totalPageSize += page.size(); } return ((inMemSorter == null) ? 0 : inMemSorter.getMemoryUsage()) + totalPageSize; } private void updatePeakMemoryUsed() { long mem = getMemoryUsage(); if (mem > peakMemoryUsedBytes) { peakMemoryUsedBytes = mem; } } /** * Return the peak memory used so far, in bytes. */ public long getPeakMemoryUsedBytes() { updatePeakMemoryUsed(); return peakMemoryUsedBytes; } /** * @return the total amount of time spent sorting data (in-memory only). */ public long getSortTimeNanos() { UnsafeInMemorySorter sorter = inMemSorter; if (sorter != null) { return sorter.getSortTimeNanos(); } return totalSortTimeNanos; } /** * Return the total number of bytes that has been spilled into disk so far. */ public long getSpillSize() { return totalSpillBytes; } @VisibleForTesting public int getNumberOfAllocatedPages() { return allocatedPages.size(); } /** * Free this sorter's data pages. * * @return the number of bytes freed. */ private long freeMemory() { updatePeakMemoryUsed(); long memoryFreed = 0; for (MemoryBlock block : allocatedPages) { memoryFreed += block.size(); freePage(block); } allocatedPages.clear(); currentPage = null; pageCursor = 0; return memoryFreed; } /** * Deletes any spill files created by this sorter. */ private void deleteSpillFiles() { for (UnsafeSorterSpillWriter spill : spillWriters) { File file = spill.getFile(); if(file == null) continue; try { JavaUtils.deleteRecursively(file.getParentFile().getParentFile()); } catch (IOException e) { logger.error(e.getMessage()); } if (file.exists()) { if (!file.delete()) { logger.error("Was unable to delete spill file {}", file.getAbsolutePath()); } } } } /** * Frees this sorter's in-memory data structures and cleans up its spill files. */ public void cleanupResources() { synchronized (this) { deleteSpillFiles(); freeMemory(); if (inMemSorter != null) { inMemSorter.free(); inMemSorter = null; } } } /** * Checks whether there is enough space to insert an additional record in to the sort pointer * array and grows the array if additional space is required. If the required space cannot be * obtained, then the in-memory data will be spilled to disk. */ private void growPointerArrayIfNecessary() throws IOException { assert(inMemSorter != null); if (!inMemSorter.hasSpaceForAnotherRecord()) { long used = inMemSorter.getMemoryUsage(); LongArray array; try { // could trigger spilling array = allocateLongArray(used / 8 * 2); } catch (OutOfMemoryError e) { // should have trigger spilling if (!inMemSorter.hasSpaceForAnotherRecord()) { logger.error("Unable to grow the pointer array"); throw e; } return; } // check if spilling is triggered or not if (inMemSorter.hasSpaceForAnotherRecord()) { freeLongArray(array); } else { inMemSorter.expandPointerArray(array); } } } /** * Allocates more memory in order to insert an additional record. This will request additional * memory from the memory manager and spill if the requested memory can not be obtained. * * @param required the required space in the data page, in bytes, including space for storing * the record size. This must be less than or equal to the page size (records * that exceed the page size are handled via a different code path which uses * special overflow pages). */ private void acquireNewPageIfNecessary(int required) { if (currentPage == null || pageCursor + required > currentPage.getBaseOffset() + currentPage.size()) { // TODO: try to find space on previous pages currentPage = allocatePage(required); pageCursor = currentPage.getBaseOffset(); allocatedPages.add(currentPage); } } /** * Write a record to the sorter. */ public void insertRecord(Object recordBase, long recordOffset, int length, long prefix) throws IOException { growPointerArrayIfNecessary(); // Need 4 bytes to store the record length. final int required = length + 4; acquireNewPageIfNecessary(required); final Object base = currentPage.getBaseObject(); final long recordAddress = dataNodeMemoryManager.encodePageNumberAndOffset(currentPage,pageCursor); Platform.putInt(base, pageCursor, length); pageCursor += 4; Platform.copyMemory(recordBase,recordOffset,base,pageCursor,length); pageCursor += length; assert(inMemSorter != null); inMemSorter.insertRecord(recordAddress,prefix); } /** * Write a key-value record to the sorter. The key and value will be put together in-memory, * using the following format: * * record length (4 bytes), key length (4 bytes), key data, value data * * record length = key length + value length + 4 */ public void insertKVRecord(Object keyBase, long keyOffset, int keyLen, Object valueBase, long valueOffset, int valueLen, long prefix) throws IOException { growPointerArrayIfNecessary(); final int required = keyLen + valueLen + 4 + 4; acquireNewPageIfNecessary(required); /** * 数据k-v插入currentPage(MemoryBlock)页内，当前插入位置pageCursor */ final Object base = currentPage.getBaseObject(); /** * 通过currentPage和pageCursor页内偏移量，codec一个地址处理，该条记录存数据的 * 存数据的起始位置 */ final long recordAddress = dataNodeMemoryManager.encodePageNumberAndOffset(currentPage,pageCursor); /** * 一条记录的总长度=keyLen + valueLen + record length (一般是int类型4个字节) */ Platform.putInt(base,pageCursor, keyLen + valueLen + 4/**record length所占的长度*/); /** * 移动4个bytes */ pageCursor += 4; /** * 存key len的size */ Platform.putInt(base,pageCursor, keyLen); /** * 移动4个bytes */ pageCursor += 4; /** * 存key的值 */ Platform.copyMemory(keyBase, keyOffset, base, pageCursor, keyLen); /** * 移动keyLen个bytes */ pageCursor += keyLen; /** * 存value的值 */ Platform.copyMemory(valueBase, valueOffset, base, pageCursor, valueLen); /** * 移动valueLen个bytes */ pageCursor += valueLen; assert(inMemSorter != null); /** * 把对应的指针插入到longArray数组中， * longArray存指向Page内一个指针的所存储的值 */ inMemSorter.insertRecord(recordAddress, prefix); } /** * Merges another UnsafeExternalSorters into this one, the other one will be emptied. * * @throws IOException */ public void merge(UnsafeExternalSorter other) throws IOException { other.spill(); spillWriters.addAll(other.spillWriters); // remove them from `spillWriters`, or the files will be deleted in `cleanupResources`. other.spillWriters.clear(); other.cleanupResources(); } /** * SpillableIterator是一个支持内存+外存排序的迭代器 * Returns a sorted iterator. It is the caller's responsibility to call `cleanupResources()` * after consuming this iterator. */ public UnsafeSorterIterator getSortedIterator() throws IOException { assert(recordComparator != null); if (spillWriters.isEmpty()) { assert(inMemSorter != null); readingIterator = new SpillableIterator(inMemSorter.getSortedIterator()); return readingIterator; } else { /** * 合并多个UnsafeSorterSpillWriter对应的文件，进行排序???? */ final UnsafeSorterSpillMerger spillMerger = new UnsafeSorterSpillMerger(recordComparator, prefixComparator, spillWriters.size()); for (UnsafeSorterSpillWriter spillWriter : spillWriters) { /** * 通过UnsafeSorterSpillReader迭代器放入要合并的UnsafeSorterSpillMerger中 */ spillMerger.addSpillIfNotEmpty(spillWriter.getReader(serializerManager)); } if (inMemSorter != null) { readingIterator = new SpillableIterator(inMemSorter.getSortedIterator()); spillMerger.addSpillIfNotEmpty(readingIterator); } /** * 最终调用排序器排序，重点分析函数 */ return spillMerger.getSortedIterator(); } } /** * An UnsafeSorterIterator that support spilling. */ public class SpillableIterator extends UnsafeSorterIterator { private UnsafeSorterIterator upstream; private UnsafeSorterIterator nextUpstream = null; private MemoryBlock lastPage = null; private boolean loaded = false; private int numRecords = 0; SpillableIterator(UnsafeInMemorySorter.SortedIterator inMemIterator) { this.upstream = inMemIterator; this.numRecords = inMemIterator.getNumRecords(); } public int getNumRecords() { return numRecords; } public long spill() throws IOException { synchronized (this) { if (!(upstream instanceof UnsafeInMemorySorter.SortedIterator && nextUpstream == null && numRecords > 0)) { return 0L; } UnsafeInMemorySorter.SortedIterator inMemIterator = ((UnsafeInMemorySorter.SortedIterator) upstream).clone(); // Iterate over the records that have not been returned and spill them. final UnsafeSorterSpillWriter spillWriter = new UnsafeSorterSpillWriter(blockManager, fileBufferSizeBytes,/**writeMetrics,*/ numRecords); while (inMemIterator.hasNext()) { inMemIterator.loadNext(); final Object baseObject = inMemIterator.getBaseObject(); final long baseOffset = inMemIterator.getBaseOffset(); final int recordLength = inMemIterator.getRecordLength(); spillWriter.write(baseObject, baseOffset, recordLength, inMemIterator.getKeyPrefix()); } spillWriter.close(); spillWriters.add(spillWriter); nextUpstream = spillWriter.getReader(serializerManager); long released = 0L; synchronized (UnsafeExternalSorter.this) { // release the pages except the one that is used. There can still be a caller that // is accessing the current record. We free this page in that caller's next loadNext() // call. for (MemoryBlock page : allocatedPages) { if (!loaded || page.getBaseObject() != upstream.getBaseObject()) { released += page.size(); freePage(page); } else { lastPage = page; } } allocatedPages.clear(); } // in-memory sorter will not be used after spilling assert(inMemSorter != null); released += inMemSorter.getMemoryUsage(); totalSortTimeNanos += inMemSorter.getSortTimeNanos(); inMemSorter.free(); inMemSorter = null; totalSpillBytes += released; return released; } } @Override public boolean hasNext() { return numRecords > 0; } @Override public void loadNext() throws IOException { synchronized (this) { loaded = true; if (nextUpstream != null) { // Just consumed the last record from in memory iterator if (lastPage != null) { freePage(lastPage); lastPage = null; } upstream = nextUpstream; nextUpstream = null; } numRecords--; upstream.loadNext(); } } @Override public Object getBaseObject() { return upstream.getBaseObject(); } @Override public long getBaseOffset() { return upstream.getBaseOffset(); } @Override public int getRecordLength() { return upstream.getRecordLength(); } @Override public long getKeyPrefix() { return upstream.getKeyPrefix(); } } /** * Returns a iterator, which will return the rows in the order as inserted. * * It is the caller's responsibility to call `cleanupResources()` * after consuming this iterator. * * TODO: support forced spilling */ public UnsafeSorterIterator getIterator() throws IOException { /** * 如果spillWriters为空说明，直接读取内存中即可 */ if (spillWriters.isEmpty()) { assert(inMemSorter != null); return inMemSorter.getSortedIterator(); } else { /** * 否则将spillWriters对应的file中的数据，通过getReader对应UnsafeSorterSpillReader的 * 读取器反序列化到UnsafeSorterIterator中，然后到添加到queue队列中 * UnsafeSorterSpillReader也是UnsafeSorterIterator的子类 */ LinkedList queue = new LinkedList(); for (UnsafeSorterSpillWriter spillWriter : spillWriters) { queue.add(spillWriter.getReader(serializerManager)); } if (inMemSorter != null) { queue.add(inMemSorter.getSortedIterator()); } /** * ChainedIterator是一个UnsafeSorterIterator的子类 * 实现将将多个UnsafeSorterIterator合成一个UnsafeSorterIterator * 提供给应用使用 */ return new ChainedIterator(queue); } } /** * Chain multiple UnsafeSorterIterator together as single one. */ static class ChainedIterator extends UnsafeSorterIterator { private final Queue iterators; private UnsafeSorterIterator current; private int numRecords; ChainedIterator(Queue iterators) { assert iterators.size() > 0; this.numRecords = 0; for (UnsafeSorterIterator iter: iterators) { this.numRecords += iter.getNumRecords(); } this.iterators = iterators; this.current = iterators.remove(); } @Override public int getNumRecords() { return numRecords; } @Override public boolean hasNext() { while (!current.hasNext() && !iterators.isEmpty()) { current = iterators.remove(); /**从队列中移除一个已经遍历完的UnsafeSorterIterator*/ } return current.hasNext(); } @Override public void loadNext() throws IOException { while (!current.hasNext() && !iterators.isEmpty()) { current = iterators.remove(); /**从队列中移除一个已经遍历完的UnsafeSorterIterator*/ } current.loadNext(); } @Override public Object getBaseObject() { return current.getBaseObject(); } @Override public long getBaseOffset() { return current.getBaseOffset(); } @Override public int getRecordLength() { return current.getRecordLength(); } @Override public long getKeyPrefix() { return current.getKeyPrefix(); } } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/UnsafeInMemorySorter.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.utils.sort; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.array.LongArray; import io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager; import io.mycat.memory.unsafe.memory.mm.MemoryConsumer; import javax.annotation.Nullable; import java.util.Comparator; /** * Sorts records using an AlphaSort-style key-prefix sort. This sort stores pointers to records * alongside a user-defined prefix of the record's sorting key. When the underlying sort algorithm * compares records, it will first compare the stored key prefixes; if the prefixes are not equal, * then we do not need to traverse the record pointers to compare the actual records. Avoiding these * random memory accesses improves cache hit rates. */ public final class UnsafeInMemorySorter { private static final class SortComparator implements Comparator { private final RecordComparator recordComparator; private final PrefixComparator prefixComparator; private final DataNodeMemoryManager memoryManager; SortComparator( RecordComparator recordComparator, PrefixComparator prefixComparator, DataNodeMemoryManager memoryManager) { this.recordComparator = recordComparator; this.prefixComparator = prefixComparator; this.memoryManager = memoryManager; } @Override public int compare(RecordPointerAndKeyPrefix r1, RecordPointerAndKeyPrefix r2) { final int prefixComparisonResult = prefixComparator.compare(r1.keyPrefix, r2.keyPrefix); if (prefixComparisonResult == 0) { final Object baseObject1 = memoryManager.getPage(r1.recordPointer); final long baseOffset1 = memoryManager.getOffsetInPage(r1.recordPointer) + 4; // skip length final Object baseObject2 = memoryManager.getPage(r2.recordPointer); final long baseOffset2 = memoryManager.getOffsetInPage(r2.recordPointer) + 4; // skip length return recordComparator.compare(baseObject1, baseOffset1, baseObject2, baseOffset2); } else { return prefixComparisonResult; } } } private final MemoryConsumer consumer; private final DataNodeMemoryManager memoryManager; @Nullable private final Sorter sorter; @Nullable private final Comparator sortComparator; /** * If non-null, specifies the radix sort parameters and that radix sort will be used. */ @Nullable private final PrefixComparators.RadixSortSupport radixSortSupport; /** * Set to 2x for radix sort to reserve extra memory for sorting, otherwise 1x. */ private final int memoryAllocationFactor; /** * Within this buffer, position {@code 2 * i} holds a pointer pointer to the record at * index {@code i}, while position {@code 2 * i + 1} in the array holds an 8-byte key prefix. */ private LongArray array; /** * The position in the sort buffer where new records can be inserted. */ private int pos = 0; private long initialSize; private long totalSortTimeNanos = 0L; private boolean enableSort = true; public UnsafeInMemorySorter( final MemoryConsumer consumer, final DataNodeMemoryManager memoryManager, final RecordComparator recordComparator, final PrefixComparator prefixComparator, long initialSize, boolean canUseRadixSort,boolean enableSort) { this(consumer, memoryManager, recordComparator, prefixComparator, consumer.allocateLongArray(initialSize * 2), canUseRadixSort,enableSort); } public UnsafeInMemorySorter( final MemoryConsumer consumer, final DataNodeMemoryManager memoryManager, final RecordComparator recordComparator, final PrefixComparator prefixComparator, LongArray array, boolean canUseRadixSort, boolean enableSort) { this.consumer = consumer; this.memoryManager = memoryManager; this.initialSize = array.size(); if (recordComparator != null) { this.sorter = new Sorter(UnsafeSortDataFormat.INSTANCE); this.sortComparator = new SortComparator(recordComparator, prefixComparator, memoryManager); if (canUseRadixSort && prefixComparator instanceof PrefixComparators.RadixSortSupport) { this.radixSortSupport = (PrefixComparators.RadixSortSupport)prefixComparator; } else { this.radixSortSupport = null; } } else { this.sorter = null; this.sortComparator = null; this.radixSortSupport = null; } this.enableSort = enableSort; this.memoryAllocationFactor = this.radixSortSupport != null ? 2 : 1; this.array = array; } /** * Free the memory used by pointer array. */ public void free() { if (consumer != null) { consumer.freeLongArray(array); array = null; } } public void reset() { if (consumer != null) { consumer.freeLongArray(array); this.array = consumer.allocateLongArray(initialSize); } pos = 0; } /** * @return the number of records that have been inserted into this sorter. */ public int numRecords() { return pos / 2; } /** * @return the total amount of time spent sorting data (in-memory only). */ public long getSortTimeNanos() { return totalSortTimeNanos; } public long getMemoryUsage() { return array.size() * 8; } public boolean hasSpaceForAnotherRecord() { return pos + 1 < (array.size() / memoryAllocationFactor); } public void expandPointerArray(LongArray newArray) { if (newArray.size() < array.size()) { throw new OutOfMemoryError("Not enough memory to grow pointer array"); } Platform.copyMemory( array.getBaseObject(), array.getBaseOffset(), newArray.getBaseObject(), newArray.getBaseOffset(), array.size() * (8 / memoryAllocationFactor)); consumer.freeLongArray(array); array = newArray; } /** * Inserts a record to be sorted. Assumes that the record pointer points to a record length * stored as a 4-byte integer, followed by the record's bytes. * * @param recordPointer pointer to a record in a data page, encoded by {@link DataNodeMemoryManager}. * @param keyPrefix a user-defined key prefix */ public void insertRecord(long recordPointer, long keyPrefix) { if (!hasSpaceForAnotherRecord()) { throw new IllegalStateException("There is no space for new record"); } /** * 先插入recordPointer，然后插入keyPrefix值 * */ array.set(pos, recordPointer); pos++; array.set(pos, keyPrefix); pos++; } public final class SortedIterator extends UnsafeSorterIterator implements Cloneable { private final int numRecords; private int position; private int offset; private Object baseObject; private long baseOffset; private long keyPrefix; private int recordLength; private SortedIterator(int numRecords, int offset) { this.numRecords = numRecords; this.position = 0; this.offset = offset; } public SortedIterator clone() { SortedIterator iter = new SortedIterator(numRecords, offset); iter.position = position; iter.baseObject = baseObject; iter.baseOffset = baseOffset; iter.keyPrefix = keyPrefix; iter.recordLength = recordLength; return iter; } @Override public int getNumRecords() { return numRecords; } @Override public boolean hasNext() { return position / 2 < numRecords; } /** * 更新迭代器相关指针信息，和当前记录内容的大小 */ @Override public void loadNext() { // This pointer points to a 4-byte record length, followed by the record's bytes final long recordPointer = array.get(offset + position); baseObject = memoryManager.getPage(recordPointer); baseOffset = memoryManager.getOffsetInPage(recordPointer) + 4; // Skip over record length recordLength = Platform.getInt(baseObject, baseOffset - 4); keyPrefix = array.get(offset + position + 1); position += 2; } @Override public Object getBaseObject() { return baseObject; } @Override public long getBaseOffset() { return baseOffset; } @Override public int getRecordLength() { return recordLength; } @Override public long getKeyPrefix() { return keyPrefix; } } /** * Return an iterator over record pointers in sorted order. For efficiency, all calls to * {@code next()} will return the same mutable object. */ public SortedIterator getSortedIterator() { int offset = 0; long start = System.nanoTime(); if (sorter != null && enableSort) { if (this.radixSortSupport != null) { // TODO(ekl) we should handle NULL values before radix sort for efficiency, since they // force a full-width sort (and we cannot radix-sort nullable long fields at all). offset = RadixSort.sortKeyPrefixArray(array, pos / 2, 0, 7, radixSortSupport.sortDescending(),radixSortSupport.sortSigned()); } else { sorter.sort(array,0,pos / 2,sortComparator); } } totalSortTimeNanos += System.nanoTime() - start; return new SortedIterator(pos / 2, offset); } /** * Return an iterator over record pointers int not sorted order. For efficiency, all calls to * {@code next()} will return the same mutable object. */ public SortedIterator getIterator() { return new SortedIterator(pos / 2,0); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/UnsafeKVExternalSorter.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.utils.sort; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Ordering; import io.mycat.memory.unsafe.KVIterator; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.map.BytesToBytesMap; import io.mycat.memory.unsafe.memory.MemoryBlock; import io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager; import io.mycat.memory.unsafe.row.StructType; import io.mycat.memory.unsafe.row.UnsafeRow; import io.mycat.memory.unsafe.storage.DataNodeDiskManager; import io.mycat.memory.unsafe.storage.SerializerManager; import javax.annotation.Nullable; import java.io.IOException; /** * A class for performing external sorting on key-value records. Both key and value are UnsafeRows. * * Note that this class allows optionally passing in a {@link io.mycat.memory.unsafe.map.BytesToBytesMap} directly in order * to perform in-place sorting of records in the map. */ public final class UnsafeKVExternalSorter { private final StructType keySchema; private final StructType valueSchema; private final UnsafeExternalRowSorter.PrefixComputer prefixComputer; private final UnsafeExternalSorter sorter; public UnsafeKVExternalSorter( StructType keySchema, StructType valueSchema, DataNodeDiskManager blockManager, SerializerManager serializerManager, long pageSizeBytes) throws IOException { this(keySchema, valueSchema, blockManager, serializerManager, pageSizeBytes, null); } public UnsafeKVExternalSorter( StructType keySchema, StructType valueSchema, DataNodeDiskManager blockManager, SerializerManager serializerManager, long pageSizeBytes, @Nullable BytesToBytesMap map) throws IOException { this.keySchema = keySchema; this.valueSchema = valueSchema; /** * 排序的key，根据前缀排序规则，进行比较 * */ prefixComputer = SortPrefixUtils.createPrefixGenerator(keySchema); /** * 排序的key，根据前缀排序规则，进行比较 * */ PrefixComparator prefixComparator = SortPrefixUtils.getPrefixComparator(keySchema); /** * */ BaseOrdering ordering = new BaseOrdering(){ @Override public int compare(@Nullable UnsafeRow unsafeRow, @Nullable UnsafeRow t1) { return 1; } }; KVComparator recordComparator = new KVComparator(ordering, keySchema.length()); DataNodeMemoryManager dataNodeMemoryManager = null; if (map == null) { sorter = UnsafeExternalSorter.create( dataNodeMemoryManager, blockManager, serializerManager, recordComparator, prefixComparator, 4096, pageSizeBytes, keySchema.length() == 1 && SortPrefixUtils.canSortFullyWithPrefix(keySchema.apply(0)),true); } else { // During spilling, the array in map will not be used, so we can borrow that and use it // as the underline array for in-memory sorter (it's always large enough). // Since we will not grow the array, it's fine to pass `null` as consumer. /** * map.getArray()获取longArray指针数组 */ final UnsafeInMemorySorter inMemSorter = new UnsafeInMemorySorter( null, dataNodeMemoryManager, recordComparator, prefixComparator, map.getArray(), false /* TODO(ekl) we can only radix sort if the BytesToBytes load factor is <= 0.5 */,true); // We cannot use the destructive iterator here because we are reusing the existing memory // pages in BytesToBytesMap to hold records during sorting. // The only new memory we are allocating is the pointer/prefix array. BytesToBytesMap.MapIterator iter = map.iterator(); final int numKeyFields = keySchema.length(); UnsafeRow row = new UnsafeRow(numKeyFields); while (iter.hasNext()) { final BytesToBytesMap.Location loc = iter.next(); final Object baseObject = loc.getKeyBase(); final long baseOffset = loc.getKeyOffset(); // Get encoded memory address // baseObject + baseOffset point to the beginning of the key data in the map, but that // the KV-pair's length data is stored in the word immediately before that address MemoryBlock page = loc.getMemoryPage(); long address = dataNodeMemoryManager.encodePageNumberAndOffset(page, baseOffset - 8); // Compute prefix row.pointTo(baseObject, baseOffset, loc.getKeyLength()); final long prefix = prefixComputer.computePrefix(row); inMemSorter.insertRecord(address, prefix); } sorter = UnsafeExternalSorter.createWithExistingInMemorySorter( dataNodeMemoryManager, blockManager, serializerManager, new KVComparator(ordering, keySchema.length()), prefixComparator, 4096, pageSizeBytes, inMemSorter,true); // reset the map, so we can re-use it to insert new records. the inMemSorter will not used // anymore, so the underline array could be used by map again. map.reset(); } } /** * Inserts a key-value record into the sorter. If the sorter no longer has enough memory to hold * the record, the sorter sorts the existing records in-memory, writes them out as partially * sorted runs, and then reallocates memory to hold the new record. */ public void insertKV(UnsafeRow key, UnsafeRow value) throws IOException { final long prefix = prefixComputer.computePrefix(key); sorter.insertKVRecord( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), value.getBaseObject(), value.getBaseOffset(), value.getSizeInBytes(), prefix); } /** * Merges another UnsafeKVExternalSorter into `this`, the other one will be emptied. * * @throws IOException */ public void merge(UnsafeKVExternalSorter other) throws IOException { sorter.merge(other.sorter); } /** * Returns a sorted iterator. It is the caller's responsibility to call `cleanupResources()` * after consuming this iterator. */ public KVSorterIterator sortedIterator() throws IOException { try { final UnsafeSorterIterator underlying = sorter.getSortedIterator(); if (!underlying.hasNext()) { // Since we won't ever call next() on an empty iterator, we need to clean up resources // here in order to prevent memory leaks. cleanupResources(); } return new KVSorterIterator(underlying); } catch (IOException e) { cleanupResources(); throw e; } } /** * Return the total number of bytes that has been spilled into disk so far. */ public long getSpillSize() { return sorter.getSpillSize(); } /** * Return the peak memory used so far, in bytes. */ public long getPeakMemoryUsedBytes() { return sorter.getPeakMemoryUsedBytes(); } /** * Marks the current page as no-more-space-available, and as a result, either allocate a * new page or spill when we see the next record. */ @VisibleForTesting void closeCurrentPage() { sorter.closeCurrentPage(); } /** * Frees this sorter's in-memory data structures and cleans up its spill files. */ public void cleanupResources() { sorter.cleanupResources(); } private static final class KVComparator extends RecordComparator { private final BaseOrdering ordering; private final UnsafeRow row1; private final UnsafeRow row2; private final int numKeyFields; KVComparator(BaseOrdering ordering, int numKeyFields) { this.numKeyFields = numKeyFields; this.row1 = new UnsafeRow(numKeyFields); this.row2 = new UnsafeRow(numKeyFields); this.ordering = ordering; } @Override public int compare(Object baseObj1, long baseOff1, Object baseObj2, long baseOff2) { // Note that since ordering doesn't need the total length of the record, we just pass -1 // into the row. row1.pointTo(baseObj1, baseOff1 + 4, -1); row2.pointTo(baseObj2, baseOff2 + 4, -1); return ordering.compare(row1, row2); } } public class KVSorterIterator extends KVIterator { private UnsafeRow key = new UnsafeRow(keySchema.length()); private UnsafeRow value = new UnsafeRow(valueSchema.length()); private final UnsafeSorterIterator underlying; private KVSorterIterator(UnsafeSorterIterator underlying) { this.underlying = underlying; } @Override public boolean next() throws IOException { try { if (underlying.hasNext()) { underlying.loadNext(); Object baseObj = underlying.getBaseObject(); long recordOffset = underlying.getBaseOffset(); int recordLen = underlying.getRecordLength(); // Note that recordLen = keyLen + valueLen + 4 bytes (for the keyLen itself) int keyLen = Platform.getInt(baseObj, recordOffset); int valueLen = recordLen - keyLen - 4; key.pointTo(baseObj, recordOffset + 4, keyLen); value.pointTo(baseObj, recordOffset + 4 + keyLen, valueLen); return true; } else { key = null; value = null; cleanupResources(); return false; } } catch (IOException e) { cleanupResources(); throw e; } } @Override public UnsafeRow getKey() { return key; } @Override public UnsafeRow getValue() { return value; } @Override public void close() { cleanupResources(); } } class BaseOrdering extends Ordering { @Override public int compare(@Nullable UnsafeRow unsafeRow, @Nullable UnsafeRow t1) { throw new UnsupportedOperationException(); } } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/UnsafeKeyValueSorter.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.utils.sort; import io.mycat.memory.unsafe.KVIterator; import io.mycat.memory.unsafe.row.UnsafeRow; import java.io.IOException; public abstract class UnsafeKeyValueSorter { public abstract void insert(UnsafeRow key, UnsafeRow value); public abstract KVIterator sort() throws IOException; } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/UnsafeRowsMerger.java ================================================ package io.mycat.memory.unsafe.utils.sort; import java.io.IOException; import java.util.Comparator; import java.util.PriorityQueue; /** * Created by zagnix on 2016/6/25. */ public final class UnsafeRowsMerger { private int numRecords = 0; private final PriorityQueue priorityQueue; UnsafeRowsMerger( final RecordComparator recordComparator, final PrefixComparator prefixComparator, final int numSpills) { final Comparator comparator = new Comparator() { @Override public int compare(UnsafeSorterIterator left, UnsafeSorterIterator right) { final int prefixComparisonResult = prefixComparator.compare(left.getKeyPrefix(), right.getKeyPrefix()); if (prefixComparisonResult == 0) { return recordComparator.compare( left.getBaseObject(), left.getBaseOffset(), right.getBaseObject(), right.getBaseOffset()); } else { return prefixComparisonResult; } } }; /** * 使用优先级队列实现多个Spill File 合并排序,并且支持已经排序内存记录 * 重新写入一个排序文件中。 */ priorityQueue = new PriorityQueue(numSpills,comparator); } /** * Add an UnsafeSorterIterator to this merger * */ public void addSpillIfNotEmpty(UnsafeSorterIterator iterator) throws IOException { /** * 添加迭代器到priorityQueue中 */ if (iterator.hasNext()) { iterator.loadNext(); priorityQueue.add(iterator); numRecords += iterator.getNumRecords(); } } public int getNumRecords() { return numRecords; } public UnsafeSorterIterator getSortedIterator() throws IOException { return new UnsafeSorterIterator() { private UnsafeSorterIterator spillReader; @Override public int getNumRecords() { return numRecords; } @Override public boolean hasNext() { return !priorityQueue.isEmpty() || (spillReader != null && spillReader.hasNext()); } @Override public void loadNext() throws IOException { if (spillReader != null) { if (spillReader.hasNext()) { spillReader.loadNext(); priorityQueue.add(spillReader); } } spillReader = priorityQueue.remove(); } @Override public Object getBaseObject() { return spillReader.getBaseObject(); } @Override public long getBaseOffset() { return spillReader.getBaseOffset(); } @Override public int getRecordLength() { return spillReader.getRecordLength(); } @Override public long getKeyPrefix() { return spillReader.getKeyPrefix(); } }; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/UnsafeSortDataFormat.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.utils.sort; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.array.LongArray; import io.mycat.memory.unsafe.memory.MemoryBlock; /** * Supports sorting an array of (record pointer, key prefix) pairs. * Used in {@link UnsafeInMemorySorter}. *

* Within each long[] buffer, position {@code 2 * i} holds a pointer pointer to the record at * index {@code i}, while position {@code 2 * i + 1} in the array holds an 8-byte key prefix. */ public final class UnsafeSortDataFormat extends SortDataFormat { public static final UnsafeSortDataFormat INSTANCE = new UnsafeSortDataFormat(); private UnsafeSortDataFormat() { } @Override public RecordPointerAndKeyPrefix getKey(LongArray data, int pos) { // Since we re-use keys, this method shouldn't be called. throw new UnsupportedOperationException(); } @Override public RecordPointerAndKeyPrefix newKey() { return new RecordPointerAndKeyPrefix(); } @Override public RecordPointerAndKeyPrefix getKey(LongArray data, int pos, RecordPointerAndKeyPrefix reuse) { reuse.recordPointer = data.get(pos * 2); reuse.keyPrefix = data.get(pos * 2 + 1); return reuse; } @Override public void swap(LongArray data, int pos0, int pos1) { long tempPointer = data.get(pos0 * 2); long tempKeyPrefix = data.get(pos0 * 2 + 1); data.set(pos0 * 2, data.get(pos1 * 2)); data.set(pos0 * 2 + 1, data.get(pos1 * 2 + 1)); data.set(pos1 * 2, tempPointer); data.set(pos1 * 2 + 1, tempKeyPrefix); } @Override public void copyElement(LongArray src, int srcPos, LongArray dst, int dstPos) { dst.set(dstPos * 2, src.get(srcPos * 2)); dst.set(dstPos * 2 + 1, src.get(srcPos * 2 + 1)); } @Override public void copyRange(LongArray src, int srcPos, LongArray dst, int dstPos, int length) { Platform.copyMemory( src.getBaseObject(), src.getBaseOffset() + srcPos * 16, dst.getBaseObject(), dst.getBaseOffset() + dstPos * 16, length * 16); } @Override public LongArray allocate(int length) { assert (length < Integer.MAX_VALUE / 2) : "Length " + length + " is too large"; return new LongArray(MemoryBlock.fromLongArray(new long[length * 2])); } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/UnsafeSorterIterator.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.utils.sort; import java.io.IOException; public abstract class UnsafeSorterIterator { public abstract boolean hasNext(); public abstract void loadNext() throws IOException; public abstract Object getBaseObject(); public abstract long getBaseOffset(); public abstract int getRecordLength(); public abstract long getKeyPrefix(); public abstract int getNumRecords(); } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/UnsafeSorterSpillMerger.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.utils.sort; import java.io.IOException; import java.util.Comparator; import java.util.PriorityQueue; final class UnsafeSorterSpillMerger { private int numRecords = 0; private final PriorityQueue priorityQueue; UnsafeSorterSpillMerger( final RecordComparator recordComparator, final PrefixComparator prefixComparator, final int numSpills) { final Comparator comparator = new Comparator() { @Override public int compare(UnsafeSorterIterator left, UnsafeSorterIterator right) { final int prefixComparisonResult = prefixComparator.compare(left.getKeyPrefix(), right.getKeyPrefix()); if (prefixComparisonResult == 0) { return recordComparator.compare( left.getBaseObject(), left.getBaseOffset(), right.getBaseObject(), right.getBaseOffset()); } else { return prefixComparisonResult; } } }; /** * 使用优先级队列实现多个Spill File 合并排序,并且支持已经排序内存记录 * 重新写入一个排序文件中。 */ priorityQueue = new PriorityQueue(numSpills,comparator); } /** * Add an UnsafeSorterIterator to this merger * */ public void addSpillIfNotEmpty(UnsafeSorterIterator spillReader) throws IOException { /** * 添加迭代器到priorityQueue中 */ if (spillReader.hasNext()) { // We only add the spillReader to the priorityQueue if it is not empty. We do this to // make sure the hasNext method of UnsafeSorterIterator returned by getSortedIterator // does not return wrong result because hasNext will returns true // at least priorityQueue.size() times. If we allow n spillReaders in the // priorityQueue, we will have n extra empty records in the result of UnsafeSorterIterator. spillReader.loadNext(); priorityQueue.add(spillReader); numRecords += spillReader.getNumRecords(); } } /** * 非常重要的一个排序迭代器 * @return * @throws IOException */ public UnsafeSorterIterator getSortedIterator() throws IOException { return new UnsafeSorterIterator() { /** * 当前迭代器 */ private UnsafeSorterIterator spillReader; @Override public int getNumRecords() { return numRecords; } @Override public boolean hasNext() { return !priorityQueue.isEmpty() || (spillReader != null && spillReader.hasNext()); } @Override public void loadNext() throws IOException { if (spillReader != null) { if (spillReader.hasNext()) { spillReader.loadNext(); /** *添加一个完整迭代器集合给优先级队列， *优先级队列为根据比较器自动调整想要的数据大小 * 每次都将spillReader添加到队列中进行新的调整 * 最后得到最小的元素，为出优先级队列做准备 */ priorityQueue.add(spillReader); } } /** * 出队列，当前spillreader最小的元素出优先级队列 */ spillReader = priorityQueue.remove(); } @Override public Object getBaseObject() { return spillReader.getBaseObject(); } @Override public long getBaseOffset() { return spillReader.getBaseOffset(); } @Override public int getRecordLength() { return spillReader.getRecordLength(); } @Override public long getKeyPrefix() { return spillReader.getKeyPrefix(); } }; } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/UnsafeSorterSpillReader.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.utils.sort; import com.google.common.io.ByteStreams; import com.google.common.io.Closeables; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.storage.ConnectionId; import io.mycat.memory.unsafe.storage.SerializerManager; import java.io.*; /** * Reads spill files written by {@link UnsafeSorterSpillWriter} (see that class for a description * of the file format). */ public final class UnsafeSorterSpillReader extends UnsafeSorterIterator implements Closeable { private InputStream in; private DataInputStream din; // Variables that change with every record read: private int recordLength; private long keyPrefix; private int numRecords; private int numRecordsRemaining; private byte[] arr = new byte[1024 * 1024]; private Object baseObject = arr; private final long baseOffset = Platform.BYTE_ARRAY_OFFSET; public UnsafeSorterSpillReader( SerializerManager serializerManager, File file, ConnectionId blockId) throws IOException { assert (file.length() > 0); final BufferedInputStream bs = new BufferedInputStream(new FileInputStream(file)); try { this.in = serializerManager.wrapForCompression(blockId,bs); this.din = new DataInputStream(this.in); numRecords = numRecordsRemaining = din.readInt(); } catch (IOException e) { Closeables.close(bs, /* swallowIOException = */ true); throw e; } } @Override public int getNumRecords() { return numRecords; } @Override public boolean hasNext() { return (numRecordsRemaining > 0); } @Override public void loadNext() throws IOException { recordLength = din.readInt(); keyPrefix = din.readLong(); if (recordLength > arr.length) { arr = new byte[recordLength]; baseObject = arr; } ByteStreams.readFully(in, arr, 0, recordLength); numRecordsRemaining--; if (numRecordsRemaining == 0) { close(); } } @Override public Object getBaseObject() { return baseObject; } @Override public long getBaseOffset() { return baseOffset; } @Override public int getRecordLength() { return recordLength; } @Override public long getKeyPrefix() { return keyPrefix; } @Override public void close() throws IOException { if (in != null) { try { in.close(); } finally { in = null; din = null; } } } } ================================================ FILE: src/main/java/io/mycat/memory/unsafe/utils/sort/UnsafeSorterSpillWriter.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.utils.sort; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.storage.*; import java.io.File; import java.io.IOException; /** * Spills a list of sorted records to disk. Spill files have the following format: * * [# of records (int)] [[len (int)][prefix (long)][data (bytes)]...] */ public final class UnsafeSorterSpillWriter { static final int DISK_WRITE_BUFFER_SIZE = 1024 * 1024; // Small writes to DiskRowWriter will be fairly inefficient. Since there doesn't seem to // be an API to directly transfer bytes from managed memory to the disk writer, we buffer // data through a byte array. private byte[] writeBuffer = new byte[DISK_WRITE_BUFFER_SIZE]; private final File file; private final ConnectionId conId; private final int numRecordsToWrite; private DiskRowWriter writer; private DataNodeFileManager diskBlockManager; private int numRecordsSpilled = 0; public UnsafeSorterSpillWriter( DataNodeDiskManager blockManager, int fileBufferSize, int numRecordsToWrite) throws IOException { this.diskBlockManager = blockManager.diskBlockManager(); this.conId = diskBlockManager.createTempLocalBlock(); this.file = diskBlockManager.getFile(this.conId); this.numRecordsToWrite = numRecordsToWrite; // Unfortunately, we need a serializer instance in order to construct a DiskRowWriter. // Our write path doesn't actually use this serializer (since we end up calling the `write()` // OutputStream methods), but DiskRowWriter still calls some methods on it. To work // around this, we pass a dummy no-op serializer. writer = blockManager.getDiskWriter(conId, file, DummySerializerInstance.INSTANCE, fileBufferSize/**,writeMetrics*/); // Write the number of records writeIntToBuffer(numRecordsToWrite, 0); writer.write(writeBuffer, 0, 4); } // Based on DataOutputStream.writeLong. private void writeLongToBuffer(long v, int offset) throws IOException { writeBuffer[offset + 0] = (byte)(v >>> 56); writeBuffer[offset + 1] = (byte)(v >>> 48); writeBuffer[offset + 2] = (byte)(v >>> 40); writeBuffer[offset + 3] = (byte)(v >>> 32); writeBuffer[offset + 4] = (byte)(v >>> 24); writeBuffer[offset + 5] = (byte)(v >>> 16); writeBuffer[offset + 6] = (byte)(v >>> 8); writeBuffer[offset + 7] = (byte)(v >>> 0); } // Based on DataOutputStream.writeInt. private void writeIntToBuffer(int v, int offset) throws IOException { writeBuffer[offset + 0] = (byte)(v >>> 24); writeBuffer[offset + 1] = (byte)(v >>> 16); writeBuffer[offset + 2] = (byte)(v >>> 8); writeBuffer[offset + 3] = (byte)(v >>> 0); } /** * Write a record to a spill file. * * @param baseObject the base object / memory page containing the record * @param baseOffset the base offset which points directly to the record data. * @param recordLength the length of the record. * @param keyPrefix a sort key prefix */ public void write( Object baseObject, long baseOffset, int recordLength, long keyPrefix) throws IOException { if (numRecordsSpilled == numRecordsToWrite) { throw new IllegalStateException( "Number of records written exceeded numRecordsToWrite = " + numRecordsToWrite); } else { numRecordsSpilled++; } /** * [# of records (int)] [[len (int)][prefix (long)][data (bytes)]...] * 一条记录在文件中格式 * */ /** * recordLength记录长度 4个bytes */ writeIntToBuffer(recordLength, 0); /** * 排序key，8个bytes */ writeLongToBuffer(keyPrefix, 4); /** * dataRemaining要写的真实数据长度bytes */ int dataRemaining = recordLength; /** * 写buffer剩余的空间 */ int freeSpaceInWriteBuffer = DISK_WRITE_BUFFER_SIZE - 4 - 8; // space used by prefix + len /** *记录在内存中的地址偏移量 */ long recordReadPosition = baseOffset; while (dataRemaining > 0) { /** * 计算本次需要从内存中读取的实际数据，取freeSpaceInWriteBuffer和dataRemaining * 中的最小值 */ final int toTransfer = Math.min(freeSpaceInWriteBuffer, dataRemaining); /** * 执行数据拷贝动作，将baseObject的数据拷贝到writeBuffer中 */ Platform.copyMemory( baseObject,/**srd*/ recordReadPosition,/**offset*/ writeBuffer, /**write dst*/ Platform.BYTE_ARRAY_OFFSET + (DISK_WRITE_BUFFER_SIZE - freeSpaceInWriteBuffer),/**write offset*/ toTransfer); /** * 将writeBuffer中数据写到磁盘中 */ writer.write(writeBuffer, 0, (DISK_WRITE_BUFFER_SIZE - freeSpaceInWriteBuffer) + toTransfer); /** * 读指针移动toTransfer实际写的数据大小 */ recordReadPosition += toTransfer; /** * record还剩下多少数据要写入磁盘中 */ dataRemaining -= toTransfer; /** * 本次WriteBuffer初始化大小初始化为DISK_WRITE_BUFFER_SIZE */ freeSpaceInWriteBuffer = DISK_WRITE_BUFFER_SIZE; } /** * 写剩余数据到磁盘中 */ if (freeSpaceInWriteBuffer < DISK_WRITE_BUFFER_SIZE) { writer.write(writeBuffer, 0, (DISK_WRITE_BUFFER_SIZE - freeSpaceInWriteBuffer)); } /** * writer类中数据统计 */ writer.recordWritten(); } public void close() throws IOException { writer.commitAndClose(); writer = null; writeBuffer = null; } public File getFile() { return file; } public UnsafeSorterSpillReader getReader(SerializerManager serializerManager) throws IOException { return new UnsafeSorterSpillReader(serializerManager, file, conId); } } ================================================ FILE: src/main/java/io/mycat/migrate/BinlogIdleCheck.java ================================================ package io.mycat.migrate; import com.alibaba.fastjson.JSON; import io.mycat.util.ZKUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.sql.SQLException; import java.util.Date; import java.util.List; /** * Created by magicdoom on 2016/12/14. */ public class BinlogIdleCheck implements Runnable { private BinlogStream binlogStream; private static final Logger LOGGER = LoggerFactory.getLogger(BinlogIdleCheck.class); public BinlogIdleCheck(BinlogStream binlogStream) { this.binlogStream = binlogStream; } @Override public void run() { ListmigrateTaskList= binlogStream.getMigrateTaskList(); int sucessSwitchTask=0; int fullSucessSwitchTask=0; String taskPath=null; String dataHost=null; for (MigrateTask migrateTask : migrateTaskList) { String zkPath=migrateTask.getZkpath(); if(taskPath==null){ taskPath=zkPath.substring(0,zkPath.lastIndexOf("/")) ; dataHost=zkPath.substring(zkPath.lastIndexOf("/")+1); } if(migrateTask.isHaserror()||migrateTask.isHasExecute()) { continue; } Date lastDate= migrateTask.getLastBinlogDate(); long diff = (new Date().getTime() - lastDate.getTime())/1000; if((!migrateTask.isHaserror())&&diff>30){ //暂定30秒空闲 则代表增量任务结束，开始切换 sucessSwitchTask=sucessSwitchTask+1; fullSucessSwitchTask=fullSucessSwitchTask+1; }else if(!migrateTask.isHaserror()){ String sql=MigrateUtils.makeCountSql(migrateTask); try { long oldCount=MigrateUtils.execulteCount(sql,migrateTask.getFrom()); long newCount=MigrateUtils.execulteCount(sql,migrateTask.getTo()); if(oldCount!=0) { double percent = newCount / oldCount; if(percent>=0.9) { sucessSwitchTask=sucessSwitchTask+1; } } } catch (SQLException e) { LOGGER.error("error:",e); } catch (IOException e) { LOGGER.error("error:",e); } } } try { byte[] taskNodeStr = ZKUtils.getConnection().getData().forPath(taskPath); System.out.println(new String(taskNodeStr)); TaskNode taskNode = JSON.parseObject(taskNodeStr,TaskNode.class); if(taskNode.getStatus()>=3){ binlogStream.disconnect(); } } catch (Exception e) { LOGGER.error("error:",e); } if(sucessSwitchTask==migrateTaskList.size()){ String childTaskPath=taskPath+"/_prepare/"+dataHost; try { if( ZKUtils.getConnection().checkExists().forPath(childTaskPath)==null) { ZKUtils.getConnection().create().creatingParentsIfNeeded().forPath(childTaskPath); } } catch (Exception e) { LOGGER.error("error:",e); } } //全部空闲后，如果已经开始切换了，则修改每个子任务状态 if(fullSucessSwitchTask==migrateTaskList.size()){ try { TaskNode taskNode=JSON.parseObject(new String( ZKUtils.getConnection().getData().forPath(taskPath),"UTF-8"),TaskNode.class); if(taskNode.getStatus()==2) { for (MigrateTask migrateTask : migrateTaskList) { String zkPath = migrateTask.getZkpath() + "/" + migrateTask.getFrom() + "-" + migrateTask.getTo(); if (ZKUtils.getConnection().checkExists().forPath(zkPath) != null) { TaskStatus taskStatus = JSON.parseObject( new String(ZKUtils.getConnection().getData().forPath(zkPath), "UTF-8"), TaskStatus.class); if (taskStatus.getStatus() == 1) { taskStatus.setStatus(3); ZKUtils.getConnection().setData().forPath(zkPath, JSON.toJSONBytes(taskStatus)); } } } } } catch (Exception e) { LOGGER.error("error:",e); } } } } ================================================ FILE: src/main/java/io/mycat/migrate/BinlogStream.java ================================================ package io.mycat.migrate; import com.alibaba.druid.util.JdbcUtils; import com.github.shyiko.mysql.binlog.BinaryLogClient; import com.github.shyiko.mysql.binlog.event.*; import com.google.common.base.Strings; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.route.function.PartitionByCRC32PreSlot; import io.mycat.server.util.SchemaUtil; import io.mycat.sqlengine.OneRawSQLQueryResultHandler; import io.mycat.sqlengine.SQLJob; import io.mycat.util.DateUtil; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.Serializable; import java.math.BigInteger; import java.nio.charset.Charset; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.util.*; import java.util.concurrent.*; import static io.mycat.util.dataMigrator.DataMigratorUtil.executeQuery; public class BinlogStream { private static Logger logger = LoggerFactory.getLogger(BinlogStream.class); private final String hostname; private final int port; private final String username; private final String password; private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); private BinaryLogClient binaryLogClient; private Charset charset; private long slaveID; private String binglogFile; private long binlogPos; private Set databaseSet = new HashSet<>(); private Map semaphoreMap = new ConcurrentHashMap<>(); private List migrateTaskList; public List getMigrateTaskList() { return migrateTaskList; } public void setMigrateTaskList(List migrateTaskList) { this.migrateTaskList = migrateTaskList; for (MigrateTask migrateTask : migrateTaskList) { databaseSet.add(MigrateUtils.getDatabaseFromDataNode(migrateTask.getFrom())); String dataHostTo = MigrateUtils.getDataHostFromDataNode(migrateTask.getTo()); if (!semaphoreMap.containsKey(dataHostTo)) { int count = Double.valueOf(MycatServer.getInstance().getConfig().getDataHosts().get(dataHostTo).getSource().getSize() * 0.8).intValue(); semaphoreMap.put(dataHostTo, new Semaphore(1)); } } } public long getSlaveID() { return slaveID; } public void setSlaveID(long slaveID) { this.slaveID = slaveID; } public String getBinglogFile() { return binglogFile; } public void setBinglogFile(String binglogFile) { this.binglogFile = binglogFile; } public long getBinlogPos() { return binlogPos; } public void setBinlogPos(long binlogPos) { this.binlogPos = binlogPos; } private volatile boolean groupEventsByTX = true; public BinlogStream(String hostname, int port, String username, String password, Charset charset) { this.hostname = hostname; this.port = port; this.username = username; this.password = password; this.charset = charset; } public void setGroupEventsByTX(boolean groupEventsByTX) { this.groupEventsByTX = groupEventsByTX; } public void connect() throws IOException { initTaskDate(); scheduler.scheduleAtFixedRate(new BinlogIdleCheck(this), 5, 15, TimeUnit.SECONDS); allocateBinaryLogClient().connect(); } private void initTaskDate() { Date curDate = new Date(); for (MigrateTask migrateTask : migrateTaskList) { migrateTask.setLastBinlogDate(curDate); } } public void connect(long timeoutInMilliseconds) throws IOException, TimeoutException { initTaskDate(); scheduler.scheduleAtFixedRate(new BinlogIdleCheck(this), 5, 15, TimeUnit.SECONDS); allocateBinaryLogClient().connect(timeoutInMilliseconds); } private synchronized BinaryLogClient allocateBinaryLogClient() { if (isConnected()) { throw new IllegalStateException("MySQL replication stream is already open"); } binaryLogClient = new BinaryLogClient(hostname, port, username, password); binaryLogClient.setBinlogFilename(getBinglogFile()); binaryLogClient.setBinlogPosition(getBinlogPos()); binaryLogClient.setServerId(getSlaveID()); binaryLogClient.registerEventListener(new DelegatingEventListener()); return binaryLogClient; } public synchronized boolean isConnected() { return binaryLogClient != null && binaryLogClient.isConnected(); } public synchronized void disconnect() throws IOException { if (binaryLogClient != null) { binaryLogClient.disconnect(); binaryLogClient = null; } shutdownAndAwaitTermination(scheduler); } void shutdownAndAwaitTermination(ExecutorService pool) { pool.shutdown(); // Disable new tasks from being submitted try { // Wait a while for existing tasks to terminate if (!pool.awaitTermination(60, TimeUnit.SECONDS)) { pool.shutdownNow(); // Cancel currently executing tasks // Wait a while for tasks to respond to being cancelled if (!pool.awaitTermination(60, TimeUnit.SECONDS)) logger.warn("Pool did not terminate"); } } catch (InterruptedException ie) { // (Re-)Cancel if current thread also interrupted pool.shutdownNow(); // Preserve interrupt status Thread.currentThread().interrupt(); } } private final class DelegatingEventListener implements BinaryLogClient.EventListener { private final Map tablesById = new HashMap(); private final Map>> tablesColumnMap = new HashMap<>(); private boolean transactionInProgress; private String binlogFilename; //当发现ddl语句时 需要更新重新取列名 private Map> loadColumn(String database, String table) { Map> rtn = new HashMap<>(); List> list = null; Connection con = null; try { con = DriverManager.getConnection("jdbc:mysql://" + hostname + ":" + port, username, password); list = executeQuery(con, "select COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE, CHARACTER_SET_NAME from INFORMATION_SCHEMA.COLUMNS where table_name='" + table + "' and TABLE_SCHEMA='" + database + "'"); } catch (SQLException e) { throw new RuntimeException(e); } finally { JdbcUtils.close(con); } for (Map stringObjectMap : list) { BigInteger pos = (BigInteger) stringObjectMap.get("ORDINAL_POSITION"); rtn.put(pos.intValue(), stringObjectMap); } return rtn; } @Override public void onEvent(Event event) { logger.debug("----->migrate binlog event:" + event.toString()); EventType eventType = event.getHeader().getEventType(); switch (eventType) { case TABLE_MAP: TableMapEventData tableMapEventData = event.getData(); tablesById.put(tableMapEventData.getTableId(), tableMapEventData); if (!tablesColumnMap.containsKey(tableMapEventData.getDatabase() + "." + tableMapEventData.getTable())) { tablesColumnMap.put(tableMapEventData.getDatabase() + "." + tableMapEventData.getTable(), loadColumn(tableMapEventData.getDatabase(), tableMapEventData.getTable())); } break; case ROTATE: RotateEventData data = event.getData(); binlogFilename = data.getBinlogFilename(); break; case PRE_GA_WRITE_ROWS: case WRITE_ROWS: case EXT_WRITE_ROWS: handleWriteRowsEvent(event); break; case PRE_GA_UPDATE_ROWS: case UPDATE_ROWS: case EXT_UPDATE_ROWS: handleUpdateRowsEvent(event); break; case PRE_GA_DELETE_ROWS: case DELETE_ROWS: case EXT_DELETE_ROWS: handleDeleteRowsEvent(event); break; case QUERY: if (groupEventsByTX) { QueryEventData queryEventData = event.getData(); String query = queryEventData.getSql(); if ("BEGIN".equals(query)) { transactionInProgress = true; } else if (!query.startsWith("#")) { handleOtherSqlEvent(event); } } break; case XID: if (groupEventsByTX) { transactionInProgress = false; } break; default: // ignore } } private void exeSql(MigrateTask task, String sql) { if (task.isHaserror()) return; task.setHasExecute(true); String dataHostTo = MigrateUtils.getDataHostFromDataNode(task.getTo()); Semaphore semaphore = semaphoreMap.get(dataHostTo); try { semaphore.acquire(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } SqlExecuteListener listener = new SqlExecuteListener(task, sql, BinlogStream.this, semaphore); OneRawSQLQueryResultHandler resultHandler = new OneRawSQLQueryResultHandler(new String[0], listener); resultHandler.setMark("binlog execute"); PhysicalDBNode dn = MycatServer.getInstance().getConfig().getDataNodes().get(task.getTo()); SQLJob sqlJob = new SQLJob(sql, dn.getDatabase(), resultHandler, dn.getDbPool().getSource()); listener.setSqlJob(sqlJob); sqlJob.run(); } private void handleOtherSqlEvent(Event event) { QueryEventData queryEventData = event.getData(); logger.debug("receve sql:", queryEventData.getSql()); SchemaUtil.SchemaInfo schemaInfo = SchemaUtil.parseSchema(queryEventData.getSql()); if (isShouldBeFilter(queryEventData.getDatabase(), schemaInfo.table)) return; String query = queryEventData.getSql(); for (MigrateTask migrateTask : migrateTaskList) { if (schemaInfo.table.equalsIgnoreCase(migrateTask.getTable()) && queryEventData.getDatabase().equalsIgnoreCase(MigrateUtils.getDatabaseFromDataNode(migrateTask.getFrom()))) { exeSql(migrateTask, query); } } } private boolean isShouldBeFilter(String database, String table) { if (Strings.isNullOrEmpty(database)) return true; if (Strings.isNullOrEmpty(table)) return true; if (!databaseSet.contains(database.toLowerCase())) { return true; } for (MigrateTask migrateTask : migrateTaskList) { if (database.equals(MigrateUtils.getDatabaseFromDataNode(migrateTask.getFrom())) && table.equalsIgnoreCase(migrateTask.getTable())) { return false; } } return true; } private void handleWriteRowsEvent(Event event) { WriteRowsEventData eventData = event.getData(); TableMapEventData tableMapEvent = tablesById.get(eventData.getTableId()); if (isShouldBeFilter(tableMapEvent.getDatabase(), tableMapEvent.getTable())) return; Map> xxx = tablesColumnMap.get(tableMapEvent.getDatabase() + "." + tableMapEvent.getTable()); BitSet inculudeColumn = eventData.getIncludedColumns(); StringBuilder sb = new StringBuilder("insert into "); sb.append(tableMapEvent.getTable()); sb.append("("); int size = inculudeColumn.length(); List rows = eventData.getRows(); int slot = -1; for (int i = 0; i < size; i++) { int column = inculudeColumn.nextSetBit(i); Map coumnMap = xxx.get(column + 1); sb.append(coumnMap.get("COLUMN_NAME")); if (i != size - 1) { sb.append(","); } } sb.append(") values "); for (int i = 0; i < rows.size(); i++) { Serializable[] value = rows.get(i); sb.append(" ("); for (int y = 0; y < size; y++) { int column = inculudeColumn.nextSetBit(y); Map coumnMap = xxx.get(column + 1); String dataType = (String) coumnMap.get("DATA_TYPE"); String columnName = (String) coumnMap.get("COLUMN_NAME"); if ("_slot".equalsIgnoreCase(columnName)) { slot = value[y] instanceof BigInteger ? ((BigInteger) value[y]).intValue() : ((Integer) value[y]); } sb.append(convertBinlogValue(value[y], dataType)); if (y != size - 1) { sb.append(","); } } sb.append(")"); if (i != rows.size() - 1) { sb.append(","); } } checkIfExeSql(tableMapEvent, sb, slot); } private void checkIfExeSql(TableMapEventData tableMapEvent, StringBuilder sb, int slot) { for (MigrateTask migrateTask : migrateTaskList) { if (tableMapEvent.getTable().equalsIgnoreCase(migrateTask.getTable()) && tableMapEvent.getDatabase().equalsIgnoreCase(MigrateUtils.getDatabaseFromDataNode(migrateTask.getFrom()))) { for (PartitionByCRC32PreSlot.Range range : migrateTask.getSlots()) { if (range.end >= slot && range.start <= slot) { exeSql(migrateTask, sb.toString()); return; } } } } } private Object convertBinlogValue(Serializable value, String dataType) { if (value instanceof String) { return "'" + ((String) value).replace("'", "\\'") + "'"; } else if (value instanceof byte[]) { //确认编码 return "'" + new String((byte[]) value, charset).replace("'", "\\'") + "'"; } else if (value instanceof Date) { return "'" + dateToString((Date) value, dataType) + "'"; } else if (("date".equalsIgnoreCase(dataType)) && value instanceof Long) { return "'" + dateToStringFromUTC((Long) value) + "'"; //mariadb date } else if ("datetime".equalsIgnoreCase(dataType) && value instanceof Long) { return "'" + datetimeToStringFromUTC((Long) value) + "'"; //mariadb date } else if (("timestamp".equalsIgnoreCase(dataType)) && value instanceof Long) { return "'" + dateToString((Long) value) + "'"; //mariadb date } else { return value; } } private String dateToStringFromUTC(Long date) { DateTime dt = new DateTime(date, DateTimeZone.UTC); return dt.toString(DateUtil.DATE_PATTERN_ONLY_DATE); } private String datetimeToStringFromUTC(Long date) { DateTime dt = new DateTime(date, DateTimeZone.UTC); return dt.toString(DateUtil.DATE_PATTERN_FULL); } private String dateToString(Long date) { DateTime dt = new DateTime(date); return dt.toString(DateUtil.DATE_PATTERN_FULL); } private String dateToString(Date date, String dateType) { if ("timestamp".equalsIgnoreCase(dateType)) { DateTime dt = new DateTime(date); return dt.toString(DateUtil.DATE_PATTERN_FULL); } else if ("datetime".equalsIgnoreCase(dateType)) { DateTime dt = new DateTime(date, DateTimeZone.UTC); return dt.toString(DateUtil.DATE_PATTERN_FULL); } else if ("date".equalsIgnoreCase(dateType)) { DateTime dt = new DateTime(date, DateTimeZone.UTC); return dt.toString(DateUtil.DATE_PATTERN_ONLY_DATE); } else { DateTime dt = new DateTime(date); return dt.toString(DateUtil.DATE_PATTERN_FULL); } } private void handleUpdateRowsEvent(Event event) { UpdateRowsEventData eventData = event.getData(); TableMapEventData tableMapEvent = tablesById.get(eventData.getTableId()); if (isShouldBeFilter(tableMapEvent.getDatabase(), tableMapEvent.getTable())) return; Map> xxx = tablesColumnMap.get(tableMapEvent.getDatabase() + "." + tableMapEvent.getTable()); BitSet inculudeColumn = eventData.getIncludedColumns(); StringBuilder sba = new StringBuilder("update "); sba.append(tableMapEvent.getTable()); sba.append(" set "); int size = inculudeColumn.length(); List> rows = eventData.getRows(); for (Map.Entry row : rows) { StringBuilder sb = new StringBuilder(sba); int slot = -1; Map.Entry rowMap = row; Serializable[] value = rowMap.getValue(); Serializable[] key = rowMap.getKey(); for (int i = 0; i < size; i++) { int column = inculudeColumn.nextSetBit(i); Map coumnMap = xxx.get(column + 1); sb.append(coumnMap.get("COLUMN_NAME")); sb.append("="); String dataType = (String) coumnMap.get("DATA_TYPE"); sb.append(convertBinlogValue(value[i], dataType)); if (i != size - 1) { sb.append(","); } } sb.append(" where "); BitSet includedColumnsBeforeUpdate = eventData.getIncludedColumnsBeforeUpdate(); for (int i = 0; i < size; i++) { int column = includedColumnsBeforeUpdate.nextSetBit(i); Map coumnMap = xxx.get(column + 1); sb.append(coumnMap.get("COLUMN_NAME")); Serializable value1 = key[i]; if (value1 == null) { sb.append(" is null"); } else { sb.append("="); String dataType = (String) coumnMap.get("DATA_TYPE"); sb.append(convertBinlogValue(value1, dataType)); } String columnName = (String) coumnMap.get("COLUMN_NAME"); if ("_slot".equalsIgnoreCase(columnName)) { slot = value1 instanceof BigInteger ? ((BigInteger) value1).intValue() : ((Integer) value1); } if (i != size - 1) { sb.append(" and "); } } checkIfExeSql(tableMapEvent, sb, slot); } } private void handleDeleteRowsEvent(Event event) { DeleteRowsEventData eventData = event.getData(); TableMapEventData tableMapEvent = tablesById.get(eventData.getTableId()); if (isShouldBeFilter(tableMapEvent.getDatabase(), tableMapEvent.getTable())) return; Map> xxx = tablesColumnMap.get(tableMapEvent.getDatabase() + "." + tableMapEvent.getTable()); BitSet inculudeColumn = eventData.getIncludedColumns(); StringBuilder sba = new StringBuilder("delete from "); sba.append(tableMapEvent.getTable()); sba.append(" where "); int size = inculudeColumn.length(); List rows = eventData.getRows(); for (Serializable[] row : rows) { StringBuilder sb = new StringBuilder(sba); Serializable[] value = row; int slot = -1; for (int i = 0; i < size; i++) { int column = inculudeColumn.nextSetBit(i); Map coumnMap = xxx.get(column + 1); sb.append(coumnMap.get("COLUMN_NAME")); Serializable value1 = value[i]; if (value1 == null) { sb.append(" is null"); } else { sb.append("="); String dataType = (String) coumnMap.get("DATA_TYPE"); sb.append(convertBinlogValue(value1, dataType)); } String columnName = (String) coumnMap.get("COLUMN_NAME"); if ("_slot".equalsIgnoreCase(columnName)) { slot = value1 instanceof BigInteger ? ((BigInteger) value1).intValue() : ((Integer) value1); } if (i != size - 1) { sb.append(" and "); } } checkIfExeSql(tableMapEvent, sb, slot); } } } public static void main(String[] args) { BinlogStream stream = new BinlogStream("localhost", 3306, "root", "123", Charset.defaultCharset()); try { stream.setSlaveID(23511); stream.setBinglogFile("mysql-bin.000028"); stream.setBinlogPos(1082); stream.connect(); } catch (IOException e) { e.printStackTrace(); } // String sql="2'aa\"啊啊402"; // System.out.println(sql.replace("'","\\'")); } } ================================================ FILE: src/main/java/io/mycat/migrate/BinlogStreamHoder.java ================================================ package io.mycat.migrate; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; /** * Created by magicdoom on 2016/12/25. */ public class BinlogStreamHoder { public static ConcurrentMap binlogStreamMap=new ConcurrentHashMap<>(); } ================================================ FILE: src/main/java/io/mycat/migrate/MigrateDumpRunner.java ================================================ package io.mycat.migrate; import com.alibaba.druid.util.JdbcUtils; import com.alibaba.fastjson.JSON; import com.google.common.base.Joiner; import com.google.common.base.Strings; import com.google.common.io.Files; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.config.model.DBHostConfig; import io.mycat.config.model.SystemConfig; import io.mycat.memory.environment.OperatingSystem; import io.mycat.route.function.PartitionByCRC32PreSlot.Range; import io.mycat.util.ProcessUtil; import io.mycat.util.StringUtil; import io.mycat.util.ZKUtils; import io.mycat.util.dataMigrator.DataMigratorUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.nio.charset.Charset; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; import static io.mycat.util.dataMigrator.DataMigratorUtil.executeQuery; /** * Created by nange on 2016/12/1. */ public class MigrateDumpRunner implements Runnable { private static final Logger LOGGER = LoggerFactory.getLogger(MigrateDumpRunner.class); private MigrateTask task; private CountDownLatch latch; private AtomicInteger sucessTask; public MigrateDumpRunner(MigrateTask task, CountDownLatch latch, AtomicInteger sucessTask) { this.task = task; this.latch = latch; this.sucessTask = sucessTask; } @Override public void run() { try { String mysqldump = "?mysqldump -h? -P? -u? -p? ? ? --single-transaction -q --default-character-set=utf8mb4 --hex-blob --where=\"?\" --master-data=1 -T \"?\" --fields-enclosed-by=\\\" --fields-terminated-by=, --lines-terminated-by=\\n --fields-escaped-by=\\\\ "; PhysicalDBPool dbPool = MycatServer.getInstance().getConfig().getDataNodes().get(task.getFrom()).getDbPool(); PhysicalDatasource datasource = dbPool.getSources()[dbPool.getActivedIndex()]; DBHostConfig config = datasource.getConfig(); File file = null; String spath = querySecurePath(config); if (Strings.isNullOrEmpty(spath) || "NULL".equalsIgnoreCase(spath) || "empty".equalsIgnoreCase(spath)) { file = new File(SystemConfig.getHomePath() + File.separator + "temp", "dump" + File.separator + task.getFrom() + "_" + task.getTo()); // task.getFrom() + "_" + task.getTo() + Thread.currentThread().getId() + System.currentTimeMillis() + ""); } else { spath += Thread.currentThread().getId() + System.currentTimeMillis(); file = new File(spath); } file.mkdirs(); String encose = OperatingSystem.isWindows() ? "\\" : ""; String finalCmd = DataMigratorUtil .paramsAssignment(mysqldump, "?", "", config.getIp(), String.valueOf(config.getPort()), config.getUser(), config.getPassword(), MigrateUtils.getDatabaseFromDataNode(task.getFrom()), task.getTable(), makeWhere(task), file.getPath()); List args = Arrays.asList("mysqldump", "-h" + config.getIp(), "-P" + String.valueOf(config.getPort()), "-u" + config.getUser(), !StringUtil.isEmpty(config.getPassword()) ? "-p" + config.getPassword() : "", MigrateUtils.getDatabaseFromDataNode(task.getFrom()), task.getTable(), "--single-transaction", "-q", "--default-character-set=utf8mb4", "--hex-blob", "--where=" + makeWhere(task), "--master-data=1", "-T" + file.getPath() , "--fields-enclosed-by=" + encose + "\"", "--fields-terminated-by=,", "--lines-terminated-by=\\n", "--fields-escaped-by=\\\\"); LOGGER.info("migrate 中 mysqldump准备执行命令,如果超长时间没有响应则可能出错"); LOGGER.info(args.toString()); String result = ProcessUtil.execReturnString(args); int logIndex = result.indexOf("MASTER_LOG_FILE='"); int logPosIndex = result.indexOf("MASTER_LOG_POS="); String logFile = result.substring(logIndex + 17, logIndex + 17 + result.substring(logIndex + 17).indexOf("'")); String logPos = result.substring(logPosIndex + 15, logPosIndex + 15 + result.substring(logPosIndex + 15).indexOf(";")); task.setBinlogFile(logFile); task.setPos(Integer.parseInt(logPos)); File dataFile = new File(file, task.getTable() + ".txt"); File sqlFile = new File(file, task.getTable() + ".sql"); if (!sqlFile.exists()) { LOGGER.debug(sqlFile.getAbsolutePath() + "not exists"); } List createTable = Files.readLines(sqlFile, Charset.forName("UTF-8")); LOGGER.info("migrate 中 准备自动创建新的table:"+createTable); exeCreateTableToDn(extractCreateSql(createTable), task.getTo(), task.getTable()); if (dataFile.length() > 0) { loaddataToDn(dataFile, task.getTo(), task.getTable()); } pushMsgToZK(task.getZkpath(), task.getFrom() + "-" + task.getTo(), 1, "sucess", logFile, logPos); DataMigratorUtil.deleteDir(file); sucessTask.getAndIncrement(); } catch (Exception e) { try { pushMsgToZK(task.getZkpath(), task.getFrom() + "-" + task.getTo(), 0, e.getLocalizedMessage(), "", ""); } catch (Exception e1) { } LOGGER.error("error:", e); } finally { latch.countDown(); } } private String extractCreateSql(List lines) { StringBuilder sb = new StringBuilder(); boolean isAdd = false; for (String line : lines) { if (Strings.isNullOrEmpty(line) || line.startsWith("--") || line.startsWith("/*") || line.startsWith("DROP")) { isAdd = false; continue; } if (line.startsWith("CREATE")) { isAdd = true; } if (isAdd) { sb.append(line).append("\n"); } } String rtn = sb.toString(); if (rtn.endsWith(";\n")) { rtn = rtn.substring(0, rtn.length() - 2); } return rtn.replace("CREATE TABLE", "CREATE TABLE IF not EXISTS "); } private void exeCreateTableToDn(String sql, String toDn, String table) throws SQLException { PhysicalDBNode dbNode = MycatServer.getInstance().getConfig().getDataNodes().get(toDn); PhysicalDBPool dbPool = dbNode.getDbPool(); PhysicalDatasource datasource = dbPool.getSources()[dbPool.getActivedIndex()]; DBHostConfig config = datasource.getConfig(); Connection con = null; try { con = DriverManager.getConnection("jdbc:mysql://" + config.getUrl() + "/" + dbNode.getDatabase(), config.getUser(), config.getPassword()); JdbcUtils.execute(con, sql, new ArrayList<>()); } finally { JdbcUtils.close(con); } } private void pushMsgToZK(String rootZkPath, String child, int status, String msg, String binlogFile, String pos) throws Exception { LOGGER.error(msg); String path = rootZkPath + "/" + child; TaskStatus taskStatus = new TaskStatus(); taskStatus.setMsg(msg); taskStatus.setStatus(status); task.setStatus(status); taskStatus.setBinlogFile(binlogFile); taskStatus.setPos(Long.parseLong(pos)); if (ZKUtils.getConnection().checkExists().forPath(path) == null) { ZKUtils.getConnection().create().forPath(path, JSON.toJSONBytes(taskStatus)); } else { ZKUtils.getConnection().setData().forPath(path, JSON.toJSONBytes(taskStatus)); } } private void loaddataToDn(File loaddataFile, String toDn, String table) throws SQLException, IOException { PhysicalDBNode dbNode = MycatServer.getInstance().getConfig().getDataNodes().get(toDn); PhysicalDBPool dbPool = dbNode.getDbPool(); PhysicalDatasource datasource = dbPool.getSources()[dbPool.getActivedIndex()]; DBHostConfig config = datasource.getConfig(); Connection con = null; try { con = DriverManager.getConnection("jdbc:mysql://" + config.getUrl() + "/" + dbNode.getDatabase(), config.getUser(), config.getPassword()); String sql = "load data local infile '" + loaddataFile.getPath().replace("\\", "//") + "' replace into table " + table + " character set 'utf8mb4' fields terminated by ',' enclosed by '\"' ESCAPED BY '\\\\' lines terminated by '\\n'"; JdbcUtils.execute(con, sql, new ArrayList<>()); } catch (Exception e){ try { pushMsgToZK(task.getZkpath(), task.getFrom() + "-" + task.getTo(), 0, e.getLocalizedMessage(), "", ""); } catch (Exception e1) { } } finally { JdbcUtils.close(con); } } private String makeWhere(MigrateTask task) { List whereList = new ArrayList<>(); List slotRanges = task.getSlots(); for (Range slotRange : slotRanges) { if (slotRange.start == slotRange.end) { whereList.add("_slot =" + slotRange.start); } else { whereList.add("(_slot >=" + slotRange.start + " and _slot <=" + slotRange.end + ")"); } } return Joiner.on(" or ").join(whereList); } private static String querySecurePath(DBHostConfig config) { List> list = null; String path = null; Connection con = null; try { con = DriverManager.getConnection("jdbc:mysql://" + config.getUrl(), config.getUser(), config.getPassword()); list = executeQuery(con, "show variables like 'secure_file_priv'"); if (list != null && list.size() == 1) path = (String) list.get(0).get("Value"); } catch (SQLException e) { throw new RuntimeException(e); } finally { JdbcUtils.close(con); } return path; } public static void main(String[] args) { String result = "\n" + "--\n" + "-- Position to start replication or point-in-time recovery from\n" + "--\n" + "\n" + "CHANGE MASTER TO MASTER_LOG_FILE='NANGE-PC-bin.000021', MASTER_LOG_POS=154;\n"; int logIndex = result.indexOf("MASTER_LOG_FILE='"); int logPosIndex = result.indexOf("MASTER_LOG_POS="); String logFile = result.substring(logIndex + 17, logIndex + 17 + result.substring(logIndex + 17).indexOf("'")); String logPos = result.substring(logPosIndex + 15, logPosIndex + 15 + result.substring(logPosIndex + 15).indexOf(";")); System.out.println(logFile + logPos); } } ================================================ FILE: src/main/java/io/mycat/migrate/MigrateMainRunner.java ================================================ package io.mycat.migrate; import com.alibaba.fastjson.JSON; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.config.model.DBHostConfig; import io.mycat.util.ZKUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.charset.Charset; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; /** * Created by magicdoom on 2016/12/8. */ public class MigrateMainRunner implements Runnable { private static final Logger LOGGER = LoggerFactory.getLogger(MigrateMainRunner.class); private String dataHost; private List migrateTaskList; private int timeout; private Charset charset; private boolean forceBinlog; public MigrateMainRunner(String dataHost, List migrateTaskList, int timeout, Charset charset,boolean forceBinlog) { this.dataHost = dataHost; this.migrateTaskList = migrateTaskList; this.timeout = timeout; this.charset = charset; this.forceBinlog = forceBinlog; } @Override public void run() { try{ AtomicInteger sucessTask = new AtomicInteger(0); if (!forceBinlog) { LOGGER.info("migrate 中 进入 mysqldump阶段"); CountDownLatch downLatch = new CountDownLatch(migrateTaskList.size()); for (MigrateTask migrateTask : migrateTaskList) { MycatServer.getInstance().getBusinessExecutor().submit(new MigrateDumpRunner(migrateTask, downLatch, sucessTask)); } try { //modify by jian.xie 需要等到dumprunner执行结束 timeout需要改成用户指定的超时时间@cjw downLatch.await(this.timeout, TimeUnit.MINUTES); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } }else { LOGGER.info("migrate 中 不进入 mysqldump阶段,直接进入binlog stream"); } //同一个dataHost的任务合并执行，避免过多流量浪费 if (forceBinlog||(sucessTask.get() == migrateTaskList.size())) { long binlogFileNum = -1; String binlogFile = ""; long pos = -1; for (MigrateTask migrateTask : migrateTaskList) { if (binlogFileNum == -1) { binlogFileNum = Integer.parseInt(migrateTask.getBinlogFile().substring(migrateTask.getBinlogFile().lastIndexOf(".") + 1)); binlogFile = migrateTask.getBinlogFile(); pos = migrateTask.getPos(); } else { int tempBinlogFileNum = Integer.parseInt(migrateTask.getBinlogFile().substring(migrateTask.getBinlogFile().lastIndexOf(".") + 1)); if (tempBinlogFileNum <= binlogFileNum && migrateTask.getPos() <= pos) { binlogFileNum = tempBinlogFileNum; binlogFile = migrateTask.getBinlogFile(); pos = migrateTask.getPos(); } } } String taskPath = migrateTaskList.get(0).getZkpath(); taskPath = taskPath.substring(0, taskPath.lastIndexOf("/")); String taskID = taskPath.substring(taskPath.lastIndexOf('/') + 1, taskPath.length()); //开始增量数据迁移 PhysicalDBPool dbPool = MycatServer.getInstance().getConfig().getDataHosts().get(dataHost); PhysicalDatasource datasource = dbPool.getSources()[dbPool.getActivedIndex()]; DBHostConfig config = datasource.getConfig(); BinlogStream stream = new BinlogStream(config.getUrl().substring(0, config.getUrl().indexOf(":")), config.getPort(), config.getUser(), config.getPassword(), charset); try { stream.setSlaveID(migrateTaskList.get(0).getSlaveId()); stream.setBinglogFile(binlogFile); stream.setBinlogPos(pos); stream.setMigrateTaskList(migrateTaskList); BinlogStreamHoder.binlogStreamMap.put(taskID, stream); stream.connect(); } catch (IOException e) { LOGGER.error("migrate 中 binlog 连接 异常"); try { TaskNode taskNode = JSON.parseObject(ZKUtils.getConnection().getData().forPath(taskPath), TaskNode.class); taskNode.addException(e.getLocalizedMessage()); ZKUtils.getConnection().setData().forPath(taskPath, JSON.toJSONBytes(taskNode)); } catch (Exception e1) { LOGGER.error("error:", e); } LOGGER.error("error:", e); } } }catch (Exception e){ LOGGER.error("migrate 中 binlog 连接 异常"); e.printStackTrace(); LOGGER.error(e.getLocalizedMessage()); } } } ================================================ FILE: src/main/java/io/mycat/migrate/MigrateTask.java ================================================ package io.mycat.migrate; import io.mycat.route.function.PartitionByCRC32PreSlot; import io.mycat.route.function.PartitionByCRC32PreSlot.Range; import java.io.Serializable; import java.util.ArrayList; import java.util.Date; import java.util.List; /** * Created by magicdoom on 2016/9/15. */ public class MigrateTask implements Serializable { private String from; //from dataNode private String to; // to dataNode private String table; // private List slots=new ArrayList<>(); // crc range private String method; private String fclass=PartitionByCRC32PreSlot.class.getName(); private String schema; private int slaveId; private transient String zkpath; //mycat/mycat-cluster-1/migrate/TESTDB/411c53ae7da84e418aed1a3909926933/localhost1 // private transient String binlogFile; private transient int pos; private transient volatile Date lastBinlogDate; private transient volatile boolean haserror=false; private transient volatile int status; private transient volatile boolean hasExecute=false; public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public boolean isHaserror() { return haserror; } public void setHaserror(boolean haserror) { this.haserror = haserror; } public List getSlots() { return slots; } public void setSlots(List slots) { this.slots = slots; } public int getSize() { int size=0; for (Range slot : slots) { size=size+slot.size; } return size; } public boolean isHasExecute() { return hasExecute; } public void setHasExecute(boolean hasExecute) { this.hasExecute = hasExecute; } public String getBinlogFile() { return binlogFile; } public void setBinlogFile(String binlogFile) { this.binlogFile = binlogFile; } public int getPos() { return pos; } public void setPos(int pos) { this.pos = pos; } public String getFrom() { return from; } public Date getLastBinlogDate() { return lastBinlogDate; } public void setLastBinlogDate(Date lastBinlogDate) { this.lastBinlogDate = lastBinlogDate; } public void setFrom(String from) { this.from = from; } public String getTo() { return to; } public void setTo(String to) { this.to = to; } public String getTable() { return table; } public void setTable(String table) { this.table = table; } public String getMethod() { return method; } public void setMethod(String method) { this.method = method; } public String getFclass() { return fclass; } public void setFclass(String fclass) { this.fclass = fclass; } public String getSchema() { return schema; } public void setSchema(String schema) { this.schema = schema; } public int getSlaveId() { return slaveId; } public void setSlaveId(int slaveId) { this.slaveId = slaveId; } public String getZkpath() { return zkpath; } public void setZkpath(String zkpath) { this.zkpath = zkpath; } public void addSlots(Range range) { slots.add(range); } public void addSlots(List ranges) { slots.addAll(ranges); } } ================================================ FILE: src/main/java/io/mycat/migrate/MigrateTaskWatch.java ================================================ package io.mycat.migrate; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.google.common.base.Splitter; import com.google.common.base.Strings; import com.google.common.collect.Lists; import io.mycat.MycatServer; import io.mycat.config.loader.zkprocess.comm.ZkConfig; import io.mycat.config.loader.zkprocess.comm.ZkParamCfg; import io.mycat.util.ZKUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.nio.charset.Charset; import java.util.*; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; /** * ......./migrate/schemal/taskid/datahost [任务数据] * Created by magicdoom on 2016/9/28. */ public class MigrateTaskWatch { private static final Logger LOGGER = LoggerFactory.getLogger(MigrateTaskWatch.class); public static void start() { String migratePath = ZKUtils.getZKBasePath() + "migrate"; // modify by jian.xie,cjw,zwy 如果migrate 启动的时候不存在，无法监听，需要这里监听一次 // 如果第一次没有migrate节点这里应该无法使用集群 还需优化 try { CuratorFramework client = ZKUtils.getConnection(); if (client.checkExists().forPath(migratePath) == null) { client.create().creatingParentsIfNeeded().forPath(migratePath); } }catch (Exception e){ throw new RuntimeException(e); } ZKUtils.addChildPathCache(migratePath, new PathChildrenCacheListener() { @Override public void childEvent(CuratorFramework curatorFramework, PathChildrenCacheEvent fevent) throws Exception { switch (fevent.getType()) { case CHILD_ADDED: LOGGER.info("table CHILD_ADDED: " + fevent.getData().getPath()); ZKUtils.addChildPathCache(fevent.getData().getPath(), new TaskPathChildrenCacheListener()); break; default: break; } } }); } private static class TaskPathChildrenCacheListener implements PathChildrenCacheListener { @Override public void childEvent(CuratorFramework curatorFramework, PathChildrenCacheEvent event) throws Exception { switch (event.getType()) { case CHILD_ADDED: if (isTaskErrorOrSucess(event)) break; addOrUpdate(event); String path = event.getData().getPath() + "/_prepare"; if (curatorFramework.checkExists().forPath(path) == null) { curatorFramework.create().creatingParentsIfNeeded().forPath(path); } ZKUtils.addChildPathCache(path, new SwitchPrepareListener()); String commitPath = event.getData().getPath() + "/_commit"; if (curatorFramework.checkExists().forPath(commitPath) == null) { curatorFramework.create().creatingParentsIfNeeded().forPath(commitPath); } ZKUtils.addChildPathCache(commitPath, new SwitchCommitListener()); String cleanPath = event.getData().getPath() + "/_clean"; if (curatorFramework.checkExists().forPath(cleanPath) == null) { curatorFramework.create().creatingParentsIfNeeded().forPath(cleanPath); } ZKUtils.addChildPathCache(cleanPath, new SwitchCleanListener()); LOGGER.info("table CHILD_ADDED: " + event.getData().getPath()); break; case CHILD_UPDATED: if (isTaskErrorOrSucess(event)) break; addOrUpdate(event); LOGGER.info("CHILD_UPDATED: " + event.getData().getPath()); break; default: break; } } private boolean isTaskErrorOrSucess(PathChildrenCacheEvent event) { try { TaskNode pTaskNode = JSON.parseObject(event.getData().getData(), TaskNode.class); if (pTaskNode.getStatus() >= 4) { return true; } } catch (Exception e) { } return false; } private void addOrUpdate(PathChildrenCacheEvent event) throws Exception { InterProcessMutex taskLock = null; try { String tpath = event.getData().getPath(); String taskID = tpath.substring(tpath.lastIndexOf("/") + 1, tpath.length()); String lockPath = ZKUtils.getZKBasePath() + "lock/" + taskID + ".lock"; taskLock = new InterProcessMutex(ZKUtils.getConnection(), lockPath); taskLock.acquire(2000, TimeUnit.SECONDS); String text = new String(ZKUtils.getConnection().getData().forPath(event.getData().getPath()), "UTF-8"); // /migrate/taskId/* 所有的数据 List dataNodeList = ZKUtils.getConnection().getChildren().forPath(event.getData().getPath()); if (!dataNodeList.isEmpty()) { if ((!Strings.isNullOrEmpty(text)) && text.startsWith("{")) { TaskNode taskNode = JSON.parseObject(text, TaskNode.class); if (taskNode.getStatus() == 0) { String boosterDataHosts = ZkConfig.getInstance().getValue(ZkParamCfg.MYCAT_BOOSTER_DATAHOSTS); Set dataNodes = new HashSet<>(Splitter.on(",").trimResults().omitEmptyStrings().splitToList(boosterDataHosts)); List finalMigrateList = new ArrayList<>(); for (String s : dataNodeList) { if ("_prepare".equals(s)) continue; if (dataNodes.contains(s)) { String zkpath = event.getData().getPath() + "/" + s; String data = new String(ZKUtils.getConnection().getData().forPath(zkpath), "UTF-8"); List migrateTaskList = JSONArray.parseArray(data, MigrateTask.class); for (MigrateTask migrateTask : migrateTaskList) { migrateTask.setZkpath(zkpath); } finalMigrateList.addAll(migrateTaskList); } } Map> taskMap = mergerTaskForDataHost(finalMigrateList); for (Map.Entry> stringListEntry : taskMap.entrySet()) { String key = stringListEntry.getKey(); List value = stringListEntry.getValue(); MycatServer.getInstance().getBusinessExecutor().submit(new MigrateMainRunner(key, value,taskNode.getTimeout(), Charset.forName(taskNode.getCharset()),taskNode.isForceBinlog())); } // taskNode.setStatus(1); ZKUtils.getConnection().setData().forPath(event.getData().getPath(), JSON.toJSONBytes(taskNode)); } else if (taskNode.getStatus() == 2) { //start switch ScheduledExecutorService scheduledExecutorService = MycatServer.getInstance().getScheduler(); Set allRunnerSet = SwitchPrepareCheckRunner.allSwitchRunnerSet; if (!allRunnerSet.contains(taskID)) { List dataHosts = ZKUtils.getConnection().getChildren().forPath(tpath); List allTaskList = MigrateUtils.queryAllTask(tpath, removeStatusNode(dataHosts)); allRunnerSet.add(taskID); scheduledExecutorService.schedule(new SwitchPrepareCheckRunner(taskID, tpath, taskNode, MigrateUtils.convertAllTask(allTaskList)), 1, TimeUnit.SECONDS); } } } } } finally { if (taskLock != null) { taskLock.release(); } } } private List removeStatusNode(List dataHosts) { List resultList = new ArrayList<>(); for (String dataHost : dataHosts) { if ("_prepare".equals(dataHost) || "_commit".equals(dataHost) || "_clean".equals(dataHost)) { continue; } resultList.add(dataHost); } return resultList; } private static String getDataHostNameFromNode(String dataNode) { return MycatServer.getInstance().getConfig().getDataNodes().get(dataNode).getDbPool().getHostName(); } //将所有有相同的来源的dataNode放置到一个任务当中。 private static Map> mergerTaskForDataHost(List migrateTaskList) { Map> taskMap = new HashMap<>(); for (MigrateTask migrateTask : migrateTaskList) { String dataHost = getDataHostNameFromNode(migrateTask.getFrom()); if (taskMap.containsKey(dataHost)) { taskMap.get(dataHost).add(migrateTask); } else { taskMap.put(dataHost, Lists.newArrayList(migrateTask)); } } return taskMap; } } } ================================================ FILE: src/main/java/io/mycat/migrate/MigrateUtils.java ================================================ package io.mycat.migrate; import com.alibaba.druid.util.JdbcUtils; import com.alibaba.fastjson.JSON; import com.google.common.base.Joiner; import com.google.common.base.Splitter; import io.mycat.MycatServer; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.config.model.DBHostConfig; import io.mycat.util.ZKUtils; import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import static io.mycat.route.function.PartitionByCRC32PreSlot.Range; /** * Created by magicdoom on 2016/9/16. */ public class MigrateUtils { /** * 扩容计算，以每一个源节点到一个目标节点为一个任务 * * @param table * @param integerListMap 会进行修改，所以传入前请自己clone一份 * @param oldDataNodes * @param newDataNodes * @param slotsTotalNum * @return */ public static Map> balanceExpand(String table, Map> integerListMap, List oldDataNodes, List newDataNodes, int slotsTotalNum) { int newNodeSize = oldDataNodes.size() + newDataNodes.size(); int newSlotPerNode = slotsTotalNum / newNodeSize; Map> newNodeTask = new HashMap<>(); int gb = slotsTotalNum - newSlotPerNode * (newNodeSize); for (int i = 0; i < integerListMap.size(); i++) { List rangeList = integerListMap.get(i); int needMoveNum = getCurTotalSize(rangeList) - newSlotPerNode; List allMoveList = getPartAndRemove(rangeList, needMoveNum); for (int i1 = 0; i1 < newDataNodes.size(); i1++) { String newDataNode = newDataNodes.get(i1); if (allMoveList.size() == 0) break; List curRangeList = newNodeTask.get(newDataNode); if (curRangeList == null) curRangeList = new ArrayList<>(); int hasSlots = getCurTotalSizeForTask(curRangeList); int needMove = (i1 == 0) ? newSlotPerNode - hasSlots + gb : newSlotPerNode - hasSlots; if (needMove > 0) { List moveList = getPartAndRemove(allMoveList, needMove); MigrateTask task = new MigrateTask(); if (i >= oldDataNodes.size()) { throw new RuntimeException(String.format("crc32slot_%s.properties does not match schema table dataNode.", table.toUpperCase())); } task.setFrom(oldDataNodes.get(i)); task.setTo(newDataNode); task.setTable(table); task.setSlots(moveList); curRangeList.add(task); newNodeTask.put(newDataNode, curRangeList); } } if (allMoveList.size() > 0) { throw new RuntimeException("some slot has not moved to"); } } return newNodeTask; } private static List getPartAndRemove(List rangeList, int size) { List result = new ArrayList<>(); for (int i = 0; i < rangeList.size(); i++) { Range range = rangeList.get(i); if (range == null) continue; if (range.size == size) { result.add(new Range(range.start, range.end)); rangeList.set(i, null); break; } else if (range.size < size) { result.add(new Range(range.start, range.end)); size = size - range.size; rangeList.set(i, null); } else if (range.size > size) { result.add(new Range(range.start, range.start + size - 1)); rangeList.set(i, new Range(range.start + size, range.end)); break; } } for (int i = rangeList.size() - 1; i >= 0; i--) { Range range = rangeList.get(i); if (range == null) rangeList.remove(i); } return result; } private static int getCurTotalSizeForTask(List rangeList) { int size = 0; for (MigrateTask task : rangeList) { size = size + getCurTotalSize(task.getSlots()); } return size; } public static List removeAndGetRemain(List oriRangeList, List rangeList) { for (Range range : rangeList) { oriRangeList = removeAndGetRemain(oriRangeList, range); } return oriRangeList; } private static List removeAndGetRemain(List oriRangeList, Range newRange) { List result = new ArrayList<>(); for (Range range : oriRangeList) { result.addAll(removeAndGetRemain(range, newRange)); } return result; } private static List removeAndGetRemain(Range oriRange, Range newRange) { List result = new ArrayList<>(); if (newRange.start > oriRange.end || newRange.end < oriRange.start) { result.add(oriRange); } else if (newRange.start <= oriRange.start && newRange.end >= oriRange.end) { return result; } else if (newRange.start > oriRange.start && newRange.end < oriRange.end) { result.add(new Range(oriRange.start, newRange.start - 1)); result.add(new Range(newRange.end + 1, oriRange.end)); } else if (newRange.start <= oriRange.start && newRange.end < oriRange.end) { result.add(new Range(newRange.end + 1, oriRange.end)); } else if (newRange.start > oriRange.start && newRange.end >= oriRange.end) { result.add(new Range(oriRange.start, newRange.start - 1)); } return result; } public static String convertRangeListToString(List rangeList) { List rangeStringList = new ArrayList<>(); for (Range range : rangeList) { if (range.start == range.end) { rangeStringList.add(String.valueOf(range.start)); } else { rangeStringList.add(range.start + "-" + range.end); } } return Joiner.on(',').join(rangeStringList); } public static List convertRangeStringToList(String rangeStr) { List ranges = Splitter.on(",").omitEmptyStrings().trimResults().splitToList(rangeStr); List rangeList = new ArrayList<>(); for (String range : ranges) { List vv = Splitter.on("-").omitEmptyStrings().trimResults().splitToList(range); if (vv.size() == 2) { Range ran = new Range(Integer.parseInt(vv.get(0)), Integer.parseInt(vv.get(1))); rangeList.add(ran); } else if (vv.size() == 1) { Range ran = new Range(Integer.parseInt(vv.get(0)), Integer.parseInt(vv.get(0))); rangeList.add(ran); } else { throw new RuntimeException("load crc32slot datafile error:dn=value=" + range); } } return rangeList; } public static int getCurTotalSize(List rangeList) { int size = 0; for (Range range : rangeList) { size = size + range.size; } return size; } public static String getDatabaseFromDataNode(String dn) { return MycatServer.getInstance().getConfig().getDataNodes().get(dn).getDatabase(); } public static String getDataHostFromDataNode(String dn) { return MycatServer.getInstance().getConfig().getDataNodes().get(dn).getDbPool().getHostName(); } public static List convertAllTask(List allTasks) { List resutlList = new ArrayList<>(); for (MigrateTask allTask : allTasks) { resutlList.addAll(allTask.getSlots()); } return resutlList; } public static List queryAllTask(String basePath, List dataHost) throws Exception { List resutlList = new ArrayList<>(); for (String dataHostName : dataHost) { if ("_prepare".equals(dataHostName) || "_commit".equals(dataHostName) || "_clean".equals(dataHostName)) continue; resutlList.addAll(JSON .parseArray(new String(ZKUtils.getConnection().getData().forPath(basePath + "/" + dataHostName), "UTF-8"), MigrateTask.class)); } return resutlList; } public static String makeCountSql(MigrateTask task) { StringBuilder sb = new StringBuilder(); sb.append("select count(*) as count from "); sb.append(task.getTable()).append(" where "); List slots = task.getSlots(); for (int i = 0; i < slots.size(); i++) { Range range = slots.get(i); if (i != 0) sb.append(" or "); if (range.start == range.end) { sb.append(" _slot=").append(range.start); } else { sb.append(" (_slot>=").append(range.start); sb.append(" and _slot<=").append(range.end).append(")"); } } return sb.toString(); } public static void execulteSql(String sql, String toDn) throws SQLException, IOException { PhysicalDBNode dbNode = MycatServer.getInstance().getConfig().getDataNodes().get(toDn); PhysicalDBPool dbPool = dbNode.getDbPool(); PhysicalDatasource datasource = dbPool.getSources()[dbPool.getActivedIndex()]; DBHostConfig config = datasource.getConfig(); Connection con = null; try { con = DriverManager .getConnection("jdbc:mysql://" + config.getUrl() + "/" + dbNode.getDatabase(), config.getUser(), config.getPassword()); JdbcUtils.execute(con, sql, new ArrayList<>()); } finally { JdbcUtils.close(con); } } public static long execulteCount(String sql, String toDn) throws SQLException, IOException { PhysicalDBNode dbNode = MycatServer.getInstance().getConfig().getDataNodes().get(toDn); PhysicalDBPool dbPool = dbNode.getDbPool(); PhysicalDatasource datasource = dbPool.getSources()[dbPool.getActivedIndex()]; DBHostConfig config = datasource.getConfig(); Connection con = null; try { con = DriverManager.getConnection("jdbc:mysql://" + config.getUrl() + "/" + dbNode.getDatabase(), config.getUser(), config.getPassword()); List> result = JdbcUtils.executeQuery(con, sql, new ArrayList<>()); if (result.size() == 1) { return (long) result.get(0).get("count"); } } finally { JdbcUtils.close(con); } return 0; } } ================================================ FILE: src/main/java/io/mycat/migrate/SqlExecuteListener.java ================================================ package io.mycat.migrate; import com.alibaba.fastjson.JSON; import io.mycat.sqlengine.SQLJob; import io.mycat.sqlengine.SQLQueryResult; import io.mycat.sqlengine.SQLQueryResultListener; import io.mycat.util.ZKUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Map; import java.util.concurrent.Semaphore; /** * Created by nange on 2016/12/13. */ public class SqlExecuteListener implements SQLQueryResultListener>> { private static final Logger LOGGER = LoggerFactory.getLogger(SqlExecuteListener.class); private MigrateTask task; private String sql; private BinlogStream binlogStream; private Semaphore semaphore; private volatile SQLJob sqlJob; public SQLJob getSqlJob() { return sqlJob; } public void setSqlJob(SQLJob sqlJob) { this.sqlJob = sqlJob; } public SqlExecuteListener(MigrateTask task, String sql, BinlogStream binlogStream, Semaphore semaphore) { this.task = task; this.sql = sql; this.binlogStream = binlogStream; this.semaphore = semaphore; } @Override public void onResult(SQLQueryResult> result) { try { if (!result.isSuccess()) { try { task.setHaserror(true); pushMsgToZK(task.getZkpath(), task.getFrom() + "-" + task.getTo(), 2, "sql:" + sql + ";" + result.getErrMsg()); close("sucess"); binlogStream.disconnect(); } catch (Exception e) { LOGGER.error("error:", e); close(e.getMessage()); } } else { close("sucess"); } task.setHasExecute(false); } finally { semaphore.release(); } } private void pushMsgToZK(String rootZkPath, String child, int status, String msg) throws Exception { String path = rootZkPath + "/" + child; TaskStatus taskStatus = new TaskStatus(); taskStatus.setMsg(msg); taskStatus.setStatus(status); task.setStatus(status); if (ZKUtils.getConnection().checkExists().forPath(path) == null) { ZKUtils.getConnection().create().forPath(path, JSON.toJSONBytes(taskStatus)); } else { ZKUtils.getConnection().setData().forPath(path, JSON.toJSONBytes(taskStatus)); } } public void close(String msg) { SQLJob curJob = sqlJob; if (curJob != null) { curJob.teminate(msg); sqlJob = null; } } } ================================================ FILE: src/main/java/io/mycat/migrate/SwitchCleanListener.java ================================================ package io.mycat.migrate; import com.alibaba.fastjson.JSON; import com.google.common.base.Splitter; import io.mycat.config.loader.zkprocess.comm.ZkConfig; import io.mycat.config.loader.zkprocess.comm.ZkParamCfg; import io.mycat.config.loader.zkprocess.zookeeper.ClusterInfo; import io.mycat.route.RouteCheckRule; import io.mycat.util.ZKUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.concurrent.TimeUnit; /** 清理本地的阻止写的规则 slaveID relese create table * Ceated by magicdoom on 2016/12/19. */ public class SwitchCleanListener implements PathChildrenCacheListener { private static final Logger LOGGER = LoggerFactory.getLogger(SwitchCleanListener.class); @Override public void childEvent(CuratorFramework curatorFramework, PathChildrenCacheEvent event) throws Exception { switch (event.getType()) { case CHILD_ADDED: checkSwitch(event); break; default: break; } } private void checkSwitch(PathChildrenCacheEvent event) { InterProcessMutex taskLock =null; try { String path=event.getData().getPath(); String taskPath=path.substring(0,path.lastIndexOf("/_clean/")) ; String taskID=taskPath.substring(taskPath.lastIndexOf('/')+1,taskPath.length()); String lockPath= ZKUtils.getZKBasePath()+"lock/"+taskID+".lock"; List sucessDataHost= ZKUtils.getConnection().getChildren().forPath(path.substring(0,path.lastIndexOf('/'))); TaskNode pTaskNode= JSON.parseObject(ZKUtils.getConnection().getData().forPath(taskPath),TaskNode.class); String custerName = ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_CLUSTERID); ClusterInfo clusterInfo= JSON.parseObject(ZKUtils.getConnection().getData().forPath("/mycat/"+custerName) , ClusterInfo.class); List clusterNodeList= Splitter.on(',').omitEmptyStrings().splitToList(clusterInfo.getClusterNodes()); if(sucessDataHost.size()==clusterNodeList.size()) { RouteCheckRule.migrateRuleMap.remove(pTaskNode.getSchema().toUpperCase()); List needToCloseWatch=new ArrayList<>(); List dataHosts= ZKUtils.getConnection().getChildren().forPath(taskPath); for (String dataHostName : dataHosts) { if ("_prepare".equals(dataHostName) || "_commit".equals(dataHostName) || "_clean".equals(dataHostName)) { needToCloseWatch.add( taskPath+"/"+dataHostName ); } } ZKUtils.closeWatch(needToCloseWatch); taskLock= new InterProcessMutex(ZKUtils.getConnection(), lockPath); taskLock.acquire(20, TimeUnit.SECONDS); TaskNode taskNode= JSON.parseObject(ZKUtils.getConnection().getData().forPath(taskPath),TaskNode.class); if(taskNode.getStatus()==3){ taskNode.setStatus(5); //clean sucess //释放slaveIDs for (String dataHostName : dataHosts) { if("_prepare".equals(dataHostName)||"_commit".equals(dataHostName)||"_clean".equals(dataHostName)) continue; List migrateTaskList= JSON .parseArray(new String(ZKUtils.getConnection().getData().forPath(taskPath+"/"+dataHostName),"UTF-8") ,MigrateTask.class); int slaveId= migrateTaskList.get(0).getSlaveId(); String slavePath=ZKUtils.getZKBasePath()+"slaveIDs/"+dataHostName+"/"+slaveId; if( ZKUtils.getConnection().checkExists().forPath(slavePath)!=null) { ZKUtils.getConnection().delete().forPath(slavePath); } } ZKUtils.getConnection().setData().forPath(taskPath,JSON.toJSONBytes(taskNode)) ; LOGGER.info("task end",new Date()); } } } catch (Exception e) { LOGGER.error("migrate 中 clean 阶段异常"); LOGGER.error("error:",e); } finally { if(taskLock!=null){ try { taskLock.release(); } catch (Exception ignored) { } } } } } ================================================ FILE: src/main/java/io/mycat/migrate/SwitchCommitListener.java ================================================ package io.mycat.migrate; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.google.common.base.Joiner; import com.google.common.base.Splitter; import io.mycat.MycatServer; import io.mycat.config.loader.console.ZookeeperPath; import io.mycat.config.loader.zkprocess.comm.ZkConfig; import io.mycat.config.loader.zkprocess.comm.ZkParamCfg; import io.mycat.config.loader.zkprocess.entity.Rules; import io.mycat.config.loader.zkprocess.entity.Schemas; import io.mycat.config.loader.zkprocess.entity.rule.function.Function; import io.mycat.config.loader.zkprocess.entity.rule.tablerule.TableRule; import io.mycat.config.loader.zkprocess.entity.schema.datahost.DataHost; import io.mycat.config.loader.zkprocess.entity.schema.datanode.DataNode; import io.mycat.config.loader.zkprocess.entity.schema.schema.Schema; import io.mycat.config.loader.zkprocess.entity.schema.schema.Table; import io.mycat.config.loader.zkprocess.parse.XmlProcessBase; import io.mycat.config.loader.zkprocess.parse.entryparse.rule.json.FunctionJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.rule.json.TableRuleJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.rule.xml.RuleParseXmlImpl; import io.mycat.config.loader.zkprocess.parse.entryparse.schema.json.DataHostJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.schema.json.DataNodeJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.schema.json.SchemaJsonParse; import io.mycat.config.loader.zkprocess.parse.entryparse.schema.xml.SchemasParseXmlImpl; import io.mycat.config.loader.zkprocess.zookeeper.ClusterInfo; import io.mycat.config.loader.zkprocess.zookeeper.DataInf; import io.mycat.config.loader.zkprocess.zookeeper.process.ZkDataImpl; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.config.model.TableConfig; import io.mycat.config.model.rule.RuleConfig; import io.mycat.manager.response.ReloadConfig; import io.mycat.route.function.PartitionByCRC32PreSlot.Range; import io.mycat.route.function.TableRuleAware; import io.mycat.util.ZKUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.api.transaction.CuratorTransactionFinal; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.Properties; import java.util.concurrent.TimeUnit; /** * Created by magicdoom on 2016/12/19. */ public class SwitchCommitListener implements PathChildrenCacheListener { private static final Logger LOGGER = LoggerFactory.getLogger(SwitchCommitListener.class); @Override public void childEvent(CuratorFramework curatorFramework, PathChildrenCacheEvent event) throws Exception { switch (event.getType()) { case CHILD_ADDED: checkCommit(event); break; default: break; } } private void checkCommit(PathChildrenCacheEvent event) { InterProcessMutex taskLock = null; try { String path = event.getData().getPath(); String taskPath = path.substring(0, path.lastIndexOf("/_commit/")); String taskID = taskPath.substring(taskPath.lastIndexOf('/') + 1, taskPath.length()); String lockPath = ZKUtils.getZKBasePath() + "lock/" + taskID + ".lock"; List sucessDataHost = ZKUtils.getConnection().getChildren().forPath(path.substring(0, path.lastIndexOf('/'))); String custerName = ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_CLUSTERID); ClusterInfo clusterInfo = JSON.parseObject(ZKUtils.getConnection().getData().forPath("/mycat/" + custerName), ClusterInfo.class); List clusterNodeList = Splitter.on(',').omitEmptyStrings().splitToList(clusterInfo.getClusterNodes()); //等待所有的dataHost都导出数据完毕。 dataHost的数量== booster的数量 //判断条件需要进行修改 todo if (sucessDataHost.size() == clusterNodeList.size()) { List taskDataHost = ZKUtils.getConnection().getChildren().forPath(taskPath); List allTaskList = MigrateUtils.queryAllTask(taskPath, taskDataHost); taskLock = new InterProcessMutex(ZKUtils.getConnection(), lockPath); taskLock.acquire(120, TimeUnit.SECONDS); TaskNode taskNode = JSON.parseObject(ZKUtils.getConnection().getData().forPath(taskPath), TaskNode.class); if (taskNode.getStatus() == 2) { taskNode.setStatus(3); //开始切换 且个节点已经禁止写入并且无原有写入在执行 try { CuratorTransactionFinal transactionFinal = null; check(taskID, allTaskList); SchemaConfig schemaConfig = MycatServer.getInstance().getConfig().getSchemas().get(taskNode.getSchema()); TableConfig tableConfig = schemaConfig.getTables().get(taskNode.getTable().toUpperCase()); List newDataNodes = Splitter.on(",").omitEmptyStrings().trimResults().splitToList(taskNode.getAdd()); List allNewDataNodes = tableConfig.getDataNodes(); allNewDataNodes.addAll(newDataNodes); //先修改rule config InterProcessMutex ruleLock = new InterProcessMutex(ZKUtils.getConnection(), ZKUtils.getZKBasePath() + "lock/rules.lock"); ; try { ruleLock.acquire(30, TimeUnit.SECONDS); //transactionFinal = modifyZkRules(transactionFinal, tableConfig.getRule().getFunctionName(), newDataNodes); transactionFinal = modifyTableConfigRules(transactionFinal, taskNode.getSchema(), taskNode.getTable(), newDataNodes); } finally { ruleLock.release(); } transactionFinal = modifyRuleData(transactionFinal, allTaskList, tableConfig, allNewDataNodes); transactionFinal.setData().forPath(taskPath, JSON.toJSONBytes(taskNode)); clean(taskID, allTaskList); transactionFinal.commit(); forceTableRuleToLocal(taskPath, taskNode); pushACKToClean(taskPath); } catch (Exception e) { taskNode.addException(e.getLocalizedMessage()); //异常to Zk ZKUtils.getConnection().setData().forPath(taskPath, JSON.toJSONBytes(taskNode)); LOGGER.error("error:", e); return; } //todo 清理规则 顺利拉下ruledata保证一定更新到本地 } else if (taskNode.getStatus() == 3) { forceTableRuleToLocal(taskPath, taskNode); pushACKToClean(taskPath); } } } catch (Exception e) { LOGGER.error("migrate 中 commit 阶段异常"); LOGGER.error("error:", e); } finally { if (taskLock != null) { try { taskLock.release(); } catch (Exception ignored) { } } } } private void forceTableRuleToLocal(String path, TaskNode taskNode) throws Exception { Path localPath = Paths.get(this.getClass() .getClassLoader() .getResource(ZookeeperPath.ZK_LOCAL_WRITE_PATH.getKey()).toURI()); // 获得公共的xml转换器对象 XmlProcessBase xmlProcess = new XmlProcessBase(); try { forceTableToLocal(localPath, xmlProcess); forceRulesToLocal(localPath, xmlProcess); //保证先有table再有rule forceRuleDataToLocal(taskNode); ReloadConfig.reload(); LOGGER.error("migrate 中 reload 配置成功"); } catch (Exception e) { taskNode.addException(e.getLocalizedMessage()); //异常to Zk ZKUtils.getConnection().setData().forPath(path, JSON.toJSONBytes(taskNode)); LOGGER.error("migrate 中 强制更新本地文件失败"); LOGGER.error("error:", e); } } private void forceRuleDataToLocal(TaskNode taskNode) throws Exception { SchemaConfig schemaConfig = MycatServer.getInstance().getConfig().getSchemas().get(taskNode.getSchema()); TableConfig tableConfig = schemaConfig.getTables().get(taskNode.getTable().toUpperCase()); RuleConfig ruleConfig = tableConfig.getRule(); String ruleName = ((TableRuleAware) ruleConfig.getRuleAlgorithm()).getRuleName() + ".properties"; String rulePath = ZKUtils.getZKBasePath() + "ruledata/" + ruleName; CuratorFramework zk = ZKUtils.getConnection(); byte[] ruleData = zk.getData().forPath(rulePath); LOGGER.info("-----------------------------------从zookeeper中拉取的新的ruleData的信息--------------------------------------------------\n"); LOGGER.info(new String(ruleData)); Path file = Paths.get(SystemConfig.getHomePath(), "conf", "ruledata"); Files.write(file.resolve(ruleName), ruleData); } private static void forceTableToLocal(Path localPath, XmlProcessBase xmlProcess) throws Exception { // 获得当前集群的名称 String schemaPath = ZKUtils.getZKBasePath(); schemaPath = schemaPath + ZookeeperPath.FOW_ZK_PATH_SCHEMA.getKey() + ZookeeperPath.ZK_SEPARATOR.getKey(); // 生成xml与类的转换信息 SchemasParseXmlImpl schemasParseXml = new SchemasParseXmlImpl(xmlProcess); Schemas schema = new Schemas(); String str = ""; // 得到schema对象的目录信息 str = new String(ZKUtils.getConnection().getData().forPath(schemaPath + ZookeeperPath.FLOW_ZK_PATH_SCHEMA_SCHEMA.getKey())); LOGGER.info("-----------------------------------从zookeeper中拉取的新的schema的信息--------------------------------------------------"); LOGGER.info(str); SchemaJsonParse schemaJsonParse = new SchemaJsonParse(); List schemaList = schemaJsonParse.parseJsonToBean(str); schema.setSchema(schemaList); // 得到dataNode的信息 str = new String(ZKUtils.getConnection().getData().forPath(schemaPath + ZookeeperPath.FLOW_ZK_PATH_SCHEMA_DATANODE.getKey())); LOGGER.info("-----------------------------------从zookeeper中拉取的新的dataNode的信息--------------------------------------------------"); LOGGER.info(str); DataNodeJsonParse dataNodeJsonParse = new DataNodeJsonParse(); List dataNodeList = dataNodeJsonParse.parseJsonToBean(str); schema.setDataNode(dataNodeList); // 得到dataHost的信息 str = new String(ZKUtils.getConnection().getData().forPath(schemaPath + ZookeeperPath.FLOW_ZK_PATH_SCHEMA_DATAHOST.getKey())); LOGGER.info("-----------------------------------从zookeeper中拉取的新的dataHost的信息--------------------------------------------------"); LOGGER.info(str); DataHostJsonParse dataHostJsonParse = new DataHostJsonParse(); List dataHostList = dataHostJsonParse.parseJsonToBean(str); schema.setDataHost(dataHostList); xmlProcess.addParseClass(DataNode.class); xmlProcess.addParseClass(DataHost.class); xmlProcess.addParseClass(Schema.class); xmlProcess.addParseClass(Schemas.class); xmlProcess.initJaxbClass(); schemasParseXml.parseToXmlWrite(schema, localPath.resolve("schema.xml").toString(), "schema"); } private static void forceRulesToLocal(Path localPath, XmlProcessBase xmlProcess) throws Exception { Rules rules = new Rules(); // tablerule信息 String value = new String(ZKUtils.getConnection().getData().forPath(ZKUtils.getZKBasePath() + "rules/tableRule"), "UTF-8"); LOGGER.info("-----------------------------------从zookeeper中拉取的新的tablerule的信息--------------------------------------------------"); LOGGER.info(value); DataInf RulesZkData = new ZkDataImpl("tableRule", value); TableRuleJsonParse tableRuleJsonParse = new TableRuleJsonParse(); List tableRuleData = tableRuleJsonParse.parseJsonToBean(RulesZkData.getDataValue()); rules.setTableRule(tableRuleData); // 得到function信息 String fucValue = new String(ZKUtils.getConnection().getData().forPath(ZKUtils.getZKBasePath() + "rules/function"), "UTF-8"); LOGGER.info("-----------------------------------从zookeeper中拉取的新的function的信息--------------------------------------------------"); LOGGER.info(fucValue); DataInf functionZkData = new ZkDataImpl("function", fucValue); FunctionJsonParse functionJsonParse = new FunctionJsonParse(); List functionList = functionJsonParse.parseJsonToBean(functionZkData.getDataValue()); rules.setFunction(functionList); xmlProcess.addParseClass(Table.class); xmlProcess.addParseClass(Function.class); RuleParseXmlImpl ruleParseXml = new RuleParseXmlImpl(xmlProcess); xmlProcess.initJaxbClass(); ruleParseXml.parseToXmlWrite(rules, localPath.resolve("rule.xml").toString(), "rule"); } private void pushACKToClean(String taskPath) throws Exception { String myID = ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_MYID); String path = taskPath + "/_clean/" + myID; if (ZKUtils.getConnection().checkExists().forPath(path) == null) { ZKUtils.getConnection().create().creatingParentsIfNeeded().forPath(path); } } private void clean(String taskID, List allTaskList) throws IOException, SQLException { //clean for (MigrateTask migrateTask : allTaskList) { String sql = makeCleanSql(migrateTask); MigrateUtils.execulteSql(sql, migrateTask.getFrom()); } } private void check(String taskID, List allTaskList) throws SQLException, IOException { for (MigrateTask migrateTask : allTaskList) { String sql = MigrateUtils.makeCountSql(migrateTask); long oldCount = MigrateUtils.execulteCount(sql, migrateTask.getFrom()); long newCount = MigrateUtils.execulteCount(sql, migrateTask.getTo()); if (oldCount != newCount) { throw new RuntimeException("migrate task (" + taskID + ") check fail,because fromNode:" + migrateTask.getFrom() + "(" + oldCount + ")" + " and toNode:" + migrateTask.getTo() + "(" + newCount + ") and sql is " + sql); } } } private String makeCleanSql(MigrateTask task) { StringBuilder sb = new StringBuilder(); sb.append("delete from "); sb.append(task.getTable()).append(" where "); List slots = task.getSlots(); for (int i = 0; i < slots.size(); i++) { Range range = slots.get(i); if (i != 0) sb.append(" or "); if (range.start == range.end) { sb.append(" _slot=").append(range.start); } else { sb.append("( _slot>=").append(range.start); sb.append(" and _slot<=").append(range.end).append(")"); } } return sb.toString(); } private CuratorTransactionFinal modifyRuleData(CuratorTransactionFinal transactionFinal, List allTaskList, TableConfig tableConfig, List allNewDataNodes) throws Exception { InterProcessMutex ruleDataLock = null; try { String path = ZKUtils.getZKBasePath() + "lock/ruledata.lock"; ruleDataLock = new InterProcessMutex(ZKUtils.getConnection(), path); ruleDataLock.acquire(30, TimeUnit.SECONDS); RuleConfig ruleConfig = tableConfig.getRule(); String ruleName = ((TableRuleAware) ruleConfig.getRuleAlgorithm()).getRuleName() + ".properties"; String rulePath = ZKUtils.getZKBasePath() + "ruledata/" + ruleName; CuratorFramework zk = ZKUtils.getConnection(); byte[] ruleData = zk.getData().forPath(rulePath); Properties prop = new Properties(); prop.load(new ByteArrayInputStream(ruleData)); for (MigrateTask migrateTask : allTaskList) { modifyRuleData(prop, migrateTask, allNewDataNodes); } ByteArrayOutputStream out = new ByteArrayOutputStream(); prop.store(out, "WARNING !!!Please do not modify or delete this file!!!"); if (transactionFinal == null) { transactionFinal = ZKUtils.getConnection().inTransaction().setData().forPath(rulePath, out.toByteArray()).and(); } else { transactionFinal.setData().forPath(rulePath, out.toByteArray()); } } finally { try { if (ruleDataLock != null) ruleDataLock.release(); } catch (Exception e) { throw new RuntimeException(e); } } return transactionFinal; } private void modifyRuleData(Properties prop, MigrateTask task, List allNewDataNodes) { int fromIndex = -1; int toIndex = -1; List dataNodes = allNewDataNodes; for (int i = 0; i < dataNodes.size(); i++) { String dataNode = dataNodes.get(i); if (dataNode.equalsIgnoreCase(task.getFrom())) { fromIndex = i; } else if (dataNode.equalsIgnoreCase(task.getTo())) { toIndex = i; } } String from = prop.getProperty(String.valueOf(fromIndex)); String to = prop.getProperty(String.valueOf(toIndex)); String fromRemain = removeRangesRemain(from, task.getSlots()); String taskRanges = MigrateUtils.convertRangeListToString(task.getSlots()); String newTo = to == null ? taskRanges : to + "," + taskRanges; prop.setProperty(String.valueOf(fromIndex), fromRemain); prop.setProperty(String.valueOf(toIndex), newTo); } private String removeRangesRemain(String ori, List rangeList) { List ranges = MigrateUtils.convertRangeStringToList(ori); List ramain = MigrateUtils.removeAndGetRemain(ranges, rangeList); return MigrateUtils.convertRangeListToString(ramain); } private static CuratorTransactionFinal modifyZkRules(CuratorTransactionFinal transactionFinal, String ruleName, List newDataNodes) throws Exception { CuratorFramework client = ZKUtils.getConnection(); String rulePath = ZKUtils.getZKBasePath() + "rules/function"; JSONArray jsonArray = JSON.parseArray(new String(client.getData().forPath(rulePath), "UTF-8")); for (Object obj : jsonArray) { JSONObject func = (JSONObject) obj; if (ruleName.equalsIgnoreCase(func.getString("name"))) { JSONArray property = func.getJSONArray("property"); for (Object o : property) { JSONObject count = (JSONObject) o; if ("count".equals(count.getString("name"))) { Integer xcount = Integer.parseInt(count.getString("value")); count.put("value", String.valueOf(xcount + newDataNodes.size())); if (transactionFinal == null) { transactionFinal = ZKUtils.getConnection().inTransaction().setData().forPath(rulePath, JSON.toJSONBytes(jsonArray)).and(); } else { transactionFinal.setData().forPath(rulePath, JSON.toJSONBytes(jsonArray)); } } } } } return transactionFinal; } private static CuratorTransactionFinal modifyTableConfigRules(CuratorTransactionFinal transactionFinal, String schemal, String table, List newDataNodes) throws Exception { CuratorFramework client = ZKUtils.getConnection(); String rulePath = ZKUtils.getZKBasePath() + "schema/schema"; JSONArray jsonArray = JSON.parseArray(new String(client.getData().forPath(rulePath), "UTF-8")); for (Object obj : jsonArray) { JSONObject func = (JSONObject) obj; if (schemal.equalsIgnoreCase(func.getString("name"))) { JSONArray property = func.getJSONArray("table"); for (Object o : property) { JSONObject tt = (JSONObject) o; String tableName = tt.getString("name"); String dataNode = tt.getString("dataNode"); if (table.equalsIgnoreCase(tableName)) { List allDataNodes = new ArrayList<>(); allDataNodes.add(dataNode); allDataNodes.addAll(newDataNodes); tt.put("dataNode", Joiner.on(",").join(allDataNodes)); if (transactionFinal == null) { transactionFinal = ZKUtils.getConnection().inTransaction().setData().forPath(rulePath, JSON.toJSONBytes(jsonArray)).and(); } else { transactionFinal.setData().forPath(rulePath, JSON.toJSONBytes(jsonArray)); } } } } } return transactionFinal; } } ================================================ FILE: src/main/java/io/mycat/migrate/SwitchPrepareCheckRunner.java ================================================ package io.mycat.migrate; import com.alibaba.fastjson.JSON; import com.google.common.collect.Sets; import io.mycat.MycatServer; import io.mycat.backend.BackendConnection; import io.mycat.config.loader.zkprocess.comm.ZkConfig; import io.mycat.config.loader.zkprocess.comm.ZkParamCfg; import io.mycat.net.NIOProcessor; import io.mycat.route.RouteCheckRule; import io.mycat.route.RouteResultset; import io.mycat.route.RouteResultsetNode; import io.mycat.route.function.PartitionByCRC32PreSlot; import io.mycat.util.ZKUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Collection; import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; /** * Created by nange on 2016/12/20. */ public class SwitchPrepareCheckRunner implements Runnable { private static final Logger LOGGER = LoggerFactory.getLogger(SwitchPrepareListener.class); public static Set allSwitchRunnerSet= Sets.newConcurrentHashSet(); private String taskID; private String taskPath; private TaskNode taskNode; private List rangeList; public SwitchPrepareCheckRunner( String taskID, String taskPath, TaskNode taskNode,List rangeList) { this.taskID = taskID; this.taskPath = taskPath; this.taskNode = taskNode; this.rangeList=rangeList; } @Override public void run() { if(!allSwitchRunnerSet.contains(taskID)){ return; } ScheduledExecutorService scheduledExecutorService= MycatServer.getInstance().getScheduler(); ConcurrentMap>> migrateRuleMap = RouteCheckRule.migrateRuleMap; String schemal = taskNode.getSchema().toUpperCase(); if(!migrateRuleMap.containsKey(schemal)||!migrateRuleMap.get( schemal).containsKey(taskNode.getTable().toUpperCase())){ scheduledExecutorService.schedule(this,3, TimeUnit.SECONDS); return; } boolean isHasInTransation=false; NIOProcessor[] processors=MycatServer.getInstance().getProcessors(); for (NIOProcessor processor : processors) { Collection backendConnections= processor.getBackends().values(); for (BackendConnection backendConnection : backendConnections) { isHasInTransation= checkIsInTransation(backendConnection); if(isHasInTransation){ scheduledExecutorService.schedule(this,3, TimeUnit.SECONDS); return; } } } for (BackendConnection backendConnection : NIOProcessor.backends_old) { isHasInTransation= checkIsInTransation(backendConnection); if(isHasInTransation){ scheduledExecutorService.schedule(this,3, TimeUnit.SECONDS); return; } } //增加判断binlog完成 if(!isHasInTransation){ try { //先判断后端binlog都完成了才算本任务完成 boolean allIncrentmentSucess=true; List dataHosts= ZKUtils.getConnection().getChildren().forPath(taskPath); for (String dataHostName : dataHosts) { if("_prepare".equals(dataHostName)||"_commit".equals(dataHostName)||"_clean".equals(dataHostName)) continue; List migrateTaskList= JSON .parseArray(new String(ZKUtils.getConnection().getData().forPath(taskPath+"/"+dataHostName),"UTF-8") ,MigrateTask.class); for (MigrateTask migrateTask : migrateTaskList) { String zkPath =taskPath+"/"+dataHostName+ "/" + migrateTask.getFrom() + "-" + migrateTask.getTo(); if (ZKUtils.getConnection().checkExists().forPath(zkPath) != null) { TaskStatus taskStatus = JSON.parseObject( new String(ZKUtils.getConnection().getData().forPath(zkPath), "UTF-8"), TaskStatus.class); if (taskStatus.getStatus() != 3) { allIncrentmentSucess=false; break; } }else{ allIncrentmentSucess=false; break; } } } if(allIncrentmentSucess) { //需要关闭binlog，不然后续的清楚老数据会删除数据 BinlogStream stream= BinlogStreamHoder.binlogStreamMap.get(taskID); if(stream!=null){ BinlogStreamHoder.binlogStreamMap.remove(taskID); stream.disconnect(); } String myID = ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_MYID); String path = taskPath + "/_commit/" + myID; if (ZKUtils.getConnection().checkExists().forPath(path) == null) { ZKUtils.getConnection().create().creatingParentsIfNeeded().forPath(path); } allSwitchRunnerSet.remove(taskID); } else { scheduledExecutorService.schedule(this,3, TimeUnit.SECONDS); } } catch (Exception e) { try { LOGGER.error("migrate 中 switch prepare 阶段异常"); taskNode.addException(e.getLocalizedMessage()); //异常to Zk ZKUtils.getConnection().setData().forPath(taskPath, JSON.toJSONBytes(taskNode)); }catch (Exception e1){ LOGGER.error("error:",e); } LOGGER.error("error:",e); } } } private boolean checkIsInTransation(BackendConnection backendConnection) { if(!taskNode.getSchema().equalsIgnoreCase(backendConnection.getSchema())) return false; Object attach= backendConnection.getAttachment(); if(attach instanceof RouteResultsetNode) { RouteResultsetNode resultsetNode= (RouteResultsetNode) attach; RouteResultset rrs= resultsetNode.getSource(); for (String table : rrs.getTables()) { if(table.equalsIgnoreCase(taskNode.getTable())) { int slot = resultsetNode.getSlot(); if(slot <0&&resultsetNode.isUpdateSql()) { return true; } else if(resultsetNode.isUpdateSql()) { for (PartitionByCRC32PreSlot.Range range : rangeList) { if(slot>=range.start&&slot<=range.end){ return true; } } } } } } return false; } } ================================================ FILE: src/main/java/io/mycat/migrate/SwitchPrepareListener.java ================================================ package io.mycat.migrate; import com.alibaba.fastjson.JSON; import io.mycat.route.RouteCheckRule; import io.mycat.route.function.PartitionByCRC32PreSlot; import io.mycat.util.ZKUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; /** * Created by magicdoom on 2016/12/19. */ public class SwitchPrepareListener implements PathChildrenCacheListener { private static final Logger LOGGER = LoggerFactory.getLogger(SwitchPrepareListener.class); @Override public void childEvent(CuratorFramework curatorFramework, PathChildrenCacheEvent event) throws Exception { switch (event.getType()) { case CHILD_ADDED: checkSwitch(event); break; default: break; } } private void checkSwitch(PathChildrenCacheEvent event) { InterProcessMutex taskLock = null; try { String path = event.getData().getPath(); String taskPath = path.substring(0, path.lastIndexOf("/_prepare/")); String taskID = taskPath.substring(taskPath.lastIndexOf('/') + 1, taskPath.length()); String lockPath = ZKUtils.getZKBasePath() + "lock/" + taskID + ".lock"; List sucessDataHost = ZKUtils.getConnection().getChildren().forPath(path.substring(0, path.lastIndexOf('/'))); List allTaskList = MigrateUtils.queryAllTask(taskPath, sucessDataHost); TaskNode pTaskNode = JSON.parseObject(ZKUtils.getConnection().getData().forPath(taskPath), TaskNode.class); ConcurrentMap> tableRuleMap = RouteCheckRule.migrateRuleMap.containsKey(pTaskNode.getSchema().toUpperCase()) ? RouteCheckRule.migrateRuleMap.get(pTaskNode.getSchema().toUpperCase()) : new ConcurrentHashMap(); tableRuleMap.put(pTaskNode.getTable().toUpperCase(), MigrateUtils.convertAllTask(allTaskList)); RouteCheckRule.migrateRuleMap.put(pTaskNode.getSchema().toUpperCase(), tableRuleMap); taskLock = new InterProcessMutex(ZKUtils.getConnection(), lockPath); taskLock.acquire(20, TimeUnit.SECONDS); List dataHost = ZKUtils.getConnection().getChildren().forPath(taskPath); if (getRealSize(dataHost) == sucessDataHost.size()) { TaskNode taskNode = JSON.parseObject(ZKUtils.getConnection().getData().forPath(taskPath), TaskNode.class); if (taskNode.getStatus() == 1) { taskNode.setStatus(2); //prepare switch LOGGER.info("task switch:", new Date()); ZKUtils.getConnection().setData().forPath(taskPath, JSON.toJSONBytes(taskNode)); } } } catch (Exception e) { LOGGER.error("error:", e); } finally { if (taskLock != null) { try { taskLock.release(); } catch (Exception ignored) { } } } } private int getRealSize(List dataHosts) { int size = dataHosts.size(); Set set = new HashSet(dataHosts); if (set.contains("_prepare")) { size = size - 1; } if (set.contains("_commit")) { size = size - 1; } if (set.contains("_clean")) { size = size - 1; } return size; } } ================================================ FILE: src/main/java/io/mycat/migrate/TaskNode.java ================================================ package io.mycat.migrate; import java.io.Serializable; /** * Created by magicdoom on 2016/9/28. */ public class TaskNode implements Serializable { private String sql; private int status; //0=init 1=start 2=prepare switch 3=commit sucess 4=error 5=clean sucess 6=error process end private String schema; private String table; private String add; private int timeout; private String charset; private boolean forceBinlog = false; private String backupFile; private String exception = ""; public String getSql() { return sql; } public String getTable() { return table; } public void setTable(String table) { this.table = table; } public String getAdd() { return add; } public void setAdd(String add) { this.add = add; } public void setSql(String sql) { this.sql = sql; } public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public String getSchema() { return schema; } public void setSchema(String schema) { this.schema = schema; } public int getTimeout() { return timeout; } public void setTimeout(int timeout) { this.timeout = timeout; } public String getCharset() { return charset; } public void setCharset(String charset) { this.charset = charset; } public boolean isForceBinlog() { return forceBinlog; } public void setForceBinlog(boolean forceBinlog) { this.forceBinlog = forceBinlog; } public String getBackupFile() { return backupFile; } public void setBackupFile(String backupFile) { this.backupFile = backupFile; } public String getException() { return exception; } public void addException(String exception) { this.exception = this.exception+"\n"+exception; } } ================================================ FILE: src/main/java/io/mycat/migrate/TaskStatus.java ================================================ package io.mycat.migrate; import java.io.Serializable; /** * Created by nange on 2016/12/7. */ public class TaskStatus implements Serializable { private int status; //0= dump error 1=dump sucess 2=increnment error 3=increment sucess 4=other error private String msg; private String binlogFile; private long pos; private String lastDate; public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public String getMsg() { return msg; } public void setMsg(String msg) { this.msg = msg; } public String getBinlogFile() { return binlogFile; } public void setBinlogFile(String binlogFile) { this.binlogFile = binlogFile; } public long getPos() { return pos; } public void setPos(long pos) { this.pos = pos; } } ================================================ FILE: src/main/java/io/mycat/net/AIOAcceptor.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net; import java.io.IOException; import java.net.InetSocketAddress; import java.net.StandardSocketOptions; import java.nio.channels.AsynchronousChannelGroup; import java.nio.channels.AsynchronousServerSocketChannel; import java.nio.channels.AsynchronousSocketChannel; import java.nio.channels.CompletionHandler; import java.nio.channels.NetworkChannel; import java.util.concurrent.atomic.AtomicLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.net.factory.FrontendConnectionFactory; /** * @author mycat */ public final class AIOAcceptor implements SocketAcceptor, CompletionHandler { private static final Logger LOGGER = LoggerFactory.getLogger(AIOAcceptor.class); private static final AcceptIdGenerator ID_GENERATOR = new AcceptIdGenerator(); private final int port; private final AsynchronousServerSocketChannel serverChannel; private final FrontendConnectionFactory factory; private long acceptCount; private final String name; public AIOAcceptor(String name, String ip, int port, int backlog, FrontendConnectionFactory factory, AsynchronousChannelGroup group) throws IOException { this.name = name; this.port = port; this.factory = factory; serverChannel = AsynchronousServerSocketChannel.open(group); /** 设置TCP属性 */ serverChannel.setOption(StandardSocketOptions.SO_REUSEADDR, true); serverChannel.setOption(StandardSocketOptions.SO_RCVBUF, 1024 * 16 * 2); serverChannel.bind(new InetSocketAddress(ip, port), backlog); } public String getName() { return name; } public void start() { this.pendingAccept(); } public int getPort() { return port; } public long getAcceptCount() { return acceptCount; } private void accept(NetworkChannel channel, Long id) { try { FrontendConnection c = factory.make(channel); c.setAccepted(true); c.setId(id); NIOProcessor processor = MycatServer.getInstance().nextProcessor(); c.setProcessor(processor); c.register(); } catch (Exception e) { LOGGER.error("AioAcceptorError", e); closeChannel(channel); } } private void pendingAccept() { if (serverChannel.isOpen()) { serverChannel.accept(ID_GENERATOR.getId(), this); } else { throw new IllegalStateException( "MyCAT Server Channel has been closed"); } } @Override public void completed(AsynchronousSocketChannel result, Long id) { accept(result, id); // next pending waiting pendingAccept(); } @Override public void failed(Throwable exc, Long id) { LOGGER.info("acception connect failed:" + exc); // next pending waiting pendingAccept(); } private static void closeChannel(NetworkChannel channel) { if (channel == null) { return; } try { channel.close(); } catch (IOException e) { LOGGER.error("AioAcceptorError", e); } } /** * 前端连接ID生成器 * * @author mycat */ private static class AcceptIdGenerator { private static final long MAX_VALUE = 0xffffffffL; private AtomicLong acceptId = new AtomicLong(); private final Object lock = new Object(); private long getId() { long newValue = acceptId.getAndIncrement(); if (newValue >= MAX_VALUE) { synchronized (lock) { newValue = acceptId.getAndIncrement(); if (newValue >= MAX_VALUE) { acceptId.set(0); } } return acceptId.getAndDecrement(); } else { return newValue; } } } } ================================================ FILE: src/main/java/io/mycat/net/AIOConnector.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net; import java.nio.channels.CompletionHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import java.util.concurrent.atomic.AtomicLong; /** * @author mycat */ public final class AIOConnector implements SocketConnector, CompletionHandler { private static final Logger LOGGER = LoggerFactory.getLogger(AIOConnector.class); private static final ConnectIdGenerator ID_GENERATOR = new ConnectIdGenerator(); public AIOConnector() { } @Override public void completed(Void result, BackendAIOConnection attachment) { finishConnect(attachment); } @Override public void failed(Throwable exc, BackendAIOConnection conn) { conn.onConnectFailed(exc); } private void finishConnect(BackendAIOConnection c) { try { if (c.finishConnect()) { c.setId(ID_GENERATOR.getId()); NIOProcessor processor = MycatServer.getInstance() .nextProcessor(); c.setProcessor(processor); c.register(); } } catch (Exception e) { c.onConnectFailed(e); LOGGER.info("connect err " , e); c.close(e.toString()); } } /** * 后端连接ID生成器 * * @author mycat */ private static class ConnectIdGenerator { private static final long MAX_VALUE = Long.MAX_VALUE; private AtomicLong connectId = new AtomicLong(0); private long getId() { return connectId.incrementAndGet(); } } } ================================================ FILE: src/main/java/io/mycat/net/AIOSocketWR.java ================================================ package io.mycat.net; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.AsynchronousSocketChannel; import java.nio.channels.CompletionHandler; import java.util.concurrent.atomic.AtomicBoolean; import io.mycat.util.TimeUtil; public class AIOSocketWR extends SocketWR { private static final AIOReadHandler aioReadHandler = new AIOReadHandler(); private static final AIOWriteHandler aioWriteHandler = new AIOWriteHandler(); private final AsynchronousSocketChannel channel; protected final AbstractConnection con; protected final AtomicBoolean writing = new AtomicBoolean(false); public AIOSocketWR(AbstractConnection conn) { channel = (AsynchronousSocketChannel) conn.getChannel(); this.con = conn; } @Override public void asynRead() { ByteBuffer theBuffer = con.readBuffer; if (theBuffer == null) { theBuffer = con.processor.getBufferPool().allocate(con.processor.getBufferPool().getChunkSize()); con.readBuffer = theBuffer; channel.read(theBuffer, this, aioReadHandler); } else if (theBuffer.hasRemaining()) { channel.read(theBuffer, this, aioReadHandler); } else { throw new java.lang.IllegalArgumentException("full buffer to read "); } } private void asynWrite(final ByteBuffer buffer) { buffer.flip(); this.channel.write(buffer, this, aioWriteHandler); } // public int flushChannel(final AsynchronousSocketChannel channel, // final ByteBuffer bb, final long writeTimeout) // { // // if (!bb.hasRemaining()) // { // return 0; // } // int nWrite = bb.limit(); // try // { // while (bb.hasRemaining()) // { // channel.write(bb).get(writeTimeout, TimeUnit.SECONDS); // } // } catch (Exception ie) // { // con.close("write failed " + ie); // // } // return nWrite; // } /** * return true ,means no more data * * @return */ private boolean write0() { if (!writing.compareAndSet(false, true)) { return false; } ByteBuffer theBuffer = con.writeBuffer; if (theBuffer == null || !theBuffer.hasRemaining()) {// writeFinished,但要区分bufer是否NULL，不NULL，要回收 if (theBuffer != null) { con.recycle(theBuffer); con.writeBuffer = null; } // poll again ByteBuffer buffer = con.writeQueue.poll(); // more data if (buffer != null) { if (buffer.limit() == 0) { con.recycle(buffer); con.writeBuffer = null; con.close("quit cmd"); writing.set(false); return true; } else { con.writeBuffer = buffer; asynWrite(buffer); return false; } } else { // no buffer writing.set(false); return true; } } else { theBuffer.compact(); asynWrite(theBuffer); return false; } } protected void onWriteFinished(int result) { con.netOutBytes += result; con.processor.addNetOutBytes(result); con.lastWriteTime = TimeUtil.currentTimeMillis(); boolean noMoreData = this.write0(); if (noMoreData) { this.doNextWriteCheck(); } } public void doNextWriteCheck() { boolean noMoreData = false; noMoreData = this.write0(); if (noMoreData && !con.writeQueue.isEmpty()) { this.write0(); } } @Override public boolean checkAlive() { return channel.isOpen(); } @Override public void disableRead() { // TODO Auto-generated method stub } @Override public void enableRead() { // TODO Auto-generated method stub } } class AIOWriteHandler implements CompletionHandler { @Override public void completed(final Integer result, final AIOSocketWR wr) { try { wr.writing.set(false); if (result >= 0) { wr.onWriteFinished(result); } else { wr.con.close("write erro " + result); } } catch (Exception e) { AbstractConnection.LOGGER.warn("caught aio process err:", e); } } @Override public void failed(Throwable exc, AIOSocketWR wr) { wr.writing.set(false); wr.con.close("write failed " + exc); } } class AIOReadHandler implements CompletionHandler { @Override public void completed(final Integer i, final AIOSocketWR wr) { // con.getProcessor().getExecutor().execute(new Runnable() { // public void run() { if (i > 0) { try { wr.con.onReadData(i); wr.con.asynRead(); } catch (IOException e) { wr.con.close("handle err:" + e); } } else if (i == -1) { // System.out.println("read -1 xxxxxxxxx "+con); wr.con.close("client closed"); } // } // }); } @Override public void failed(Throwable exc, AIOSocketWR wr) { wr.con.close(exc.toString()); } } ================================================ FILE: src/main/java/io/mycat/net/AbstractConnection.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net; import java.io.IOException; import java.net.Socket; import java.nio.ByteBuffer; import java.nio.channels.AsynchronousChannel; import java.nio.channels.NetworkChannel; import java.nio.channels.SocketChannel; import java.util.List; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicBoolean; import com.google.common.base.Strings; import io.mycat.MycatServer; import io.mycat.backend.mysql.CharsetUtil; import io.mycat.config.model.SystemConfig; import io.mycat.util.CompressUtil; import io.mycat.util.TimeUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author mycat */ public abstract class AbstractConnection implements NIOConnection { protected static final Logger LOGGER = LoggerFactory.getLogger(AbstractConnection.class); protected String host; protected int localPort; protected int port; protected long id; protected volatile String charset; protected volatile int charsetIndex; protected final NetworkChannel channel; protected NIOProcessor processor; protected NIOHandler handler; protected int packetHeaderSize; protected int maxPacketSize; protected volatile ByteBuffer readBuffer; protected volatile ByteBuffer writeBuffer; protected final ConcurrentLinkedQueue writeQueue = new ConcurrentLinkedQueue(); protected volatile int readBufferOffset; protected long lastLargeMessageTime; protected final AtomicBoolean isClosed; protected boolean isSocketClosed; protected long startupTime; protected long lastReadTime; protected long lastWriteTime; protected long netInBytes; protected long netOutBytes; protected int writeAttempts; protected volatile boolean isSupportCompress = false; protected final ConcurrentLinkedQueue decompressUnfinishedDataQueue = new ConcurrentLinkedQueue(); protected final ConcurrentLinkedQueue compressUnfinishedDataQueue = new ConcurrentLinkedQueue(); private long idleTimeout; protected final SocketWR socketWR; protected boolean enableFlowController;// writeQueue是否开启流控 protected final int writeQueueStopThreshold;// writeQueue停止阀值 protected final int writeQueueRecoverThreshold;// writeQueue恢复阀值 public AbstractConnection(NetworkChannel channel) { this.channel = channel; boolean isAIO = (channel instanceof AsynchronousChannel); if (isAIO) { socketWR = new AIOSocketWR(this); } else { socketWR = new NIOSocketWR(this); } this.isClosed = new AtomicBoolean(false); this.startupTime = TimeUtil.currentTimeMillis(); this.lastReadTime = startupTime; this.lastWriteTime = startupTime; SystemConfig config = MycatServer.getInstance().getConfig().getSystem(); this.enableFlowController = config.isEnableWriteQueueFlowControl(); this.writeQueueStopThreshold = config.getWriteQueueStopThreshold(); this.writeQueueRecoverThreshold = config.getWriteQueueRecoverThreshold(); } public String getCharset() { return charset; } public boolean setCharset(String charset) { // 修复PHP字符集设置错误, 如： set names 'utf8' if (charset != null) { charset = charset.replace("'", ""); } int ci = CharsetUtil.getIndex(charset); if (ci > 0) { this.charset = charset.equalsIgnoreCase("utf8mb4") ? "utf8" : charset; this.charsetIndex = ci; return true; } else { return false; } } public boolean isSupportCompress() { return isSupportCompress; } public void setSupportCompress(boolean isSupportCompress) { this.isSupportCompress = isSupportCompress; } public int getCharsetIndex() { return charsetIndex; } public long getIdleTimeout() { return idleTimeout; } public SocketWR getSocketWR() { return socketWR; } public void setIdleTimeout(long idleTimeout) { this.idleTimeout = idleTimeout; } public int getLocalPort() { return localPort; } public String getHost() { return host; } public void setHost(String host) { this.host = host; } public int getPort() { return port; } public void setPort(int port) { this.port = port; } public void setLocalPort(int localPort) { this.localPort = localPort; } public long getId() { return id; } public void setId(long id) { this.id = id; } public boolean isIdleTimeout() { return TimeUtil.currentTimeMillis() > Math.max(lastWriteTime, lastReadTime) + idleTimeout; } public NetworkChannel getChannel() { return channel; } public int getPacketHeaderSize() { return packetHeaderSize; } public void setPacketHeaderSize(int packetHeaderSize) { this.packetHeaderSize = packetHeaderSize; } public int getMaxPacketSize() { return maxPacketSize; } public void setMaxPacketSize(int maxPacketSize) { this.maxPacketSize = maxPacketSize; } public long getStartupTime() { return startupTime; } public long getLastReadTime() { return lastReadTime; } public void setProcessor(NIOProcessor processor) { this.processor = processor; int size = processor.getBufferPool().getChunkSize(); this.readBuffer = processor.getBufferPool().allocate(size); } public long getLastWriteTime() { return lastWriteTime; } public void setLastWriteTime(long lasttime){ this.lastWriteTime = lasttime; } public long getNetInBytes() { return netInBytes; } public long getNetOutBytes() { return netOutBytes; } public int getWriteAttempts() { return writeAttempts; } public NIOProcessor getProcessor() { return processor; } public ByteBuffer getReadBuffer() { return readBuffer; } public ByteBuffer allocate() { int size = this.processor.getBufferPool().getChunkSize(); ByteBuffer buffer = this.processor.getBufferPool().allocate(size); return buffer; } public final void recycle(ByteBuffer buffer) { this.processor.getBufferPool().recycle(buffer); } public void setHandler(NIOHandler handler) { this.handler = handler; } @Override public void handle(byte[] data) { if (isSupportCompress()) { List packs = CompressUtil.decompressMysqlPacket(data, decompressUnfinishedDataQueue); for (byte[] pack : packs) { if (pack.length != 0) { handler.handle(pack); } } } else { handler.handle(data); } } @Override public void register() throws IOException { } public void asynRead() throws IOException { this.socketWR.asynRead(); } public void doNextWriteCheck() throws IOException { this.socketWR.doNextWriteCheck(); } /** * 读取可能的Socket字节流 */ public void onReadData(int got) throws IOException { if (isClosed.get()) { return; } lastReadTime = TimeUtil.currentTimeMillis(); if (got < 0) { this.close("stream closed"); return; } else if (got == 0 && !this.channel.isOpen()) { this.close("socket closed"); return; } netInBytes += got; processor.addNetInBytes(got); // 循环处理字节信息 int offset = readBufferOffset, length = 0, position = readBuffer.position(); for (;;) { length = getPacketLength(readBuffer, offset); if (length == -1) { if (offset != 0) { this.readBuffer = compactReadBuffer(readBuffer, offset); } else if (readBuffer != null && !readBuffer.hasRemaining()) { throw new RuntimeException( "invalid readbuffer capacity ,too little buffer size " + readBuffer.capacity()); } break; } if (position >= offset + length && readBuffer != null) { // handle this package readBuffer.position(offset); byte[] data = new byte[length]; readBuffer.get(data, 0, length); handle(data); // maybe handle stmt_close if(isClosed()) { return ; } // offset to next position offset += length; // reached end if (position == offset) { // if cur buffer is temper none direct byte buffer and not // received large message in recent 30 seconds // then change to direct buffer for performance if (readBuffer != null && !readBuffer.isDirect() && lastLargeMessageTime < lastReadTime - 30 * 1000L) { // used temp heap if (LOGGER.isDebugEnabled()) { LOGGER.debug("change to direct con read buffer ,cur temp buf size :" + readBuffer.capacity()); } recycle(readBuffer); readBuffer = processor.getBufferPool().allocate(processor.getBufferPool().getConReadBuferChunk()); } else { if (readBuffer != null) { readBuffer.clear(); } } // no more data ,break readBufferOffset = 0; break; } else { // try next package parse readBufferOffset = offset; if(readBuffer != null) { readBuffer.position(position); } continue; } } else { // not read whole message package ,so check if buffer enough and // compact readbuffer if (!readBuffer.hasRemaining()) { readBuffer = ensureFreeSpaceOfReadBuffer(readBuffer, offset, length); } break; } } } private boolean isConReadBuffer(ByteBuffer buffer) { return buffer.capacity() == processor.getBufferPool().getConReadBuferChunk() && buffer.isDirect(); } private ByteBuffer ensureFreeSpaceOfReadBuffer(ByteBuffer buffer, int offset, final int pkgLength) { // need a large buffer to hold the package if (pkgLength > maxPacketSize) { throw new IllegalArgumentException("Packet size over the limit."); } else if (buffer.capacity() < pkgLength) { ByteBuffer newBuffer = processor.getBufferPool().allocate(pkgLength); lastLargeMessageTime = TimeUtil.currentTimeMillis(); buffer.position(offset); newBuffer.put(buffer); readBuffer = newBuffer; recycle(buffer); readBufferOffset = 0; return newBuffer; } else { if (offset != 0) { // compact bytebuffer only return compactReadBuffer(buffer, offset); } else { throw new RuntimeException(" not enough space"); } } } private ByteBuffer compactReadBuffer(ByteBuffer buffer, int offset) { if(buffer == null) { return null; } buffer.limit(buffer.position()); buffer.position(offset); buffer = buffer.compact(); readBufferOffset = 0; return buffer; } public void write(byte[] data) { ByteBuffer buffer = allocate(); buffer = writeToBuffer(data, buffer); write(buffer); } private final void writeNotSend(ByteBuffer buffer) { if (isSupportCompress()) { ByteBuffer newBuffer = CompressUtil.compressMysqlPacket(buffer, this, compressUnfinishedDataQueue); writeQueue.offer(newBuffer); } else { writeQueue.offer(buffer); } if(isClosed()) { LOGGER.warn("write err:{}", this); this.close("found this connection has close , try to reClean the connection"); throw new RuntimeException("writeNotSend but found connnection close err:" + this); } } @Override public final void write(ByteBuffer buffer) { if (isSupportCompress()) { ByteBuffer newBuffer = CompressUtil.compressMysqlPacket(buffer, this, compressUnfinishedDataQueue); writeQueue.offer(newBuffer); } else { writeQueue.offer(buffer); } // if ansyn write finishe event got lock before me ,then writing // flag is set false but not start a write request // so we check again try { this.socketWR.doNextWriteCheck(); } catch (Exception e) { LOGGER.warn("write err:", e); this.close("write err:" + e); } } public ByteBuffer checkWriteBuffer(ByteBuffer buffer, int capacity, boolean writeSocketIfFull) { if (capacity > buffer.remaining()) { if (writeSocketIfFull) { writeNotSend(buffer); return processor.getBufferPool().allocate(capacity); } else {// Relocate a larger buffer buffer.flip(); ByteBuffer newBuf = processor.getBufferPool().allocate(capacity + buffer.limit() + 1); newBuf.put(buffer); this.recycle(buffer); return newBuf; } } else { return buffer; } } public ByteBuffer writeToBuffer(byte[] src, ByteBuffer buffer) { int offset = 0; int length = src.length; int remaining = buffer.remaining(); while (length > 0) { if (remaining >= length) { buffer.put(src, offset, length); break; } else { buffer.put(src, offset, remaining); writeNotSend(buffer); buffer = allocate(); offset += remaining; length -= remaining; remaining = buffer.remaining(); continue; } } return buffer; } @Override public void close(String reason) { if (!isClosed.get()) { closeSocket(); isClosed.set(true); if (processor != null) { processor.removeConnection(this); } this.cleanup(); isSupportCompress = false; // ignore null information if (Strings.isNullOrEmpty(reason)) { return; } LOGGER.info("close connection,reason:" + reason + " ," + this); if (reason.contains("connection,reason:java.net.ConnectException")) { throw new RuntimeException(" errr"); } } else { // make sure cleanup again // Fix issue#1616 this.cleanup(); } } public boolean isClosed() { return isClosed.get(); } public void idleCheck() { if (isIdleTimeout()) { LOGGER.info(toString() + " idle timeout"); close(" idle "); } } /** * 清理资源 */ synchronized protected void cleanup() { // 清理资源占用 if (readBuffer != null) { this.recycle(readBuffer); this.readBuffer = null; this.readBufferOffset = 0; } if (writeBuffer != null) { recycle(writeBuffer); this.writeBuffer = null; } if (!decompressUnfinishedDataQueue.isEmpty()) { decompressUnfinishedDataQueue.clear(); } if (!compressUnfinishedDataQueue.isEmpty()) { compressUnfinishedDataQueue.clear(); } ByteBuffer buffer = null; while ((buffer = writeQueue.poll()) != null) { recycle(buffer); } } protected int getPacketLength(ByteBuffer buffer, int offset) { int headerSize = getPacketHeaderSize(); if ( isSupportCompress() ) { headerSize = 7; } if (buffer.position() < offset + headerSize) { return -1; } else { int length = buffer.get(offset) & 0xff; length |= (buffer.get(++offset) & 0xff) << 8; length |= (buffer.get(++offset) & 0xff) << 16; return length + headerSize; } } public ConcurrentLinkedQueue getWriteQueue() { return writeQueue; } private void closeSocket() { if (channel != null) { if (channel instanceof SocketChannel) { Socket socket = ((SocketChannel) channel).socket(); if (socket != null) { try { socket.close(); } catch (IOException e) { LOGGER.error("closeChannelError", e); } } } boolean isSocketClosed = true; try { channel.close(); } catch (Exception e) { LOGGER.error("AbstractConnectionCloseError", e); } boolean closed = isSocketClosed && (!channel.isOpen()); if (closed == false) { LOGGER.warn("close socket of connnection failed " + this); } } } public void onConnectfinish() { LOGGER.debug("连接后台真正完成"); } public boolean checkAlive(){ return socketWR.checkAlive(); } public boolean isEnableFlowController() { return enableFlowController; } public int getWriteQueueStopThreshold() { return writeQueueStopThreshold; } public int getWriteQueueRecoverThreshold() { return writeQueueRecoverThreshold; } /** * 检查写队列流量，必要时候进行流控 */ abstract public void checkQueueFlow(); } ================================================ FILE: src/main/java/io/mycat/net/BIOConnection.java ================================================ package io.mycat.net; public interface BIOConnection extends ClosableConnection{ } ================================================ FILE: src/main/java/io/mycat/net/BackendAIOConnection.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.channels.NetworkChannel; import io.mycat.backend.BackendConnection; /** * @author mycat */ public abstract class BackendAIOConnection extends AbstractConnection implements BackendConnection { protected boolean isFinishConnect; public BackendAIOConnection(NetworkChannel channel) { super(channel); } public void register() throws IOException { this.asynRead(); } public void setHost(String host) { this.host = host; } public void setPort(int port) { this.port = port; } public void discardClose(String reason){ //跨节点处理,中断后端连接时关闭 } public abstract void onConnectFailed(Throwable e); public boolean finishConnect() throws IOException { localPort = ((InetSocketAddress) channel.getLocalAddress()).getPort(); isFinishConnect = true; return true; } public void setProcessor(NIOProcessor processor) { super.setProcessor(processor); processor.addBackend(this); } @Override public void checkQueueFlow() { } @Override public String toString() { return "BackendConnection [id=" + id + ", host=" + host + ", port=" + port + ", localPort=" + localPort + "]"; } } ================================================ FILE: src/main/java/io/mycat/net/ClosableConnection.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net; public interface ClosableConnection { String getCharset(); /** * 关闭连接 */ void close(String reason); boolean isClosed(); public void idleCheck(); long getStartupTime(); String getHost(); int getPort(); int getLocalPort(); long getNetInBytes(); long getNetOutBytes(); } ================================================ FILE: src/main/java/io/mycat/net/ConnectionException.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net; public class ConnectionException extends RuntimeException { /** * */ private static final long serialVersionUID = 1L; private final int code; private final String msg; public ConnectionException(int code, String msg) { super(msg); this.code = code; this.msg = msg; } @Override public String toString() { return "ConnectionException [code=" + code + ", msg=" + msg + "]"; } } ================================================ FILE: src/main/java/io/mycat/net/FrontendConnection.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.InetSocketAddress; import java.nio.channels.AsynchronousSocketChannel; import java.nio.channels.NetworkChannel; import java.nio.channels.SocketChannel; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.backend.BackendConnection; import io.mycat.backend.mysql.CharsetUtil; import io.mycat.backend.mysql.MySQLMessage; import io.mycat.config.Capabilities; import io.mycat.config.ErrorCode; import io.mycat.config.Versions; import io.mycat.net.handler.FrontendAuthenticator; import io.mycat.net.handler.FrontendPrepareHandler; import io.mycat.net.handler.FrontendPrivileges; import io.mycat.net.handler.FrontendQueryHandler; import io.mycat.net.handler.LoadDataInfileHandler; import io.mycat.net.mysql.CommandPacket; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.ErrorPacket; import io.mycat.net.mysql.HandshakePacket; import io.mycat.net.mysql.HandshakeV10Packet; import io.mycat.net.mysql.MySQLPacket; import io.mycat.net.mysql.OkPacket; import io.mycat.route.RouteResultsetNode; import io.mycat.server.parser.ServerParse; import io.mycat.util.CompressUtil; import io.mycat.util.RandomUtil; /** * @author mycat */ public abstract class FrontendConnection extends AbstractConnection { private static final Logger LOGGER = LoggerFactory.getLogger(FrontendConnection.class); protected long idleTimeout; protected byte[] seed; protected String user; protected String schema; protected String executeSql; protected volatile long executeSqlId = 0; protected AtomicLong responseSqlId = new AtomicLong(0); //新增executeSqlId ,repsonseSqlId 用于避免对一个sql 写回了多个错误的结果. protected FrontendPrivileges privileges; protected FrontendQueryHandler queryHandler; protected FrontendPrepareHandler prepareHandler; protected LoadDataInfileHandler loadDataInfileHandler; protected boolean isAccepted; protected boolean isAuthenticated; protected QueueFlowController flowController; private boolean allowMultiStatements = false; public FrontendConnection(NetworkChannel channel) throws IOException { super(channel); InetSocketAddress localAddr = (InetSocketAddress) channel.getLocalAddress(); InetSocketAddress remoteAddr = null; if (channel instanceof SocketChannel) { remoteAddr = (InetSocketAddress) ((SocketChannel) channel).getRemoteAddress(); } else if (channel instanceof AsynchronousSocketChannel) { remoteAddr = (InetSocketAddress) ((AsynchronousSocketChannel) channel).getRemoteAddress(); } this.host = remoteAddr.getHostString(); this.port = localAddr.getPort(); this.localPort = remoteAddr.getPort(); this.handler = new FrontendAuthenticator(this); if (enableFlowController) { this.flowController = new QueueFlowController(this); } } public long getId() { return id; } public void setId(long id) { this.id = id; if(LOGGER.isDebugEnabled()) { LOGGER.debug(this + " localPort:" + this.localPort + " port"+this.port); } } public String getHost() { return host; } public void setHost(String host) { this.host = host; } public int getPort() { return port; } public void setPort(int port) { this.port = port; } public int getLocalPort() { return localPort; } public void setLocalPort(int localPort) { this.localPort = localPort; } public void setAccepted(boolean isAccepted) { this.isAccepted = isAccepted; } public void setProcessor(NIOProcessor processor) { super.setProcessor(processor); processor.addFrontend(this); } public LoadDataInfileHandler getLoadDataInfileHandler() { return loadDataInfileHandler; } public void setLoadDataInfileHandler(LoadDataInfileHandler loadDataInfileHandler) { this.loadDataInfileHandler = loadDataInfileHandler; } public void setQueryHandler(FrontendQueryHandler queryHandler) { this.queryHandler = queryHandler; } public void setPrepareHandler(FrontendPrepareHandler prepareHandler) { this.prepareHandler = prepareHandler; } public void setAuthenticated(boolean isAuthenticated) { this.isAuthenticated = isAuthenticated; } public FrontendPrivileges getPrivileges() { return privileges; } public void setPrivileges(FrontendPrivileges privileges) { this.privileges = privileges; } public String getUser() { return user; } public void setUser(String user) { this.user = user; } public String getSchema() { return schema; } public void setSchema(String schema) { this.schema = schema; } public String getExecuteSql() { return executeSql; } public void setExecuteSql(String executeSql) { this.executeSql = executeSql; } public byte[] getSeed() { return seed; } public boolean setCharsetIndex(int ci) { String charset = CharsetUtil.getCharset(ci); if (charset != null) { return setCharset(charset); } else { return false; } } public boolean isAllowMultiStatements() { return allowMultiStatements; } public void setAllowMultiStatements(boolean allowMultiStatements) { this.allowMultiStatements = allowMultiStatements; } public void writeErrMessage(int errno, String msg) { if(this.canResponse()){ LOGGER.error("{}{} write errorMsg:{} error",this, msg+ getStack()); writeErrMessage((byte) 1, errno, msg); } else { LOGGER.error("{} write errorMsg:{} error",this,msg); } } //前端sql已经返回结果集 则调用这个函数 避免向前端返回多次结果。 //modify by zwy public void setResponseId() { //this.responseSqlId = this.executeSqlId; this.canResponse(); } public String getStack() { StackTraceElement stack[] = Thread.currentThread().getStackTrace(); StringBuilder sb = new StringBuilder(); for(int i=0;i resId) { boolean t = this.responseSqlId.compareAndSet(resId, this.executeSqlId); if(false) { StackTraceElement stack[] = Thread.currentThread().getStackTrace(); StringBuilder sb = new StringBuilder(); for(int i=0;i schemas = privileges.getUserSchemas(user); if (schemas == null || schemas.size() == 0 || schemas.contains(db)) { this.schema = db; write(writeToBuffer(OkPacket.OK, allocate())); } else { String s = "Access denied for user '" + user + "' to database '" + db + "'"; writeErrMessage(ErrorCode.ER_DBACCESS_DENIED_ERROR, s); } } public void loadDataInfileStart(String sql) { if (loadDataInfileHandler != null) { try { loadDataInfileHandler.start(sql); } catch (Exception e) { LOGGER.error("load data error", e); writeErrMessage(ErrorCode.ERR_HANDLE_DATA, e.getMessage()); } } else { writeErrMessage(ErrorCode.ER_UNKNOWN_COM_ERROR, "load data infile sql is not unsupported!"); } } public void loadDataInfileData(byte[] data) { if (loadDataInfileHandler != null) { try { loadDataInfileHandler.handle(data); } catch (Exception e) { LOGGER.error("load data error", e); writeErrMessage(ErrorCode.ERR_HANDLE_DATA, e.getMessage()); } } else { writeErrMessage(ErrorCode.ER_UNKNOWN_COM_ERROR, "load data infile data is not unsupported!"); } } public void loadDataInfileEnd(byte packID) { if (loadDataInfileHandler != null) { try { loadDataInfileHandler.end(packID); } catch (Exception e) { LOGGER.error("load data error", e); writeErrMessage(ErrorCode.ERR_HANDLE_DATA, e.getMessage()); } } else { writeErrMessage(ErrorCode.ER_UNKNOWN_COM_ERROR, "load data infile end is not unsupported!"); } } public void query(String sql) { if (sql == null || sql.length() == 0) { writeErrMessage(ErrorCode.ER_NOT_ALLOWED_COMMAND, "Empty SQL"); return; } if (LOGGER.isDebugEnabled()) { LOGGER.debug(new StringBuilder().append(this).append(" ").append(sql).toString()); } // remove last ';' if (sql.endsWith(";")) { sql = sql.substring(0, sql.length() - 1); } // remove like '/* ApplicationName=DBeaver 6.0.1 - Main */' tool app hints if(sql.indexOf("/* ApplicationName") >=0) { sql = sql.replaceFirst("\\/\\*.*\\*\\/\\s*", ""); } // 记录SQL this.setExecuteSql(sql); // 防火墙策略( SQL 黑名单/ 注入攻击) if ( !privileges.checkFirewallSQLPolicy( user, sql ) ) { writeErrMessage(ErrorCode.ERR_WRONG_USED, "The statement is unsafe SQL, reject for user '" + user + "'"); return; } // DML 权限检查 try { boolean isPassed = privileges.checkDmlPrivilege(user, schema, sql); if ( !isPassed ) { writeErrMessage(ErrorCode.ERR_WRONG_USED, "The statement DML privilege check is not passed, reject for user '" + user + "'"); return; } } catch( com.alibaba.druid.sql.parser.ParserException e1) { writeErrMessage(ErrorCode.ERR_WRONG_USED, e1.getMessage()); LOGGER.error("parse exception", e1 ); return; } // 执行查询 if (queryHandler != null) { queryHandler.setReadOnly(privileges.isReadOnly(user)); queryHandler.query(sql); } else { writeErrMessage(ErrorCode.ER_UNKNOWN_COM_ERROR, "Query unsupported!"); } } public void query(byte[] data) { // 取得语句 final String sql; try { MySQLMessage mm = new MySQLMessage(data); mm.position(5); sql = mm.readString(charset); } catch (UnsupportedEncodingException e) { writeErrMessage(ErrorCode.ER_UNKNOWN_CHARACTER_SET, "Unknown charset '" + charset + "'"); return; } MycatServer.getInstance().getBusinessExecutor().execute(new Runnable() { @Override public void run() { try { FrontendConnection.this.query( sql ); } catch (Throwable t) { FrontendConnection.this.writeErrMessage(ErrorCode.ERR_HANDLE_DATA, t.getMessage()); FrontendConnection.LOGGER.error("uncaught exception executing sql:"+FrontendConnection.this + ",[ " + sql+ "]", t); } } }); } public void stmtPrepare(byte[] data) { if (prepareHandler != null) { // 取得语句 MySQLMessage mm = new MySQLMessage(data); mm.position(5); String sql = null; try { sql = mm.readString(charset); } catch (UnsupportedEncodingException e) { writeErrMessage(ErrorCode.ER_UNKNOWN_CHARACTER_SET, "Unknown charset '" + charset + "'"); return; } if (sql == null || sql.length() == 0) { writeErrMessage(ErrorCode.ER_NOT_ALLOWED_COMMAND, "Empty SQL"); return; } // 记录SQL this.setExecuteSql(sql); // 执行预处理 prepareHandler.prepare(sql); } else { writeErrMessage(ErrorCode.ER_UNKNOWN_COM_ERROR, "Prepare unsupported!"); } } public void stmtSendLongData(byte[] data) { if(prepareHandler != null) { prepareHandler.sendLongData(data); } else { writeErrMessage(ErrorCode.ER_UNKNOWN_COM_ERROR, "Prepare unsupported!"); } } public void stmtReset(byte[] data) { if(prepareHandler != null) { prepareHandler.reset(data); } else { writeErrMessage(ErrorCode.ER_UNKNOWN_COM_ERROR, "Prepare unsupported!"); } } public void stmtExecute(byte[] data) { if (prepareHandler != null) { prepareHandler.execute(data); } else { writeErrMessage(ErrorCode.ER_UNKNOWN_COM_ERROR, "Prepare unsupported!"); } } public void stmtClose(byte[] data) { if (prepareHandler != null) { prepareHandler.close( data ); } else { writeErrMessage(ErrorCode.ER_UNKNOWN_COM_ERROR, "Prepare unsupported!"); } } /** * 用来模拟mysql协议命令中的com_field_list，方便通过发sql测试 * https://dev.mysql.com/doc/internals/en/com-field-list.html */ public void fieldList(byte[] data) { // 取得语句 String sql = null; try { MySQLMessage mm = new MySQLMessage(data); mm.position(5); sql = mm.readString(charset); sql = ServerParse.COM_FIELD_LIST_FLAG + sql; } catch (UnsupportedEncodingException e) { writeErrMessage(ErrorCode.ER_UNKNOWN_CHARACTER_SET, "Unknown charset '" + charset + "'"); return; } this.query( sql ); } public void ping() { write(writeToBuffer(OkPacket.OK, allocate())); } public void heartbeat(byte[] data) { write(writeToBuffer(OkPacket.OK, allocate())); } public void kill(byte[] data) { close("kill command"); } public void unknown(byte[] data) { writeErrMessage(ErrorCode.ER_UNKNOWN_COM_ERROR, "Unknown command"); } @Override public void register() throws IOException { if (!isClosed.get()) { // 生成认证数据 byte[] rand1 = RandomUtil.randomBytes(8); byte[] rand2 = RandomUtil.randomBytes(12); // 保存认证数据 byte[] seed = new byte[rand1.length + rand2.length]; System.arraycopy(rand1, 0, seed, 0, rand1.length); System.arraycopy(rand2, 0, seed, rand1.length, rand2.length); this.seed = seed; // 发送握手数据包 boolean useHandshakeV10 = MycatServer.getInstance().getConfig().getSystem().getUseHandshakeV10() == 1; if(useHandshakeV10) { HandshakeV10Packet hs = new HandshakeV10Packet(); hs.packetId = 0; hs.protocolVersion = Versions.PROTOCOL_VERSION; hs.serverVersion = Versions.SERVER_VERSION; hs.threadId = id; hs.seed = rand1; hs.serverCapabilities = getServerCapabilities(); hs.serverCharsetIndex = (byte) (charsetIndex & 0xff); hs.serverStatus = 2; hs.restOfScrambleBuff = rand2; hs.write(this); } else { HandshakePacket hs = new HandshakePacket(); hs.packetId = 0; hs.protocolVersion = Versions.PROTOCOL_VERSION; hs.serverVersion = Versions.SERVER_VERSION; hs.threadId = id; hs.seed = rand1; hs.serverCapabilities = getServerCapabilities(); hs.serverCharsetIndex = (byte) (charsetIndex & 0xff); hs.serverStatus = 2; hs.restOfScrambleBuff = rand2; hs.write(this); } // asynread response this.asynRead(); } } @Override public void handle(final byte[] data) { this.executeSqlId ++; if (isSupportCompress()) { List packs = CompressUtil.decompressMysqlPacket(data, decompressUnfinishedDataQueue); for (byte[] pack : packs) { if (pack.length != 0) { rawHandle(pack); } } } else { rawHandle(data); } } public void rawHandle(final byte[] data) { //load data infile 客户端会发空包 长度为4 if (data.length == 4 && data[0] == 0 && data[1] == 0 && data[2] == 0) { // load in data空包 loadDataInfileEnd(data[3]); return; } //修改quit的判断,当load data infile 分隔符为\001 时可能会出现误判断的bug. if (data.length>4 && data[0] == 1 && data[1] == 0 && data[2]== 0 && data[3] == 0 &&data[4] == MySQLPacket.COM_QUIT) { this.getProcessor().getCommands().doQuit(); this.close("quit cmd"); return; } handler.handle(data); } protected int getServerCapabilities() { int flag = 0; flag |= Capabilities.CLIENT_LONG_PASSWORD; flag |= Capabilities.CLIENT_FOUND_ROWS; flag |= Capabilities.CLIENT_LONG_FLAG; flag |= Capabilities.CLIENT_CONNECT_WITH_DB; // flag |= Capabilities.CLIENT_NO_SCHEMA; boolean usingCompress= MycatServer.getInstance().getConfig().getSystem().getUseCompression()==1 ; if (usingCompress) { flag |= Capabilities.CLIENT_COMPRESS; } flag |= Capabilities.CLIENT_ODBC; flag |= Capabilities.CLIENT_LOCAL_FILES; flag |= Capabilities.CLIENT_IGNORE_SPACE; flag |= Capabilities.CLIENT_PROTOCOL_41; flag |= Capabilities.CLIENT_INTERACTIVE; // flag |= Capabilities.CLIENT_SSL; flag |= Capabilities.CLIENT_IGNORE_SIGPIPE; flag |= Capabilities.CLIENT_TRANSACTIONS; // flag |= ServerDefs.CLIENT_RESERVED; flag |= Capabilities.CLIENT_SECURE_CONNECTION; // flag |= Capabilities.CLIENT_MULTI_STATEMENTS; flag |= Capabilities.CLIENT_MULTI_RESULTS; boolean useHandshakeV10 = MycatServer.getInstance().getConfig().getSystem().getUseHandshakeV10() == 1; if(useHandshakeV10) { flag |= Capabilities.CLIENT_PLUGIN_AUTH; } return flag; } protected boolean isConnectionReset(Throwable t) { if (t instanceof IOException) { String msg = t.getMessage(); return (msg != null && msg.contains("Connection reset by peer")); } return false; } @Override public String toString() { return new StringBuilder().append("[thread=") .append(Thread.currentThread().getName()).append(",class=") .append(getClass().getSimpleName()).append(",id=").append(id) .append(",host=").append(host).append(",port=").append(port) .append(",schema=").append(schema).append(']').toString(); } private final static byte[] encodeString(String src, String charset) { if (src == null) { return null; } if (charset == null) { return src.getBytes(); } try { return src.getBytes(charset); } catch (UnsupportedEncodingException e) { return src.getBytes(); } } @Override public void close(String reason) { super.close(isAuthenticated ? reason : ""); } /** * 设置是否允许批量执行命令 * @param data */ public void setOption(byte[] data) { CommandPacket command = new CommandPacket(); command.read(data); int option = 0; try { option = command.arg[0]; if (option == 0) { this.allowMultiStatements = true; } else if (option == 1) { this.allowMultiStatements = false; } EOFPacket eof = new EOFPacket(); eof.packetId = 1; eof.warningCount = option; write(eof.write(allocate(), this, true)); } catch (Throwable e) { writeErrMessage(ErrorCode.ER_UNKNOWN_COM_ERROR, "Com Set Option Error"); return; } } /** * https://dev.mysql.com/doc/dev/mysql-server/8.0.11/page_protocol_com_reset_connection.html * https://dev.mysql.com/doc/refman/5.7/en/mysql-reset-connection.html * 与连接有关的状态受到以下影响： -将回滚所有活动的事务，并重置自动提交模式。 - 所有表锁均已释放。 -会话系统变量将重新初始化为相应的全局系统变量的值，包括由诸如之类的语句隐式设置的系统变量SET NAMES。 -用户变量设置丢失。 -准备好的语句被释放。 ........ */ public void resetConnection() { // 字符集恢复为UTF8 setCharset("utf8"); write(writeToBuffer(OkPacket.OK, allocate())); } public boolean isEnableFlowController() { return enableFlowController; } public void setEnableFlowController(boolean enableFlowController) { this.enableFlowController = enableFlowController; } public QueueFlowController getFlowController() { return flowController; } /** * * 队列流量控制器，防止队列过大内存OOM，功能： * 1）超过最大阀值，关闭NIO读事件，停止从网络读取mysql数据 * 2）队列恢复到可继续写的阀值，重启NIO读事件，继续写队列 */ public class QueueFlowController { private volatile boolean readIOStopped; // 读事件的IO是否已经停止 private Collection relationedBackendConns;// 关联的后端连接 private final FrontendConnection frontendConn; public QueueFlowController(FrontendConnection c) { this.readIOStopped = false; this.relationedBackendConns = new ArrayList(); this.frontendConn = c; } /** *恢复所有后端连接的读事件 */ private void recoverIORead() { if (readIOStopped) { synchronized (relationedBackendConns) { if (readIOStopped) {// 再次判断，防止并发多次执行 readIOStopped = false; for (final BackendConnection conn : relationedBackendConns) { conn.enableRead(); } relationedBackendConns.clear(); LOGGER.info("The connection[{}] has removed flow control.", frontendConn.toString()); } } } } private void stopIORead(final Collection conns) { if (null != conns && conns.size() > 0) { synchronized (relationedBackendConns) { if (!readIOStopped) {// 再次判断，防止并发多次执行 readIOStopped = true; for (BackendConnection conn : conns) { conn.disableRead(); this.relationedBackendConns.add(conn); } LOGGER.info("Now the connection[{}] is under flow control", frontendConn.toString()); } } } } /** * 检查writeQueue的流量控制阈值 * * @param connection */ public void check(Map target) { int size = writeQueue.size(); if (!readIOStopped && size > writeQueueStopThreshold) { stopIORead(target.values()); } else { if (readIOStopped && size <= writeQueueRecoverThreshold) { recoverIORead(); } } } } } ================================================ FILE: src/main/java/io/mycat/net/NIOAcceptor.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net; import java.io.IOException; import java.net.InetSocketAddress; import java.net.Socket; import java.net.StandardSocketOptions; import java.nio.channels.SelectionKey; import java.nio.channels.Selector; import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; import java.util.Set; import io.mycat.util.SelectorUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.net.factory.FrontendConnectionFactory; /** * @author mycat */ public final class NIOAcceptor extends Thread implements SocketAcceptor{ private static final Logger LOGGER = LoggerFactory.getLogger(NIOAcceptor.class); private static final AcceptIdGenerator ID_GENERATOR = new AcceptIdGenerator(); private final int port; private volatile Selector selector; private final ServerSocketChannel serverChannel; private final FrontendConnectionFactory factory; private long acceptCount; private final NIOReactorPool reactorPool; public NIOAcceptor(String name, String bindIp, int port, int backlog, FrontendConnectionFactory factory, NIOReactorPool reactorPool) throws IOException { super.setName(name); this.port = port; this.selector = Selector.open(); this.serverChannel = ServerSocketChannel.open(); this.serverChannel.configureBlocking(false); /** 设置TCP属性 */ serverChannel.setOption(StandardSocketOptions.SO_REUSEADDR, true); serverChannel.setOption(StandardSocketOptions.SO_RCVBUF, 1024 * 16 * 2); serverChannel.bind(new InetSocketAddress(bindIp, port), backlog); this.serverChannel.register(selector, SelectionKey.OP_ACCEPT); this.factory = factory; this.reactorPool = reactorPool; } public int getPort() { return port; } public long getAcceptCount() { return acceptCount; } @Override public void run() { int invalidSelectCount = 0; for (;;) { try { final Selector tSelector = this.selector; ++acceptCount; long start = System.nanoTime(); tSelector.select(1000L); long end = System.nanoTime(); Set keys = tSelector.selectedKeys(); if (keys.size() == 0 && (end - start) < SelectorUtil.MIN_SELECT_TIME_IN_NANO_SECONDS ) { invalidSelectCount++; } else { try { for (SelectionKey key : keys) { if (key.isValid() && key.isAcceptable()) { accept(); } else { key.cancel(); } } } finally { keys.clear(); invalidSelectCount = 0; } } if (invalidSelectCount > SelectorUtil.REBUILD_COUNT_THRESHOLD) { final Selector rebuildSelector = SelectorUtil.rebuildSelector(this.selector); if (rebuildSelector != null) { this.selector = rebuildSelector; } invalidSelectCount = 0; } } catch (Exception e) { LOGGER.warn(getName(), e); } catch (final Throwable e) { LOGGER.warn("caught Throwable err: ", e); } } } private void accept() { SocketChannel channel = null; try { channel = serverChannel.accept(); channel.configureBlocking(false); FrontendConnection c = factory.make(channel); c.setAccepted(true); c.setId(ID_GENERATOR.getId()); NIOProcessor processor = (NIOProcessor) MycatServer.getInstance() .nextProcessor(); c.setProcessor(processor); NIOReactor reactor = reactorPool.getNextReactor(); reactor.postRegister(c); } catch (Exception e) { LOGGER.warn(getName(), e); closeChannel(channel); } } private static void closeChannel(SocketChannel channel) { if (channel == null) { return; } Socket socket = channel.socket(); if (socket != null) { try { socket.close(); } catch (IOException e) { LOGGER.error("closeChannelError", e); } } try { channel.close(); } catch (IOException e) { LOGGER.error("closeChannelError", e); } } /** * 前端连接ID生成器 * * @author mycat */ private static class AcceptIdGenerator { private static final long MAX_VALUE = 0xffffffffL; private long acceptId = 0L; private final Object lock = new Object(); private long getId() { synchronized (lock) { if (acceptId >= MAX_VALUE) { acceptId = 0L; } return ++acceptId; } } } } ================================================ FILE: src/main/java/io/mycat/net/NIOConnection.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net; import java.io.IOException; import java.nio.ByteBuffer; /** * @author mycat */ public interface NIOConnection extends ClosableConnection{ /** * connected */ void register() throws IOException; /** * 处理数据 */ void handle(byte[] data); /** * 写出一块缓存数据 */ void write(ByteBuffer buffer); } ================================================ FILE: src/main/java/io/mycat/net/NIOConnector.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.channels.SelectionKey; import java.nio.channels.Selector; import java.nio.channels.SocketChannel; import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import java.util.concurrent.atomic.AtomicLong; import io.mycat.util.SelectorUtil; /** * @author mycat */ public final class NIOConnector extends Thread implements SocketConnector { private static final Logger LOGGER = LoggerFactory.getLogger(NIOConnector.class); public static final ConnectIdGenerator ID_GENERATOR = new ConnectIdGenerator(); private final String name; private volatile Selector selector; private final BlockingQueue connectQueue; private long connectCount; private final NIOReactorPool reactorPool; public NIOConnector(String name, NIOReactorPool reactorPool) throws IOException { super.setName(name); this.name = name; this.selector = Selector.open(); this.reactorPool = reactorPool; this.connectQueue = new LinkedBlockingQueue(); } public long getConnectCount() { return connectCount; } public void postConnect(AbstractConnection c) { connectQueue.offer(c); selector.wakeup(); } @Override public void run() { int invalidSelectCount = 0; for (;;) { try { final Selector tSelector = this.selector; ++connectCount; long start = System.nanoTime(); tSelector.select(1000L); long end = System.nanoTime(); connect(tSelector); Set keys = tSelector.selectedKeys(); if (keys.size() == 0 && (end - start) < SelectorUtil.MIN_SELECT_TIME_IN_NANO_SECONDS ) { invalidSelectCount++; } else { try { for (SelectionKey key : keys) { Object att = key.attachment(); if (att != null && key.isValid() && key.isConnectable()) { finishConnect(key, att); } else { key.cancel(); } } } finally { invalidSelectCount = 0; keys.clear(); } } if (invalidSelectCount > SelectorUtil.REBUILD_COUNT_THRESHOLD) { final Selector rebuildSelector = SelectorUtil.rebuildSelector(this.selector); if (rebuildSelector != null) { this.selector = rebuildSelector; } invalidSelectCount = 0; } } catch (Exception e) { LOGGER.warn(name, e); } catch (final Throwable e) { LOGGER.warn("caught Throwable err: ", e); } } } private void connect(Selector selector) { AbstractConnection c = null; while ((c = connectQueue.poll()) != null) { try { SocketChannel channel = (SocketChannel) c.getChannel(); channel.register(selector, SelectionKey.OP_CONNECT, c); channel.connect(new InetSocketAddress(c.host, c.port)); } catch (Exception e) { LOGGER.error("error:",e); c.close(e.toString()); } } } private void finishConnect(SelectionKey key, Object att) { BackendAIOConnection c = (BackendAIOConnection) att; try { if (finishConnect(c, (SocketChannel) c.channel)) { clearSelectionKey(key); c.setId(ID_GENERATOR.getId()); NIOProcessor processor = MycatServer.getInstance() .nextProcessor(); c.setProcessor(processor); NIOReactor reactor = reactorPool.getNextReactor(); reactor.postRegister(c); c.onConnectfinish(); } } catch (Exception e) { clearSelectionKey(key); LOGGER.error("error:",e); c.close(e.toString()); c.onConnectFailed(e); } } private boolean finishConnect(AbstractConnection c, SocketChannel channel) throws IOException { if (channel.finishConnect()) { channel.finishConnect(); c.setLocalPort(channel.socket().getLocalPort()); return true; } else { return false; } } private void clearSelectionKey(SelectionKey key) { if (key.isValid()) { key.attach(null); key.cancel(); } } /** * 后端连接ID生成器 * * @author mycat */ public static class ConnectIdGenerator { private static final long MAX_VALUE = Long.MAX_VALUE; private AtomicLong connectId = new AtomicLong(0); public long getId() { return connectId.incrementAndGet(); } } } ================================================ FILE: src/main/java/io/mycat/net/NIOHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net; /** * @author mycat */ public interface NIOHandler { void handle(byte[] data); } ================================================ FILE: src/main/java/io/mycat/net/NIOProcessor.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net; import java.io.IOException; import java.util.Iterator; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; import io.mycat.buffer.BufferPool; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.backend.BackendConnection; import io.mycat.statistic.CommandCount; import io.mycat.util.NameableExecutor; import io.mycat.util.TimeUtil; /** * @author mycat */ public final class NIOProcessor { private static final Logger LOGGER = LoggerFactory.getLogger("NIOProcessor"); private final String name; private final BufferPool bufferPool; private final NameableExecutor executor; private final ConcurrentMap frontends; private final ConcurrentMap backends; private final CommandCount commands; private long netInBytes; private long netOutBytes; // TODO: add by zhuam // reload @@config_all 后, 老的backends 全部移往 backends_old, 待检测任务进行销毁 public final static ConcurrentLinkedQueue backends_old = new ConcurrentLinkedQueue(); //前端已连接数 private AtomicInteger frontendsLength = new AtomicInteger(0); public NIOProcessor(String name, BufferPool bufferPool, NameableExecutor executor) throws IOException { this.name = name; this.bufferPool = bufferPool; this.executor = executor; this.frontends = new ConcurrentHashMap(); this.backends = new ConcurrentHashMap(); this.commands = new CommandCount(); } public String getName() { return name; } public BufferPool getBufferPool() { return bufferPool; } public int getWriteQueueSize() { int total = 0; for (FrontendConnection fron : frontends.values()) { total += fron.getWriteQueue().size(); } for (BackendConnection back : backends.values()) { if (back instanceof BackendAIOConnection) { total += ((BackendAIOConnection) back).getWriteQueue().size(); } } return total; } public NameableExecutor getExecutor() { return this.executor; } public CommandCount getCommands() { return this.commands; } public long getNetInBytes() { return this.netInBytes; } public void addNetInBytes(long bytes) { this.netInBytes += bytes; } public long getNetOutBytes() { return this.netOutBytes; } public void addNetOutBytes(long bytes) { this.netOutBytes += bytes; } public void addFrontend(FrontendConnection c) { this.frontends.put(c.getId(), c); this.frontendsLength.incrementAndGet(); } public ConcurrentMap getFrontends() { return this.frontends; } public int getForntedsLength(){ return this.frontendsLength.get(); } public void addBackend(BackendConnection c) { this.backends.put(c.getId(), c); } public ConcurrentMap getBackends() { return this.backends; } /** * 定时执行该方法，回收部分资源。 */ public void checkBackendCons() { backendCheck(); } /** * 定时执行该方法，回收部分资源。 */ public void checkFrontCons() { frontendCheck(); } // 前端连接检查 private void frontendCheck() { Iterator> it = frontends.entrySet() .iterator(); while (it.hasNext()) { FrontendConnection c = it.next().getValue(); // 删除空连接 if (c == null) { it.remove(); this.frontendsLength.decrementAndGet(); continue; } // 清理已关闭连接，否则空闲检查。 if (c.isClosed()) { // 此处在高并发情况下会存在并发问题, fixed #1072 极有可能解决了 #700 //c.cleanup(); it.remove(); this.frontendsLength.decrementAndGet(); } else { // very important ,for some data maybe not sent checkConSendQueue(c); c.idleCheck(); } } } private void checkConSendQueue(AbstractConnection c) { // very important ,for some data maybe not sent if (!c.writeQueue.isEmpty()) { c.getSocketWR().doNextWriteCheck(); } if (c.isEnableFlowController()) { c.checkQueueFlow(); } } // 后端连接检查 private void backendCheck() { long sqlTimeout = MycatServer.getInstance().getConfig().getSystem().getSqlExecuteTimeout() * 1000L; Iterator> it = backends.entrySet().iterator(); while (it.hasNext()) { BackendConnection c = it.next().getValue(); // 删除空连接 if (c == null) { it.remove(); continue; } // SQL执行超时的连接关闭 if (c.isBorrowed() && c.getLastTime() < TimeUtil.currentTimeMillis() - sqlTimeout) { LOGGER.warn("found backend connection SQL timeout ,close it " + c); c.close("sql timeout"); } // 清理已关闭连接，否则空闲检查。 if (c.isClosed()) { it.remove(); } else { // very important ,for some data maybe not sent if (c instanceof AbstractConnection) { checkConSendQueue((AbstractConnection) c); } c.idleCheck(); } } } public void removeConnection(AbstractConnection con) { if (con instanceof BackendConnection) { this.backends.remove(con.getId()); } else { this.frontends.remove(con.getId()); this.frontendsLength.decrementAndGet(); } } //jdbc连接用这个释放 public void removeConnection(BackendConnection con){ this.backends.remove(con.getId()); } } ================================================ FILE: src/main/java/io/mycat/net/NIOReactor.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net; import java.io.IOException; import java.nio.channels.CancelledKeyException; import java.nio.channels.SelectionKey; import java.nio.channels.Selector; import java.util.Queue; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import io.mycat.util.SelectorUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * 网络事件反应器 * *

* Catch exceptions such as OOM so that the reactor can keep running for response client! *

* @since 2016-03-30 * * @author mycat, Uncle-pan * */ public final class NIOReactor { private static final Logger LOGGER = LoggerFactory.getLogger(NIOReactor.class); private final String name; private final RW reactorR; public NIOReactor(String name) throws IOException { this.name = name; this.reactorR = new RW(); } final void startup() { new Thread(reactorR, name + "-RW").start(); } final void postRegister(AbstractConnection c) { reactorR.registerQueue.offer(c); reactorR.selector.wakeup(); } final Queue getRegisterQueue() { return reactorR.registerQueue; } final long getReactCount() { return reactorR.reactCount; } private final class RW implements Runnable { private volatile Selector selector; private final ConcurrentLinkedQueue registerQueue; private long reactCount; private RW() throws IOException { this.selector = Selector.open(); this.registerQueue = new ConcurrentLinkedQueue(); } @Override public void run() { int invalidSelectCount = 0; Set keys = null; for (;;) { ++reactCount; try { final Selector tSelector = this.selector; long start = System.nanoTime(); tSelector.select(500L); long end = System.nanoTime(); register(tSelector); keys = tSelector.selectedKeys(); if (keys.size() == 0 && (end - start) < SelectorUtil.MIN_SELECT_TIME_IN_NANO_SECONDS ) { invalidSelectCount++; } else { invalidSelectCount = 0; for (SelectionKey key : keys) { AbstractConnection con = null; try { Object att = key.attachment(); if (att != null) { con = (AbstractConnection) att; if (key.isValid() && key.isReadable()) { try { con.asynRead(); } catch (IOException e) { con.close("program err:" + e.toString()); continue; } catch (Exception e) { LOGGER.warn("caught err:", e); con.close("program err:" + e.toString()); continue; } } if (key.isValid() && key.isWritable()) { con.doNextWriteCheck(); } } else { key.cancel(); } } catch (CancelledKeyException e) { if (LOGGER.isDebugEnabled()) { LOGGER.debug(con + " socket key canceled"); } } catch (Exception e) { LOGGER.warn(con + " " + e); } catch (final Throwable e) { // Catch exceptions such as OOM and close connection if exists //so that the reactor can keep running! // @author Uncle-pan // @since 2016-03-30 if (con != null) { con.close("Bad: " + e); } LOGGER.error("caught err: ", e); continue; } } } if (invalidSelectCount > SelectorUtil.REBUILD_COUNT_THRESHOLD) { final Selector rebuildSelector = SelectorUtil.rebuildSelector(this.selector); if (rebuildSelector != null) { this.selector = rebuildSelector; } invalidSelectCount = 0; } } catch (Exception e) { LOGGER.warn(name, e); } catch (final Throwable e){ // Catch exceptions such as OOM so that the reactor can keep running! // @author Uncle-pan // @since 2016-03-30 LOGGER.error("caught err: ", e); } finally { if (keys != null) { keys.clear(); } } } } private void register(Selector selector) { AbstractConnection c = null; if (registerQueue.isEmpty()) { return; } while ((c = registerQueue.poll()) != null) { try { ((NIOSocketWR) c.getSocketWR()).register(selector); c.register(); } catch (Exception e) { c.close("register err" + e.toString()); } } } } } ================================================ FILE: src/main/java/io/mycat/net/NIOReactorPool.java ================================================ package io.mycat.net; import java.io.IOException; public class NIOReactorPool { private final NIOReactor[] reactors; private volatile int nextReactor; public NIOReactorPool(String name, int poolSize) throws IOException { reactors = new NIOReactor[poolSize]; for (int i = 0; i < poolSize; i++) { NIOReactor reactor = new NIOReactor(name + "-" + i); reactors[i] = reactor; reactor.startup(); } } public NIOReactor getNextReactor() { // if (++nextReactor == reactors.length) { // nextReactor = 0; // } // return reactors[nextReactor]; int i = ++nextReactor; if (i >= reactors.length) { i=nextReactor = 0; } return reactors[i]; } } ================================================ FILE: src/main/java/io/mycat/net/NIOSocketWR.java ================================================ package io.mycat.net; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.SelectionKey; import java.nio.channels.Selector; import java.nio.channels.SocketChannel; import java.util.concurrent.atomic.AtomicBoolean; import io.mycat.util.TimeUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class NIOSocketWR extends SocketWR { private SelectionKey processKey; private static final int OP_NOT_READ = ~SelectionKey.OP_READ; private static final int OP_NOT_WRITE = ~SelectionKey.OP_WRITE; private final AbstractConnection con; private final SocketChannel channel; private final AtomicBoolean writing = new AtomicBoolean(false); protected static final Logger LOGGER = LoggerFactory.getLogger(NIOSocketWR.class); public static final ByteBuffer EMPTY_BYTEBUFFER = ByteBuffer.allocate(1); public NIOSocketWR(AbstractConnection con) { this.con = con; this.channel = (SocketChannel) con.channel; } public void register(Selector selector) throws IOException { try { processKey = channel.register(selector, SelectionKey.OP_READ, con); } finally { if (con.isClosed.get()) { clearSelectionKey(); } } } public void doNextWriteCheck() { if (!writing.compareAndSet(false, true)) { return; } try { if(!channel.isOpen()){ AbstractConnection.LOGGER.debug("caught err: {}", con); } boolean noMoreData = write0(); writing.set(false); if (noMoreData && con.writeQueue.isEmpty()) { if ((processKey.isValid() && (processKey.interestOps() & SelectionKey.OP_WRITE) != 0)) { disableWrite(); } } else { if ((processKey.isValid() && (processKey.interestOps() & SelectionKey.OP_WRITE) == 0)) { enableWrite(false); } } } catch (IOException e) { if (AbstractConnection.LOGGER.isWarnEnabled()) { AbstractConnection.LOGGER.warn("caught err:", e); } con.close("err:" + e); } finally { writing.set(false); } } @Override public boolean checkAlive() { try { return channel.read(EMPTY_BYTEBUFFER) == 0; } catch (IOException e) { LOGGER.error("",e); return false; }finally { EMPTY_BYTEBUFFER.position(0); } } private boolean write0() throws IOException { int written = 0; ByteBuffer buffer = con.writeBuffer; if (buffer != null) { while (buffer.hasRemaining()) { written = channel.write(buffer); if (written > 0) { con.netOutBytes += written; con.processor.addNetOutBytes(written); con.lastWriteTime = TimeUtil.currentTimeMillis(); } else { break; } } if (buffer.hasRemaining()) { con.writeAttempts++; return false; } else { con.writeBuffer = null; con.recycle(buffer); } } while ((buffer = con.writeQueue.poll()) != null) { if (buffer.limit() == 0) { con.recycle(buffer); con.close("quit send"); return true; } buffer.flip(); try { while (buffer.hasRemaining()) { written = channel.write(buffer);// java.io.IOException: // Connection reset by peer if (written > 0) { con.lastWriteTime = TimeUtil.currentTimeMillis(); con.netOutBytes += written; con.processor.addNetOutBytes(written); con.lastWriteTime = TimeUtil.currentTimeMillis(); } else { break; } } } catch (IOException e) { con.recycle(buffer); throw e; } if (buffer.hasRemaining()) { con.writeBuffer = buffer; con.writeAttempts++; return false; } else { con.recycle(buffer); } } return true; } private void disableWrite() { try { SelectionKey key = this.processKey; key.interestOps(key.interestOps() & OP_NOT_WRITE); } catch (Exception e) { AbstractConnection.LOGGER.warn("can't disable write " + e + " con " + con); } } private void enableWrite(boolean wakeup) { boolean needWakeup = false; try { SelectionKey key = this.processKey; key.interestOps(key.interestOps() | SelectionKey.OP_WRITE); needWakeup = true; } catch (Exception e) { AbstractConnection.LOGGER.warn("can't enable write " + e); } if (needWakeup && wakeup) { processKey.selector().wakeup(); } } public void disableRead() { SelectionKey key = this.processKey; key.interestOps(key.interestOps() & OP_NOT_READ); } public void enableRead() { boolean needWakeup = false; try { SelectionKey key = this.processKey; key.interestOps(key.interestOps() | SelectionKey.OP_READ); needWakeup = true; } catch (Exception e) { AbstractConnection.LOGGER.warn("enable read fail " + e); } if (needWakeup) { processKey.selector().wakeup(); } } private void clearSelectionKey() { try { SelectionKey key = this.processKey; if (key != null && key.isValid()) { key.attach(null); key.cancel(); } } catch (Exception e) { AbstractConnection.LOGGER.warn("clear selector keys err:" + e); } } @Override public void asynRead() throws IOException { ByteBuffer theBuffer = con.readBuffer; if (theBuffer == null) { theBuffer = con.processor.getBufferPool().allocate(con.processor.getBufferPool().getChunkSize()); con.readBuffer = theBuffer; } int got = channel.read(theBuffer); con.onReadData(got); } } ================================================ FILE: src/main/java/io/mycat/net/SocketAcceptor.java ================================================ package io.mycat.net; public interface SocketAcceptor { void start(); String getName(); int getPort(); } ================================================ FILE: src/main/java/io/mycat/net/SocketConnector.java ================================================ package io.mycat.net; public interface SocketConnector { } ================================================ FILE: src/main/java/io/mycat/net/SocketWR.java ================================================ package io.mycat.net; import java.io.IOException; public abstract class SocketWR { public abstract void asynRead() throws IOException; public abstract void doNextWriteCheck() ; public abstract boolean checkAlive(); public abstract void disableRead(); public abstract void enableRead(); } ================================================ FILE: src/main/java/io/mycat/net/WriteEventCheckRunner.java ================================================ package io.mycat.net; public class WriteEventCheckRunner implements Runnable { private final SocketWR socketWR; private volatile boolean finshed = true; public WriteEventCheckRunner(SocketWR socketWR) { this.socketWR = socketWR; } public boolean isFinished() { return finshed; } @Override public void run() { finshed = false; socketWR.doNextWriteCheck(); finshed = true; } } ================================================ FILE: src/main/java/io/mycat/net/factory/BackendConnectionFactory.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net.factory; import java.io.IOException; import java.nio.channels.AsynchronousSocketChannel; import java.nio.channels.NetworkChannel; import java.nio.channels.SocketChannel; import io.mycat.MycatServer; /** * @author mycat */ public abstract class BackendConnectionFactory { protected NetworkChannel openSocketChannel(boolean isAIO) throws IOException { if (isAIO) { return AsynchronousSocketChannel .open(MycatServer.getInstance().getNextAsyncChannelGroup()); } else { SocketChannel channel = null; channel = SocketChannel.open(); channel.configureBlocking(false); return channel; } } } ================================================ FILE: src/main/java/io/mycat/net/factory/FrontendConnectionFactory.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net.factory; import java.io.IOException; import java.net.StandardSocketOptions; import java.nio.channels.NetworkChannel; import io.mycat.MycatServer; import io.mycat.net.FrontendConnection; /** * @author mycat */ public abstract class FrontendConnectionFactory { protected abstract FrontendConnection getConnection(NetworkChannel channel) throws IOException; public FrontendConnection make(NetworkChannel channel) throws IOException { channel.setOption(StandardSocketOptions.SO_REUSEADDR, true); channel.setOption(StandardSocketOptions.SO_KEEPALIVE, true); FrontendConnection c = getConnection(channel); MycatServer.getInstance().getConfig().setSocketParams(c, true); return c; } } ================================================ FILE: src/main/java/io/mycat/net/handler/BackendAsyncHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net.handler; import java.util.concurrent.Executor; import io.mycat.net.NIOHandler; /** * @author mycat */ public abstract class BackendAsyncHandler implements NIOHandler { protected void offerData(byte[] data, Executor executor) { handleData(data); // if (dataQueue.offer(data)) { // handleQueue(executor); // } else { // offerDataError(); // } } protected abstract void offerDataError(); protected abstract void handleData(byte[] data); } ================================================ FILE: src/main/java/io/mycat/net/handler/FrontendAuthenticator.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net.handler; import java.nio.ByteBuffer; import java.security.NoSuchAlgorithmException; import java.util.Iterator; import java.util.Map; import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.backend.mysql.SecurityUtil; import io.mycat.config.Capabilities; import io.mycat.config.ErrorCode; import io.mycat.config.model.UserConfig; import io.mycat.net.FrontendConnection; import io.mycat.net.NIOHandler; import io.mycat.net.NIOProcessor; import io.mycat.net.mysql.AuthPacket; import io.mycat.net.mysql.AuthSwitchPacket; import io.mycat.net.mysql.MySQLPacket; import io.mycat.net.mysql.QuitPacket; /** * 前端认证处理器 * * @author mycat */ public class FrontendAuthenticator implements NIOHandler { private static final Logger LOGGER = LoggerFactory.getLogger(FrontendAuthenticator.class); private byte[] AUTH_OK = new byte[] { 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0 }; private static final String DEFAULT_AUTH_PLUGIN_NAME = "mysql_native_password"; protected final FrontendConnection source; private AuthPacket auth; public FrontendAuthenticator(FrontendConnection source) { this.source = source; } @Override public void handle(byte[] data) { // check quit packet if (data.length == QuitPacket.QUIT.length && data[4] == MySQLPacket.COM_QUIT) { source.close("quit packet"); return; } if(auth == null){ auth = new AuthPacket(); auth.read(data); if (auth.clientAuthPlugin != null && !DEFAULT_AUTH_PLUGIN_NAME.equals(auth.clientAuthPlugin)) { AuthSwitchPacket authSwitch = new AuthSwitchPacket(DEFAULT_AUTH_PLUGIN_NAME.getBytes(), source.getSeed()); authSwitch.packetId = 2; authSwitch.write(this.source); return; } }else{ AuthSwitchPacket authSwitch = new AuthSwitchPacket(); authSwitch.read(data); auth.password = authSwitch.getAuthMethodData(); AUTH_OK = new byte[] { 7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0 }; } //huangyiming add int nopassWordLogin = MycatServer.getInstance().getConfig().getSystem().getNonePasswordLogin(); //如果无密码登陆则跳过密码验证这个步骤 boolean skipPassWord = false; String defaultUser = ""; if(nopassWordLogin == 1){ skipPassWord = true; Map userMaps = MycatServer.getInstance().getConfig().getUsers(); if(!userMaps.isEmpty()){ setDefaultAccount(auth, userMaps); } } // check user if (!checkUser(auth.user, source.getHost())) { failure(ErrorCode.ER_ACCESS_DENIED_ERROR, "Access denied for user '" + auth.user + "' with host '" + source.getHost()+ "'"); return; } // check password if (!skipPassWord && !checkPassword(auth.password, auth.user)) { failure(ErrorCode.ER_ACCESS_DENIED_ERROR, "Access denied for user '" + auth.user + "', because password is error "); return; } // check degrade if ( isDegrade( auth.user ) ) { failure(ErrorCode.ER_ACCESS_DENIED_ERROR, "Access denied for user '" + auth.user + "', because service be degraded "); return; } // check schema switch (checkSchema(auth.database, auth.user)) { case ErrorCode.ER_BAD_DB_ERROR: failure(ErrorCode.ER_BAD_DB_ERROR, "Unknown database '" + auth.database + "'"); break; case ErrorCode.ER_DBACCESS_DENIED_ERROR: String s = "Access denied for user '" + auth.user + "' to database '" + auth.database + "'"; failure(ErrorCode.ER_DBACCESS_DENIED_ERROR, s); break; default: success(auth); } } /** * 设置了无密码登陆的情况下把客户端传过来的用户账号改变为默认账户 * @param auth * @param userMaps */ private void setDefaultAccount(AuthPacket auth, Map userMaps) { String defaultUser; Iterator items = userMaps.values().iterator(); while(items.hasNext()){ UserConfig userConfig = items.next(); if(userConfig.isDefaultAccount()){ defaultUser = userConfig.getName(); auth.user = defaultUser; } } } //TODO: add by zhuam //前端 connection 达到该用户设定的阀值后, 立马降级拒绝连接 protected boolean isDegrade(String user) { int benchmark = source.getPrivileges().getBenchmark(user); if ( benchmark > 0 ) { int forntedsLength = 0; NIOProcessor[] processors = MycatServer.getInstance().getProcessors(); for (NIOProcessor p : processors) { forntedsLength += p.getForntedsLength(); } if ( forntedsLength >= benchmark ) { return true; } } return false; } protected boolean checkUser(String user, String host) { return source.getPrivileges().userExists(user, host); } protected boolean checkPassword(byte[] password, String user) { String pass = source.getPrivileges().getPassword(user); // check null if (pass == null || pass.length() == 0) { if (password == null || password.length == 0) { return true; } else { return false; } } if (password == null || password.length == 0) { return false; } // encrypt byte[] encryptPass = null; try { encryptPass = SecurityUtil.scramble411(pass.getBytes(), source.getSeed()); } catch (NoSuchAlgorithmException e) { LOGGER.warn(source.toString(), e); return false; } if (encryptPass != null && (encryptPass.length == password.length)) { int i = encryptPass.length; while (i-- != 0) { if (encryptPass[i] != password[i]) { return false; } } } else { return false; } return true; } protected int checkSchema(String schema, String user) { if (schema == null) { return 0; } FrontendPrivileges privileges = source.getPrivileges(); if (!privileges.schemaExists(schema)) { return ErrorCode.ER_BAD_DB_ERROR; } Set schemas = privileges.getUserSchemas(user); if (schemas == null || schemas.size() == 0 || schemas.contains(schema)) { return 0; } else { return ErrorCode.ER_DBACCESS_DENIED_ERROR; } } protected void success(AuthPacket auth) { source.setAuthenticated(true); source.setUser(auth.user); source.setSchema(auth.database); source.setCharsetIndex(auth.charsetIndex); if (auth.allowMultiStatements) { /** * #2589 url里面包含allowMultiQueries=true，allowMultiStatements这个参数会在握手协议时候返回 * mysql命令行客户端或navicat都会把这个参数设置为true，表示它会发送”multiple statements “ * 参考https://dev.mysql.com/doc/internals/en/capability-flags.html。 * 因为mycat目前不能支持这个特性，只能先日志记录下，便于以后排查问题 **/ String warnMsg = "Mycat does not support the allowMultiQueries value to be true, maybe occur some error.Client connection is[{}]"; if (LOGGER.isInfoEnabled()) { LOGGER.info(warnMsg, this); } // failure(ErrorCode.ER_HANDSHAKE_ERROR, errMsg); // return; } source.setHandler(new FrontendCommandHandler(source)); if (LOGGER.isInfoEnabled()) { StringBuilder s = new StringBuilder(); s.append(source).append('\'').append(auth.user).append("' login success"); byte[] extra = auth.extra; if (extra != null && extra.length > 0) { s.append(",extra:").append(new String(extra)); } LOGGER.info(s.toString()); } ByteBuffer buffer = source.allocate(); source.write(source.writeToBuffer(AUTH_OK, buffer)); boolean clientCompress = Capabilities.CLIENT_COMPRESS==(Capabilities.CLIENT_COMPRESS & auth.clientFlags); boolean usingCompress= MycatServer.getInstance().getConfig().getSystem().getUseCompression()==1 ; if(clientCompress&&usingCompress) { source.setSupportCompress(true); } } protected void failure(int errno, String info) { LOGGER.error(source.toString() + info); source.writeErrMessage((byte) 2, errno, info); } } ================================================ FILE: src/main/java/io/mycat/net/handler/FrontendCommandHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net.handler; import io.mycat.MycatServer; import io.mycat.backend.mysql.MySQLMessage; import io.mycat.config.ErrorCode; import io.mycat.config.MycatConfig; import io.mycat.net.FrontendConnection; import io.mycat.net.NIOHandler; import io.mycat.net.mysql.MySQLPacket; import io.mycat.statistic.CommandCount; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * 前端命令处理器 * * @author mycat */ public class FrontendCommandHandler implements NIOHandler { private static final Logger LOGGER = LoggerFactory.getLogger(FrontendCommandHandler.class); protected final FrontendConnection source; protected final CommandCount commands; public FrontendCommandHandler(FrontendConnection source) { this.source = source; this.commands = source.getProcessor().getCommands(); } @Override public void handle(byte[] data) { if(source.getLoadDataInfileHandler()!=null&&source.getLoadDataInfileHandler().isStartLoadData()) { MySQLMessage mm = new MySQLMessage(data); int packetLength = mm.readUB3(); if(packetLength+4==data.length) { source.loadDataInfileData(data); } return; } switch (data[4]) { case MySQLPacket.COM_INIT_DB: commands.doInitDB(); source.initDB(data); break; case MySQLPacket.COM_QUERY: commands.doQuery(); source.query(data); break; case MySQLPacket.COM_PING: commands.doPing(); source.ping(); break; case MySQLPacket.COM_QUIT: commands.doQuit(); source.close("quit cmd"); break; case MySQLPacket.COM_PROCESS_KILL: commands.doKill(); source.kill(data); break; case MySQLPacket.COM_STMT_PREPARE: commands.doStmtPrepare(); source.stmtPrepare(data); break; case MySQLPacket.COM_STMT_SEND_LONG_DATA: commands.doStmtSendLongData(); source.stmtSendLongData(data); break; case MySQLPacket.COM_STMT_RESET: commands.doStmtReset(); source.stmtReset(data); break; case MySQLPacket.COM_STMT_EXECUTE: commands.doStmtExecute(); source.stmtExecute(data); break; case MySQLPacket.COM_STMT_CLOSE: commands.doStmtClose(); source.stmtClose(data); break; case MySQLPacket.COM_HEARTBEAT: commands.doHeartbeat(); source.heartbeat(data); break; case MySQLPacket.COM_FIELD_LIST: source.fieldList(data); break; case MySQLPacket.COM_SET_OPTION: commands.doSetOption(); source.setOption(data); break; case MySQLPacket.COM_RESET_CONNECTION: source.resetConnection(); break; default: commands.doOther(); MycatConfig config = MycatServer.getInstance().getConfig(); if( config.getSystem().getIgnoreUnknownCommand()==1){ LOGGER.warn("Unknown command:{}",data[4]); source.ping(); }else { LOGGER.error("Unknown command:{}",new String(data)); source.writeErrMessage(ErrorCode.ER_UNKNOWN_COM_ERROR, "Unknown command"); } } } } ================================================ FILE: src/main/java/io/mycat/net/handler/FrontendPrepareHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net.handler; /** * SQL预处理处理器 * * @author mycat, CrazyPig */ public interface FrontendPrepareHandler { void prepare(String sql); void sendLongData(byte[] data); void reset(byte[] data); void execute(byte[] data); void close(byte[] data); void clear(); } ================================================ FILE: src/main/java/io/mycat/net/handler/FrontendPrivileges.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net.handler; import java.util.Set; /** * 权限提供者 * * @author mycat */ public interface FrontendPrivileges { /** * 检查schema是否存在 */ boolean schemaExists(String schema); /** * 检查用户是否存在，并且可以使用host实行隔离策略。 */ boolean userExists(String user, String host); /** * 提供用户的服务器端密码 */ String getPassword(String user); /** * 提供有效的用户schema集合 */ Set getUserSchemas(String user); /** * 检查用户是否为只读权限 * @param user * @return */ Boolean isReadOnly(String user); /** * 获取设定的系统最大连接数的降级阀值 * @param user * @return */ int getBenchmark(String user); /** * 检查防火墙策略 * （白名单策略） * @param user * @param host * @return */ boolean checkFirewallWhiteHostPolicy(String user, String host); /** * 检查防火墙策略 * (SQL黑名单及注入策略) * @param sql * @return */ boolean checkFirewallSQLPolicy(String user, String sql); /** * 检查 SQL 语句的 DML 权限 * @return */ boolean checkDmlPrivilege(String user, String schema, String sql); /** * 检查针对 DataNode 的 SQL 语句的 DML 权限 * @param user * @param dataNode * @param sql * @return */ boolean checkDataNodeDmlPrivilege(String user, String dataNode, String sql); } ================================================ FILE: src/main/java/io/mycat/net/handler/FrontendQueryHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net.handler; /** * 查询处理器 * * @author mycat */ public interface FrontendQueryHandler { void query(String sql); void setReadOnly(Boolean readOnly); } ================================================ FILE: src/main/java/io/mycat/net/handler/LoadDataInfileHandler.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net.handler; /** * load data infile * * @author magicdoom */ public interface LoadDataInfileHandler { void start(String sql); void handle(byte[] data); void end(byte packID); void clear(); byte getLastPackId(); boolean isStartLoadData(); } ================================================ FILE: src/main/java/io/mycat/net/mysql/AuthPacket.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net.mysql; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; import io.mycat.backend.mysql.BufferUtil; import io.mycat.backend.mysql.MySQLMessage; import io.mycat.backend.mysql.StreamUtil; import io.mycat.config.Capabilities; import io.mycat.net.BackendAIOConnection; /** * From client to server during initial handshake. * * 
 * Bytes                        Name
 * -----                        ----
 * 4                            client_flags
 * 4                            max_packet_size
 * 1                            charset_number
 * 23                           (filler) always 0x00...
 * n (Null-Terminated String)   user
 * n (Length Coded Binary)      scramble_buff (1 + x bytes)
 * n (Null-Terminated String)   databasename (optional)
 * 
 * @see http://forge.mysql.com/wiki/MySQL_Internals_ClientServer_Protocol#Client_Authentication_Packet
 *
* * @author mycat */ public class AuthPacket extends MySQLPacket { private static final byte[] FILLER = new byte[23]; public long clientFlags; public long maxPacketSize; public int charsetIndex; public byte[] extra;// from FILLER(23) public String user; public byte[] password; public String database; public boolean allowMultiStatements; public String clientAuthPlugin; public void read(byte[] data) { MySQLMessage mm = new MySQLMessage(data); packetLength = mm.readUB3(); packetId = mm.read(); clientFlags = mm.readUB4(); maxPacketSize = mm.readUB4(); charsetIndex = (mm.read() & 0xff); // read extra int current = mm.position(); int len = (int) mm.readLength(); if (len > 0 && len < FILLER.length) { byte[] ab = new byte[len]; System.arraycopy(mm.bytes(), mm.position(), ab, 0, len); this.extra = ab; } mm.position(current + FILLER.length); user = mm.readStringWithNull(); password = mm.readBytesWithLength(); if (((clientFlags & Capabilities.CLIENT_CONNECT_WITH_DB) != 0) && mm.hasRemaining()) { database = mm.readStringWithNull(); } if ((clientFlags & Capabilities.CLIENT_MULTI_STATEMENTS) != 0) { allowMultiStatements = true; } if (((clientFlags & Capabilities.CLIENT_PLUGIN_AUTH) != 0) && mm.hasRemaining()) { clientAuthPlugin = mm.readStringWithNull(); } } public void write(OutputStream out) throws IOException { StreamUtil.writeUB3(out, calcPacketSize()); StreamUtil.write(out, packetId); StreamUtil.writeUB4(out, clientFlags); StreamUtil.writeUB4(out, maxPacketSize); StreamUtil.write(out, (byte) charsetIndex); out.write(FILLER); if (user == null) { StreamUtil.write(out, (byte) 0); } else { StreamUtil.writeWithNull(out, user.getBytes()); } if (password == null) { StreamUtil.write(out, (byte) 0); } else { StreamUtil.writeWithLength(out, password); } if (database == null) { StreamUtil.write(out, (byte) 0); } else { StreamUtil.writeWithNull(out, database.getBytes()); } } @Override public void write(BackendAIOConnection c) { ByteBuffer buffer = c.allocate(); BufferUtil.writeUB3(buffer, calcPacketSize()); buffer.put(packetId); BufferUtil.writeUB4(buffer, clientFlags); BufferUtil.writeUB4(buffer, maxPacketSize); buffer.put((byte) charsetIndex); buffer = c.writeToBuffer(FILLER, buffer); if (user == null) { buffer = c.checkWriteBuffer(buffer, 1,true); buffer.put((byte) 0); } else { byte[] userData = user.getBytes(); buffer = c.checkWriteBuffer(buffer, userData.length + 1,true); BufferUtil.writeWithNull(buffer, userData); } if (password == null) { buffer = c.checkWriteBuffer(buffer, 1,true); buffer.put((byte) 0); } else { buffer = c.checkWriteBuffer(buffer, BufferUtil.getLength(password),true); BufferUtil.writeWithLength(buffer, password); } if (database == null) { buffer = c.checkWriteBuffer(buffer, 1,true); buffer.put((byte) 0); } else { byte[] databaseData = database.getBytes(); buffer = c.checkWriteBuffer(buffer, databaseData.length + 1,true); BufferUtil.writeWithNull(buffer, databaseData); } c.write(buffer); } @Override public int calcPacketSize() { int size = 32;// 4+4+1+23; size += (user == null) ? 1 : user.length() + 1; size += (password == null) ? 1 : BufferUtil.getLength(password); size += (database == null) ? 1 : database.length() + 1; return size; } @Override protected String getPacketInfo() { return "MySQL Authentication Packet"; } } ================================================ FILE: src/main/java/io/mycat/net/mysql/AuthSwitchPacket.java ================================================ package io.mycat.net.mysql; import java.nio.ByteBuffer; import io.mycat.backend.mysql.MySQLMessage; import io.mycat.config.Capabilities; import io.mycat.backend.mysql.BufferUtil; import io.mycat.net.FrontendConnection; /** * * 1 [fe] * string[NUL] plugin name * string[EOF] auth plugin data * * @version 1.0
* @taskId
* @CreateDate Jun 18, 2021
* @since V8.1
* @see io.mycat.net.mysql
*/ public class AuthSwitchPacket extends MySQLPacket{ private static final byte STATUS = (byte) 0XFE; private byte[] authMethodName ; private byte[] authMethodData; public AuthSwitchPacket(byte[] authMethodName, byte[] authMethodData) { super(); this.authMethodName = authMethodName; this.authMethodData = authMethodData; } public AuthSwitchPacket() { } public void read(byte[] data) { MySQLMessage mm = new MySQLMessage(data); packetLength = mm.readUB3(); packetId = mm.read(); authMethodData = mm.readBytes(packetLength); } @Override public ByteBuffer write(ByteBuffer buffer, FrontendConnection c, boolean writeSocketIfFull) { int size = calcPacketSize(); buffer = c.checkWriteBuffer(buffer, c.getPacketHeaderSize() + size, writeSocketIfFull); BufferUtil.writeUB3(buffer, size); buffer.put(packetId); buffer.put(STATUS); BufferUtil.writeWithNull(buffer, authMethodName); BufferUtil.writeWithNull(buffer, authMethodData); return buffer; } public void write(FrontendConnection c) { ByteBuffer buffer = c.allocate(); buffer = this.write(buffer, c, true); c.write(buffer); } @Override public int calcPacketSize() { int size = 3; //status size += authMethodName.length; size += authMethodData.length; return size; } @Override protected String getPacketInfo() { return "MySQL Auth Switch Packet"; } public byte[] getAuthMethodName() { return authMethodName; } public void setAuthMethodName(byte[] authMethodName) { this.authMethodName = authMethodName; } public byte[] getAuthMethodData() { return authMethodData; } public void setAuthMethodData(byte[] authMethodData) { this.authMethodData = authMethodData; } } ================================================ FILE: src/main/java/io/mycat/net/mysql/BinaryPacket.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net.mysql; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; import io.mycat.backend.mysql.BufferUtil; import io.mycat.backend.mysql.StreamUtil; import io.mycat.net.BackendAIOConnection; import io.mycat.net.FrontendConnection; /** * @author mycat */ public class BinaryPacket extends MySQLPacket { public static final byte OK = 1; public static final byte ERROR = 2; public static final byte HEADER = 3; public static final byte FIELD = 4; public static final byte FIELD_EOF = 5; public static final byte ROW = 6; public static final byte PACKET_EOF = 7; public byte[] data; public void read(InputStream in) throws IOException { packetLength = StreamUtil.readUB3(in); packetId = StreamUtil.read(in); byte[] ab = new byte[packetLength]; StreamUtil.read(in, ab, 0, ab.length); data = ab; } @Override public ByteBuffer write(ByteBuffer buffer, FrontendConnection c,boolean writeSocketIfFull) { buffer = c.checkWriteBuffer(buffer, c.getPacketHeaderSize(),writeSocketIfFull); BufferUtil.writeUB3(buffer, calcPacketSize()); buffer.put(packetId); buffer = c.writeToBuffer(data, buffer); return buffer; } @Override public void write(BackendAIOConnection c) { ByteBuffer buffer = c.allocate(); buffer= c.checkWriteBuffer(buffer,c.getPacketHeaderSize()+calcPacketSize(),false); BufferUtil.writeUB3(buffer, calcPacketSize()); buffer.put(packetId); buffer.put(data); c.write(buffer); } @Override public int calcPacketSize() { return data == null ? 0 : data.length; } @Override protected String getPacketInfo() { return "MySQL Binary Packet"; } } ================================================ FILE: src/main/java/io/mycat/net/mysql/BinaryRowDataPacket.java ================================================ package io.mycat.net.mysql; import java.nio.ByteBuffer; import java.text.ParseException; import java.util.ArrayList; import java.util.Date; import java.util.List; import io.mycat.backend.mysql.BufferUtil; import io.mycat.config.Fields; import io.mycat.memory.unsafe.row.UnsafeRow; import io.mycat.net.FrontendConnection; import io.mycat.util.ByteUtil; import io.mycat.util.DateUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * ProtocolBinary::ResultsetRow: * row of a binary resultset (COM_STMT_EXECUTE) * Payload * 1 packet header [00] * string[$len] NULL-bitmap, length: (column_count + 7 + 2) / 8 * string[$len] values * * A Binary Protocol Resultset Row is made up of the NULL bitmap * containing as many bits as we have columns in the resultset + 2 * and the values for columns that are not NULL in the Binary Protocol Value format. * * @see @http://dev.mysql.com/doc/internals/en/binary-protocol-resultset-row.html#packet-ProtocolBinary::ResultsetRow * @see @http://dev.mysql.com/doc/internals/en/binary-protocol-value.html * @author CrazyPig * */ public class BinaryRowDataPacket extends MySQLPacket { private static final Logger LOGGER = LoggerFactory.getLogger(BinaryRowDataPacket.class); public int fieldCount; public List fieldValues; public byte packetHeader = (byte) 0; public byte[] nullBitMap; public List fieldPackets; public BinaryRowDataPacket() {} /** * 从UnsafeRow转换成BinaryRowDataPacket * * 说明: 当开启isOffHeapuseOffHeapForMerge参数时,会使用UnsafeRow封装数据, * 因此需要从这个对象里面将数据封装成BinaryRowDataPacket * * @param fieldPackets * @param unsafeRow */ public void read(List fieldPackets, UnsafeRow unsafeRow) { this.fieldPackets = fieldPackets; this.fieldCount = unsafeRow.numFields(); this.fieldValues = new ArrayList(fieldCount); this.packetId = unsafeRow.packetId; this.nullBitMap = new byte[(fieldCount + 7 + 2) / 8]; for(int i = 0; i < this.fieldCount; i++) { byte[] fv = unsafeRow.getBinary(i); FieldPacket fieldPk = fieldPackets.get(i); if(fv == null) { storeNullBitMap(i); this.fieldValues.add(fv); } else { convert(fv, fieldPk); } } } /** * 从RowDataPacket转换成BinaryRowDataPacket * @param fieldPackets 字段包集合 * @param rowDataPk 文本协议行数据包 */ public void read(List fieldPackets, RowDataPacket rowDataPk) { this.fieldPackets = fieldPackets; this.fieldCount = rowDataPk.fieldCount; this.fieldValues = new ArrayList(fieldCount); this.packetId = rowDataPk.packetId; this.nullBitMap = new byte[(fieldCount + 7 + 2) / 8]; List _fieldValues = rowDataPk.fieldValues; for (int i = 0; i < fieldCount; i++) { byte[] fv = _fieldValues.get(i); FieldPacket fieldPk = fieldPackets.get(i); if (fv == null) { // 字段值为null,根据协议规定存储nullBitMap storeNullBitMap(i); this.fieldValues.add(fv); } else { convert(fv, fieldPk); } } } private void storeNullBitMap(int i) { int bitMapPos = (i + 2) / 8; int bitPos = (i + 2) % 8; this.nullBitMap[bitMapPos] |= (byte) (1 << bitPos); } /** * 从RowDataPacket的fieldValue的数据转化成BinaryRowDataPacket的fieldValue数据 * @param fv * @param fieldPk */ private void convert(byte[] fv, FieldPacket fieldPk) { int fieldType = fieldPk.type; switch (fieldType) { case Fields.FIELD_TYPE_STRING: case Fields.FIELD_TYPE_VARCHAR: case Fields.FIELD_TYPE_VAR_STRING: case Fields.FIELD_TYPE_ENUM: case Fields.FIELD_TYPE_SET: case Fields.FIELD_TYPE_LONG_BLOB: case Fields.FIELD_TYPE_MEDIUM_BLOB: case Fields.FIELD_TYPE_BLOB: case Fields.FIELD_TYPE_TINY_BLOB: case Fields.FIELD_TYPE_GEOMETRY: case Fields.FIELD_TYPE_BIT: case Fields.FIELD_TYPE_DECIMAL: case Fields.FIELD_TYPE_NEW_DECIMAL: // Fields // value (lenenc_str) -- string // Example // 03 66 6f 6f -- string = "foo" this.fieldValues.add(fv); break; case Fields.FIELD_TYPE_LONGLONG: // Fields // value (8) -- integer // Example // 01 00 00 00 00 00 00 00 -- int64 = 1 long longVar = ByteUtil.getLong(fv); this.fieldValues.add(ByteUtil.getBytes(longVar)); break; case Fields.FIELD_TYPE_LONG: case Fields.FIELD_TYPE_INT24: // Fields // value (4) -- integer // Example // 01 00 00 00 -- int32 = 1 int intVar = ByteUtil.getInt(fv); this.fieldValues.add(ByteUtil.getBytes(intVar)); break; case Fields.FIELD_TYPE_SHORT: case Fields.FIELD_TYPE_YEAR: // Fields // value (2) -- integer // Example // 01 00 -- int16 = 1 if (isUnsignedField(fieldPk)) { this.fieldValues.add(ByteUtil.convertUnsignedShort2Binary(fv)); } else { short shortVar = ByteUtil.getShort(fv); this.fieldValues.add(ByteUtil.getBytes(shortVar)); } break; case Fields.FIELD_TYPE_TINY: // Fields // value (1) -- integer // Example // 01 -- int8 = 1 int tinyVar = ByteUtil.getInt(fv); byte[] bytes = new byte[1]; bytes[0] = (byte)tinyVar; this.fieldValues.add(bytes); break; case Fields.FIELD_TYPE_DOUBLE: // Fields // value (string.fix_len) -- (len=8) double // Example // 66 66 66 66 66 66 24 40 -- double = 10.2 double doubleVar = ByteUtil.getDouble(fv); this.fieldValues.add(ByteUtil.getBytes(doubleVar)); break; case Fields.FIELD_TYPE_FLOAT: // Fields // value (string.fix_len) -- (len=4) float // Example // 33 33 23 41 -- float = 10.2 float floatVar = ByteUtil.getFloat(fv); this.fieldValues.add(ByteUtil.getBytes(floatVar)); break; case Fields.FIELD_TYPE_DATE: try { Date dateVar = DateUtil.parseDate(ByteUtil.getDate(fv), DateUtil.DATE_PATTERN_ONLY_DATE); this.fieldValues.add(ByteUtil.getBytes(dateVar, false)); } catch(org.joda.time.IllegalFieldValueException e1) { // 当时间为 0000-00-00 00:00:00 的时候, 默认返回 1970-01-01 08:00:00.0 this.fieldValues.add(ByteUtil.getBytes(new Date(0L), false)); } catch (ParseException e) { LOGGER.error("error",e); } break; case Fields.FIELD_TYPE_DATETIME: case Fields.FIELD_TYPE_TIMESTAMP: String dateStr = ByteUtil.getDate(fv); Date dateTimeVar = null; try { if (dateStr.indexOf(".") > 0) { dateTimeVar = DateUtil.parseDate(dateStr, DateUtil.DATE_PATTERN_FULL); this.fieldValues.add(ByteUtil.getBytes(dateTimeVar, false)); } else { dateTimeVar = DateUtil.parseDate(dateStr, DateUtil.DEFAULT_DATE_PATTERN); this.fieldValues.add(ByteUtil.getBytes(dateTimeVar, false)); } } catch(org.joda.time.IllegalFieldValueException e1) { // 当时间为 0000-00-00 00:00:00 的时候, 默认返回 1970-01-01 08:00:00.0 this.fieldValues.add(ByteUtil.getBytes(new Date(0L), false)); } catch (ParseException e) { LOGGER.error("error",e); } break; case Fields.FIELD_TYPE_TIME: String timeStr = ByteUtil.getTime(fv); Date timeVar = null; try { if (timeStr.indexOf(".") > 0) { timeVar = DateUtil.parseDate(timeStr, DateUtil.TIME_PATTERN_FULL); this.fieldValues.add(ByteUtil.getBytes(timeVar, true)); } else { timeVar = DateUtil.parseDate(timeStr, DateUtil.DEFAULT_TIME_PATTERN); this.fieldValues.add(ByteUtil.getBytes(timeVar, true)); } } catch(org.joda.time.IllegalFieldValueException e1) { // 当时间为 0000-00-00 00:00:00 的时候, 默认返回 1970-01-01 08:00:00.0 this.fieldValues.add(ByteUtil.getBytes(new Date(0L), true)); } catch (ParseException e) { LOGGER.error("error",e); } break; } } private boolean isUnsignedField(FieldPacket field) { return (field.flags & FieldPacket.UNSIGNED_FLAG) > 0; } public void write(FrontendConnection conn) { int size = calcPacketSize(); int packetHeaderSize = conn.getPacketHeaderSize(); int totalSize = size + packetHeaderSize; ByteBuffer bb = null; bb = conn.getProcessor().getBufferPool().allocate(totalSize); BufferUtil.writeUB3(bb, calcPacketSize()); bb.put(packetId); bb.put(packetHeader); // packet header [00] bb.put(nullBitMap); // NULL-Bitmap for(int i = 0; i < fieldCount; i++) { // values byte[] fv = fieldValues.get(i); if(fv != null) { FieldPacket fieldPk = this.fieldPackets.get(i); int fieldType = fieldPk.type; switch(fieldType) { case Fields.FIELD_TYPE_STRING: case Fields.FIELD_TYPE_VARCHAR: case Fields.FIELD_TYPE_VAR_STRING: case Fields.FIELD_TYPE_ENUM: case Fields.FIELD_TYPE_SET: case Fields.FIELD_TYPE_LONG_BLOB: case Fields.FIELD_TYPE_MEDIUM_BLOB: case Fields.FIELD_TYPE_BLOB: case Fields.FIELD_TYPE_TINY_BLOB: case Fields.FIELD_TYPE_GEOMETRY: case Fields.FIELD_TYPE_BIT: case Fields.FIELD_TYPE_DECIMAL: case Fields.FIELD_TYPE_NEW_DECIMAL: // 长度编码的字符串需要一个字节来存储长度(0表示空字符串) BufferUtil.writeLength(bb, fv.length); break; default: break; } if(fv.length > 0) { bb.put(fv); } } } conn.write(bb); } @Override public ByteBuffer write(ByteBuffer bb, FrontendConnection c, boolean writeSocketIfFull) { int size = calcPacketSize(); int packetHeaderSize = c.getPacketHeaderSize(); int totalSize = size + packetHeaderSize; bb = c.checkWriteBuffer(bb, totalSize, writeSocketIfFull); BufferUtil.writeUB3(bb, size); bb.put(packetId); bb.put(packetHeader); // packet header [00] bb.put(nullBitMap); // NULL-Bitmap for(int i = 0; i < fieldCount; i++) { // values byte[] fv = fieldValues.get(i); if(fv != null) { FieldPacket fieldPk = this.fieldPackets.get(i); int fieldType = fieldPk.type; switch(fieldType) { case Fields.FIELD_TYPE_STRING: case Fields.FIELD_TYPE_VARCHAR: case Fields.FIELD_TYPE_VAR_STRING: case Fields.FIELD_TYPE_ENUM: case Fields.FIELD_TYPE_SET: case Fields.FIELD_TYPE_LONG_BLOB: case Fields.FIELD_TYPE_MEDIUM_BLOB: case Fields.FIELD_TYPE_BLOB: case Fields.FIELD_TYPE_TINY_BLOB: case Fields.FIELD_TYPE_GEOMETRY: case Fields.FIELD_TYPE_BIT: case Fields.FIELD_TYPE_DECIMAL: case Fields.FIELD_TYPE_NEW_DECIMAL: // 长度编码的字符串需要一个字节来存储长度(0表示空字符串) BufferUtil.writeLength(bb, fv.length); break; default: break; } if(fv.length > 0) { bb.put(fv); } } } return bb; } @Override public int calcPacketSize() { int size = 0; size = size + 1 + nullBitMap.length; for(int i = 0, n = fieldValues.size(); i < n; i++) { byte[] value = fieldValues.get(i); if(value != null) { FieldPacket fieldPk = this.fieldPackets.get(i); int fieldType = fieldPk.type; switch(fieldType) { case Fields.FIELD_TYPE_STRING: case Fields.FIELD_TYPE_VARCHAR: case Fields.FIELD_TYPE_VAR_STRING: case Fields.FIELD_TYPE_ENUM: case Fields.FIELD_TYPE_SET: case Fields.FIELD_TYPE_LONG_BLOB: case Fields.FIELD_TYPE_MEDIUM_BLOB: case Fields.FIELD_TYPE_BLOB: case Fields.FIELD_TYPE_TINY_BLOB: case Fields.FIELD_TYPE_GEOMETRY: case Fields.FIELD_TYPE_BIT: case Fields.FIELD_TYPE_DECIMAL: case Fields.FIELD_TYPE_NEW_DECIMAL: /* * 长度编码的字符串需要计算存储长度, 根据mysql协议文档描述 * To convert a length-encoded integer into its numeric value, check the first byte: * If it is < 0xfb, treat it as a 1-byte integer. * If it is 0xfc, it is followed by a 2-byte integer. * If it is 0xfd, it is followed by a 3-byte integer. * If it is 0xfe, it is followed by a 8-byte integer. * */ if(value.length != 0) { /* * 长度编码的字符串需要计算存储长度,不能简单默认只有1个字节是表示长度,当数据足够长,占用的就不止1个字节 */ // size = size + 1 + value.length; size = size + BufferUtil.getLength(value); } else { size = size + 1; // 处理空字符串,只计算长度1个字节 } break; default: size = size + value.length; break; } } } return size; } @Override protected String getPacketInfo() { return "MySQL Binary RowData Packet"; } } ================================================ FILE: src/main/java/io/mycat/net/mysql/CommandPacket.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net.mysql; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; import io.mycat.backend.mysql.BufferUtil; import io.mycat.backend.mysql.MySQLMessage; import io.mycat.backend.mysql.StreamUtil; import io.mycat.net.BackendAIOConnection; /** * From client to server whenever the client wants the server to do something. * * 
 * Bytes         Name
 * -----         ----
 * 1             command
 * n             arg
 * 
 * command:      The most common value is 03 COM_QUERY, because
 *               INSERT UPDATE DELETE SELECT etc. have this code.
 *               The possible values at time of writing (taken
 *               from /include/mysql_com.h for enum_server_command) are:
 * 
 *               #      Name                Associated client function
 *               -      ----                --------------------------
 *               0x00   COM_SLEEP           (none, this is an internal thread state)
 *               0x01   COM_QUIT            mysql_close
 *               0x02   COM_INIT_DB         mysql_select_db 
 *               0x03   COM_QUERY           mysql_real_query
 *               0x04   COM_FIELD_LIST      mysql_list_fields
 *               0x05   COM_CREATE_DB       mysql_create_db (deprecated)
 *               0x06   COM_DROP_DB         mysql_drop_db (deprecated)
 *               0x07   COM_REFRESH         mysql_refresh
 *               0x08   COM_SHUTDOWN        mysql_shutdown
 *               0x09   COM_STATISTICS      mysql_stat
 *               0x0a   COM_PROCESS_INFO    mysql_list_processes
 *               0x0b   COM_CONNECT         (none, this is an internal thread state)
 *               0x0c   COM_PROCESS_KILL    mysql_kill
 *               0x0d   COM_DEBUG           mysql_dump_debug_info
 *               0x0e   COM_PING            mysql_ping
 *               0x0f   COM_TIME            (none, this is an internal thread state)
 *               0x10   COM_DELAYED_INSERT  (none, this is an internal thread state)
 *               0x11   COM_CHANGE_USER     mysql_change_user
 *               0x12   COM_BINLOG_DUMP     sent by the slave IO thread to request a binlog
 *               0x13   COM_TABLE_DUMP      LOAD TABLE ... FROM MASTER (deprecated)
 *               0x14   COM_CONNECT_OUT     (none, this is an internal thread state)
 *               0x15   COM_REGISTER_SLAVE  sent by the slave to register with the master (optional)
 *               0x16   COM_STMT_PREPARE    mysql_stmt_prepare
 *               0x17   COM_STMT_EXECUTE    mysql_stmt_execute
 *               0x18   COM_STMT_SEND_LONG_DATA mysql_stmt_send_long_data
 *               0x19   COM_STMT_CLOSE      mysql_stmt_close
 *               0x1a   COM_STMT_RESET      mysql_stmt_reset
 *               0x1b   COM_SET_OPTION      mysql_set_server_option
 *               0x1c   COM_STMT_FETCH      mysql_stmt_fetch
 * 
 * arg:          The text of the command is just the way the user typed it, there is no processing
 *               by the client (except removal of the final ';').
 *               This field is not a null-terminated string; however,
 *               the size can be calculated from the packet size,
 *               and the MySQL client appends '\0' when receiving.
 *               
 * @see http://forge.mysql.com/wiki/MySQL_Internals_ClientServer_Protocol#Command_Packet_.28Overview.29
 *
* * @author mycat */ public class CommandPacket extends MySQLPacket { public byte command; public byte[] arg; public void read(byte[] data) { MySQLMessage mm = new MySQLMessage(data); packetLength = mm.readUB3(); packetId = mm.read(); command = mm.read(); arg = mm.readBytes(); } public void write(OutputStream out) throws IOException { StreamUtil.writeUB3(out, calcPacketSize()); StreamUtil.write(out, packetId); StreamUtil.write(out, command); out.write(arg); } @Override public void write(BackendAIOConnection c) { ByteBuffer buffer = c.allocate(); try { BufferUtil.writeUB3(buffer, calcPacketSize()); buffer.put(packetId); buffer.put(command); buffer = c.writeToBuffer(arg, buffer); c.write(buffer); } catch(java.nio.BufferOverflowException e1) { //fixed issues #98 #1072 buffer = c.checkWriteBuffer(buffer, c.getPacketHeaderSize() + calcPacketSize(), false); BufferUtil.writeUB3(buffer, calcPacketSize()); buffer.put(packetId); buffer.put(command); buffer = c.writeToBuffer(arg, buffer); c.write(buffer); } } @Override public int calcPacketSize() { return 1 + arg.length; } @Override protected String getPacketInfo() { return "MySQL Command Packet"; } } ================================================ FILE: src/main/java/io/mycat/net/mysql/EOFPacket.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net.mysql; import java.nio.ByteBuffer; import io.mycat.backend.mysql.BufferUtil; import io.mycat.backend.mysql.MySQLMessage; import io.mycat.buffer.BufferArray; import io.mycat.net.FrontendConnection; /** * From Server To Client, at the end of a series of Field Packets, and at the * end of a series of Data Packets.With prepared statements, EOF Packet can also * end parameter information, which we'll describe later. * * 
 * Bytes                 Name
 * -----                 ----
 * 1                     field_count, always = 0xfe
 * 2                     warning_count
 * 2                     Status Flags
 * 
 * @see http://forge.mysql.com/wiki/MySQL_Internals_ClientServer_Protocol#EOF_Packet
 *
* * @author mycat */ public class EOFPacket extends MySQLPacket { public static final byte FIELD_COUNT = (byte) 0xfe; public byte fieldCount = FIELD_COUNT; public int warningCount; public int status = 2; public void read(byte[] data) { MySQLMessage mm = new MySQLMessage(data); packetLength = mm.readUB3(); packetId = mm.read(); fieldCount = mm.read(); warningCount = mm.readUB2(); status = mm.readUB2(); } @Override public ByteBuffer write(ByteBuffer buffer, FrontendConnection c,boolean writeSocketIfFull) { int size = calcPacketSize(); buffer = c.checkWriteBuffer(buffer, c.getPacketHeaderSize() + size,writeSocketIfFull); BufferUtil.writeUB3(buffer, size); buffer.put(packetId); buffer.put(fieldCount); BufferUtil.writeUB2(buffer, warningCount); BufferUtil.writeUB2(buffer, status); return buffer; } @Override public int calcPacketSize() { return 5;// 1+2+2; } @Override protected String getPacketInfo() { return "MySQL EOF Packet"; } public void write(BufferArray bufferArray) { int size = calcPacketSize(); ByteBuffer buffer = bufferArray.checkWriteBuffer(packetHeaderSize + size); BufferUtil.writeUB3(buffer, size); buffer.put(packetId); buffer.put(fieldCount); BufferUtil.writeUB2(buffer, warningCount); BufferUtil.writeUB2(buffer, status); } } ================================================ FILE: src/main/java/io/mycat/net/mysql/EmptyPacket.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net.mysql; /** * @author mycat暂时只发现在load data infile时用到 */ public class EmptyPacket extends MySQLPacket { public static final byte[] EMPTY = new byte[] { 0, 0, 0,3 }; @Override public int calcPacketSize() { return 0; } @Override protected String getPacketInfo() { return "MySQL Empty Packet"; } } ================================================ FILE: src/main/java/io/mycat/net/mysql/ErrorPacket.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net.mysql; import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import io.mycat.backend.mysql.BufferUtil; import io.mycat.backend.mysql.MySQLMessage; import io.mycat.net.FrontendConnection; /** * From server to client in response to command, if error. * * 
 * Bytes                       Name
 * -----                       ----
 * 1                           field_count, always = 0xff
 * 2                           errno
 * 1                           (sqlstate marker), always '#'
 * 5                           sqlstate (5 characters)
 * n                           message
 * 
 * @see http://forge.mysql.com/wiki/MySQL_Internals_ClientServer_Protocol#Error_Packet
 *
* * @author mycat */ public class ErrorPacket extends MySQLPacket { public static final byte FIELD_COUNT = (byte) 0xff; private static final byte SQLSTATE_MARKER = (byte) '#'; private static final byte[] DEFAULT_SQLSTATE = "HY000".getBytes(); public byte fieldCount = FIELD_COUNT; public int errno; public byte mark = SQLSTATE_MARKER; public byte[] sqlState = DEFAULT_SQLSTATE; public byte[] message; public void read(BinaryPacket bin) { packetLength = bin.packetLength; packetId = bin.packetId; MySQLMessage mm = new MySQLMessage(bin.data); fieldCount = mm.read(); errno = mm.readUB2(); if (mm.hasRemaining() && (mm.read(mm.position()) == SQLSTATE_MARKER)) { mm.read(); sqlState = mm.readBytes(5); } message = mm.readBytes(); } public void read(byte[] data) { MySQLMessage mm = new MySQLMessage(data); packetLength = mm.readUB3(); packetId = mm.read(); fieldCount = mm.read(); errno = mm.readUB2(); if (mm.hasRemaining() && (mm.read(mm.position()) == SQLSTATE_MARKER)) { mm.read(); sqlState = mm.readBytes(5); } message = mm.readBytes(); } public byte[] writeToBytes(FrontendConnection c) { ByteBuffer buffer = c.allocate(); buffer = write(buffer, c, false); buffer.flip(); byte[] data = new byte[buffer.limit()]; buffer.get(data); c.recycle(buffer); return data; } public byte[] writeToBytes() { ByteBuffer buffer = ByteBuffer.allocate(calcPacketSize()+4); int size = calcPacketSize(); BufferUtil.writeUB3(buffer, size); buffer.put(packetId); buffer.put(fieldCount); BufferUtil.writeUB2(buffer, errno); buffer.put(mark); buffer.put(sqlState); if (message != null) { buffer.put(message); } buffer.flip(); byte[] data = new byte[buffer.limit()]; buffer.get(data); return data; } @Override public ByteBuffer write(ByteBuffer buffer, FrontendConnection c, boolean writeSocketIfFull) { int size = calcPacketSize(); buffer = c.checkWriteBuffer(buffer, c.getPacketHeaderSize() + size, writeSocketIfFull); BufferUtil.writeUB3(buffer, size); buffer.put(packetId); buffer.put(fieldCount); BufferUtil.writeUB2(buffer, errno); buffer.put(mark); buffer.put(sqlState); if (message != null) { buffer = c.writeToBuffer(message, buffer); } return buffer; } public void write(FrontendConnection c) { ByteBuffer buffer = c.allocate(); buffer = this.write(buffer, c, true); c.write(buffer); } @Override public int calcPacketSize() { int size = 9;// 1 + 2 + 1 + 5 if (message != null) { size += message.length; } return size; } @Override protected String getPacketInfo() { return "MySQL Error Packet"; } } ================================================ FILE: src/main/java/io/mycat/net/mysql/ExecutePacket.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net.mysql; import java.io.UnsupportedEncodingException; import io.mycat.backend.mysql.BindValue; import io.mycat.backend.mysql.BindValueUtil; import io.mycat.backend.mysql.MySQLMessage; import io.mycat.backend.mysql.PreparedStatement; /** * 
 *  Bytes                      Name
 *  -----                      ----
 *  1                          code
 *  4                          statement_id
 *  1                          flags
 *  4                          iteration_count 
 *  (param_count+7)/8          null_bit_map
 *  1                          new_parameter_bound_flag (if new_params_bound == 1:)
 *  n*2                        type of parameters
 *  n                          values for the parameters   
 *  --------------------------------------------------------------------------------
 *  code:                      always COM_EXECUTE
 *  
 *  statement_id:              statement identifier
 *  
 *  flags:                     reserved for future use. In MySQL 4.0, always 0.
 *                             In MySQL 5.0: 
 *                               0: CURSOR_TYPE_NO_CURSOR
 *                               1: CURSOR_TYPE_READ_ONLY
 *                               2: CURSOR_TYPE_FOR_UPDATE
 *                               4: CURSOR_TYPE_SCROLLABLE
 *  
 *  iteration_count:           reserved for future use. Currently always 1.
 *  
 *  null_bit_map:              A bitmap indicating parameters that are NULL.
 *                             Bits are counted from LSB, using as many bytes
 *                             as necessary ((param_count+7)/8)
 *                             i.e. if the first parameter (parameter 0) is NULL, then
 *                             the least significant bit in the first byte will be 1.
 *  
 *  new_parameter_bound_flag:  Contains 1 if this is the first time
 *                             that "execute" has been called, or if
 *                             the parameters have been rebound.
 *  
 *  type:                      Occurs once for each parameter; 
 *                             The highest significant bit of this 16-bit value
 *                             encodes the unsigned property. The other 15 bits
 *                             are reserved for the type (only 8 currently used).
 *                             This block is sent when parameters have been rebound
 *                             or when a prepared statement is executed for the 
 *                             first time.
 * 
 *  values:                    for all non-NULL values, each parameters appends its value
 *                             as described in Row Data Packet: Binary (column values)
 * @see https://dev.mysql.com/doc/internals/en/com-stmt-execute.html
 *
* * @author mycat, CrazyPig */ public class ExecutePacket extends MySQLPacket { public byte code; public long statementId; public byte flags; public long iterationCount; public byte[] nullBitMap; public byte newParameterBoundFlag; public BindValue[] values; protected PreparedStatement pstmt; public ExecutePacket(PreparedStatement pstmt) { this.pstmt = pstmt; this.values = new BindValue[pstmt.getParametersNumber()]; } public void read(byte[] data, String charset) throws UnsupportedEncodingException { MySQLMessage mm = new MySQLMessage(data); packetLength = mm.readUB3(); packetId = mm.read(); code = mm.read(); statementId = mm.readUB4(); flags = mm.read(); iterationCount = mm.readUB4(); // 读取NULL指示器数据 int parameterCount = values.length; if(parameterCount > 0) { nullBitMap = new byte[(parameterCount + 7) / 8]; for (int i = 0; i < nullBitMap.length; i++) { nullBitMap[i] = mm.read(); } // 当newParameterBoundFlag==1时，更新参数类型。 newParameterBoundFlag = mm.read(); } if (newParameterBoundFlag == (byte) 1) { for (int i = 0; i < parameterCount; i++) { pstmt.getParametersType()[i] = mm.readUB2(); } } // 设置参数类型和读取参数值 byte[] nullBitMap = this.nullBitMap; for (int i = 0; i < parameterCount; i++) { BindValue bv = new BindValue(); bv.type = pstmt.getParametersType()[i]; if ((nullBitMap[i / 8] & (1 << (i & 7))) != 0) { bv.isNull = true; } else { if (!pstmt.hasLongData(i)) { BindValueUtil.read(mm, bv, charset); } else { bv.value = pstmt.getLongData(i).toByteArray(); } } values[i] = bv; } } @Override public int calcPacketSize() { return 0; } @Override protected String getPacketInfo() { return "MySQL Execute Packet"; } } ================================================ FILE: src/main/java/io/mycat/net/mysql/FieldPacket.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net.mysql; import java.nio.ByteBuffer; import io.mycat.backend.mysql.BufferUtil; import io.mycat.backend.mysql.MySQLMessage; import io.mycat.buffer.BufferArray; import io.mycat.net.FrontendConnection; /** * From Server To Client, part of Result Set Packets. One for each column in the * result set. Thus, if the value of field_columns in the Result Set Header * Packet is 3, then the Field Packet occurs 3 times. * * 
 * Bytes                      Name
 * -----                      ----
 * n (Length Coded String)    catalog
 * n (Length Coded String)    db
 * n (Length Coded String)    table
 * n (Length Coded String)    org_table
 * n (Length Coded String)    name
 * n (Length Coded String)    org_name
 * 1                          (filler)
 * 2                          charsetNumber
 * 4                          length
 * 1                          type
 * 2                          flags
 * 1                          decimals
 * 2                          (filler), always 0x00
 * n (Length Coded Binary)    default
 * 
 * @see http://forge.mysql.com/wiki/MySQL_Internals_ClientServer_Protocol#Field_Packet
 *
* * @author mycat */ public class FieldPacket extends MySQLPacket { public static final int UNSIGNED_FLAG = 0x0020; private static final byte[] DEFAULT_CATALOG = "def".getBytes(); private static final byte[] FILLER = new byte[2]; public byte[] catalog = DEFAULT_CATALOG; public byte[] db; public byte[] table; public byte[] orgTable; public byte[] name; public byte[] orgName; public int charsetIndex; public long length; public int type; public int flags; public byte decimals; public byte[] definition; /** * 把字节数组转变成FieldPacket */ public void read(byte[] data) { MySQLMessage mm = new MySQLMessage(data); this.packetLength = mm.readUB3(); this.packetId = mm.read(); readBody(mm); } /** * 把BinaryPacket转变成FieldPacket */ public void read(BinaryPacket bin) { this.packetLength = bin.packetLength; this.packetId = bin.packetId; readBody(new MySQLMessage(bin.data)); } @Override public ByteBuffer write(ByteBuffer buffer, FrontendConnection c, boolean writeSocketIfFull) { int size = calcPacketSize(); buffer = c.checkWriteBuffer(buffer, c.getPacketHeaderSize() + size, writeSocketIfFull); BufferUtil.writeUB3(buffer, size); buffer.put(packetId); writeBody(buffer); return buffer; } @Override public int calcPacketSize() { int size = (catalog == null ? 1 : BufferUtil.getLength(catalog)); size += (db == null ? 1 : BufferUtil.getLength(db)); size += (table == null ? 1 : BufferUtil.getLength(table)); size += (orgTable == null ? 1 : BufferUtil.getLength(orgTable)); size += (name == null ? 1 : BufferUtil.getLength(name)); size += (orgName == null ? 1 : BufferUtil.getLength(orgName)); size += 13;// 1+2+4+1+2+1+2 if (definition != null) { size += BufferUtil.getLength(definition); } return size; } @Override protected String getPacketInfo() { return "MySQL Field Packet"; } private void readBody(MySQLMessage mm) { this.catalog = mm.readBytesWithLength(); this.db = mm.readBytesWithLength(); this.table = mm.readBytesWithLength(); this.orgTable = mm.readBytesWithLength(); this.name = mm.readBytesWithLength(); this.orgName = mm.readBytesWithLength(); mm.move(1); this.charsetIndex = mm.readUB2(); this.length = mm.readUB4(); this.type = mm.read() & 0xff; this.flags = mm.readUB2(); this.decimals = mm.read(); mm.move(FILLER.length); if (mm.hasRemaining()) { this.definition = mm.readBytesWithLength(); } } private void writeBody(ByteBuffer buffer) { byte nullVal = 0; BufferUtil.writeWithLength(buffer, catalog, nullVal); BufferUtil.writeWithLength(buffer, db, nullVal); BufferUtil.writeWithLength(buffer, table, nullVal); BufferUtil.writeWithLength(buffer, orgTable, nullVal); BufferUtil.writeWithLength(buffer, name, nullVal); BufferUtil.writeWithLength(buffer, orgName, nullVal); buffer.put((byte) 0x0C); BufferUtil.writeUB2(buffer, charsetIndex); BufferUtil.writeUB4(buffer, length); buffer.put((byte) (type & 0xff)); BufferUtil.writeUB2(buffer, flags); buffer.put(decimals); buffer.put((byte)0x00); buffer.put((byte)0x00); //buffer.position(buffer.position() + FILLER.length); if (definition != null) { BufferUtil.writeWithLength(buffer, definition); } } public void write(BufferArray bufferArray) { int size = calcPacketSize(); ByteBuffer buffer = bufferArray.checkWriteBuffer(packetHeaderSize + size); BufferUtil.writeUB3(buffer, size); buffer.put(packetId); writeBody(buffer); } } ================================================ FILE: src/main/java/io/mycat/net/mysql/HandshakePacket.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net.mysql; import java.nio.ByteBuffer; import io.mycat.backend.mysql.BufferUtil; import io.mycat.backend.mysql.MySQLMessage; import io.mycat.net.FrontendConnection; /** * From server to client during initial handshake. * * 
 * Bytes                        Name
 * -----                        ----
 * 1                            protocol_version
 * n (Null-Terminated String)   server_version
 * 4                            thread_id
 * 8                            scramble_buff
 * 1                            (filler) always 0x00
 * 2                            server_capabilities
 * 1                            server_language
 * 2                            server_status
 * 13                           (filler) always 0x00 ...
 * 13                           rest of scramble_buff (4.1)
 * 
 * @see http://forge.mysql.com/wiki/MySQL_Internals_ClientServer_Protocol#Handshake_Initialization_Packet
 *
* * @author mycat */ public class HandshakePacket extends MySQLPacket { private static final byte[] FILLER_13 = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; public byte protocolVersion; public byte[] serverVersion; public long threadId; public byte[] seed; public int serverCapabilities; public byte serverCharsetIndex; public int serverStatus; public byte[] restOfScrambleBuff; public void read(BinaryPacket bin) { packetLength = bin.packetLength; packetId = bin.packetId; MySQLMessage mm = new MySQLMessage(bin.data); protocolVersion = mm.read(); serverVersion = mm.readBytesWithNull(); threadId = mm.readUB4(); seed = mm.readBytesWithNull(); serverCapabilities = mm.readUB2(); serverCharsetIndex = mm.read(); serverStatus = mm.readUB2(); mm.move(13); restOfScrambleBuff = mm.readBytesWithNull(); } public void read(byte[] data) { MySQLMessage mm = new MySQLMessage(data); packetLength = mm.readUB3(); packetId = mm.read(); protocolVersion = mm.read(); serverVersion = mm.readBytesWithNull(); threadId = mm.readUB4(); seed = mm.readBytesWithNull(); serverCapabilities = mm.readUB2(); serverCharsetIndex = mm.read(); serverStatus = mm.readUB2(); mm.move(13); restOfScrambleBuff = mm.readBytesWithNull(); } public void write(FrontendConnection c) { ByteBuffer buffer = c.allocate(); BufferUtil.writeUB3(buffer, calcPacketSize()); buffer.put(packetId); buffer.put(protocolVersion); BufferUtil.writeWithNull(buffer, serverVersion); BufferUtil.writeUB4(buffer, threadId); BufferUtil.writeWithNull(buffer, seed); BufferUtil.writeUB2(buffer, serverCapabilities); buffer.put(serverCharsetIndex); BufferUtil.writeUB2(buffer, serverStatus); buffer.put(FILLER_13); // buffer.position(buffer.position() + 13); BufferUtil.writeWithNull(buffer, restOfScrambleBuff); c.write(buffer); } @Override public int calcPacketSize() { int size = 1; size += serverVersion.length;// n size += 5;// 1+4 size += seed.length;// 8 size += 19;// 1+2+1+2+13 size += restOfScrambleBuff.length;// 12 size += 1;// 1 return size; } @Override protected String getPacketInfo() { return "MySQL Handshake Packet"; } } ================================================ FILE: src/main/java/io/mycat/net/mysql/HandshakeV10Packet.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net.mysql; import java.nio.ByteBuffer; import io.mycat.config.Capabilities; import io.mycat.backend.mysql.BufferUtil; import io.mycat.net.FrontendConnection; /** * From mycat server to client during initial handshake. * * 
 * Bytes                        Name
 * -----                        ----
 * 1                            protocol_version (always 0x0a)
 * n (string[NULL])             server_version
 * 4                            thread_id
 * 8 (string[8])                auth-plugin-data-part-1
 * 1                            (filler) always 0x00
 * 2                            capability flags (lower 2 bytes)
 *   if more data in the packet:
 * 1                            character set
 * 2                            status flags
 * 2                            capability flags (upper 2 bytes)
 *   if capabilities & CLIENT_PLUGIN_AUTH {
 * 1                            length of auth-plugin-data
 *   } else {
 * 1                            0x00
 *   }
 * 10 (string[10])              reserved (all 0x00)
 *   if capabilities & CLIENT_SECURE_CONNECTION {
 * string[$len]   auth-plugin-data-part-2 ($len=MAX(13, length of auth-plugin-data - 8))
 *   }
 *   if capabilities & CLIENT_PLUGIN_AUTH {
 * string[NUL]    auth-plugin name
 * }
 * 
 * @see http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#Protocol::HandshakeV10
 *
* * @author CrazyPig * @since 2016-11-13 * */ public class HandshakeV10Packet extends MySQLPacket { private static final byte[] FILLER_10 = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; private static final byte[] DEFAULT_AUTH_PLUGIN_NAME = "mysql_native_password".getBytes(); public byte protocolVersion; public byte[] serverVersion; public long threadId; public byte[] seed; // auth-plugin-data-part-1 public int serverCapabilities; public byte serverCharsetIndex; public int serverStatus; public byte[] restOfScrambleBuff; // auth-plugin-data-part-2 public byte[] authPluginName = DEFAULT_AUTH_PLUGIN_NAME; public void write(FrontendConnection c) { ByteBuffer buffer = c.allocate(); BufferUtil.writeUB3(buffer, calcPacketSize()); buffer.put(packetId); buffer.put(protocolVersion); BufferUtil.writeWithNull(buffer, serverVersion); BufferUtil.writeUB4(buffer, threadId); buffer.put(seed); buffer.put((byte)0); // [00] filler BufferUtil.writeUB2(buffer, serverCapabilities); // capability flags (lower 2 bytes) buffer.put(serverCharsetIndex); BufferUtil.writeUB2(buffer, serverStatus); BufferUtil.writeUB2(buffer, (serverCapabilities >> 16)); // capability flags (upper 2 bytes) if((serverCapabilities & Capabilities.CLIENT_PLUGIN_AUTH) != 0) { if(restOfScrambleBuff.length <= 13) { buffer.put((byte) (seed.length + 13)); } else { buffer.put((byte) (seed.length + restOfScrambleBuff.length)); } } else { buffer.put((byte) 0); } buffer.put(FILLER_10); if((serverCapabilities & Capabilities.CLIENT_SECURE_CONNECTION) != 0) { buffer.put(restOfScrambleBuff); // restOfScrambleBuff.length always to be 12 if(restOfScrambleBuff.length < 13) { for(int i = 13 - restOfScrambleBuff.length; i > 0; i--) { buffer.put((byte)0); } } } if((serverCapabilities & Capabilities.CLIENT_PLUGIN_AUTH) != 0) { BufferUtil.writeWithNull(buffer, authPluginName); } c.write(buffer); } @Override public int calcPacketSize() { int size = 1; // protocol version size += (serverVersion.length + 1); // server version size += 4; // connection id size += seed.length; size += 1; // [00] filler size += 2; // capability flags (lower 2 bytes) size += 1; // character set size += 2; // status flags size += 2; // capability flags (upper 2 bytes) size += 1; size += 10; // reserved (all [00]) if((serverCapabilities & Capabilities.CLIENT_SECURE_CONNECTION) != 0) { // restOfScrambleBuff.length always to be 12 if(restOfScrambleBuff.length <= 13) { size += 13; } else { size += restOfScrambleBuff.length; } } if((serverCapabilities & Capabilities.CLIENT_PLUGIN_AUTH) != 0) { size += (authPluginName.length + 1); // auth-plugin name } return size; } @Override protected String getPacketInfo() { return "MySQL HandshakeV10 Packet"; } } ================================================ FILE: src/main/java/io/mycat/net/mysql/HeartbeatPacket.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.net.mysql; import java.nio.ByteBuffer; import io.mycat.backend.mysql.BufferUtil; import io.mycat.backend.mysql.MySQLMessage; import io.mycat.net.BackendAIOConnection; /** * From client to server when the client do heartbeat between mycat cluster. * * 
 * Bytes         Name
 * -----         ----
 * 1             command
 * n             id
 * 
 * @author mycat
 */
public class HeartbeatPacket extends MySQLPacket {

    public byte command;
    public long id;

    public void read(byte[] data) {
        MySQLMessage mm = new MySQLMessage(data);
        packetLength = mm.readUB3();
        packetId = mm.read();
        command = mm.read();
        id = mm.readLength();
    }

    @Override
    public void write(BackendAIOConnection c) {
        ByteBuffer buffer = c.allocate();
        BufferUtil.writeUB3(buffer, calcPacketSize());
        buffer.put(packetId);
        buffer.put(command);
        BufferUtil.writeLength(buffer, id);
        c.write(buffer);
    }

    @Override
    public int calcPacketSize() {
        return 1 + BufferUtil.getLength(id);
    }

    @Override
    protected String getPacketInfo() {
        return "Mycat Heartbeat Packet";
    }

}

================================================
FILE: src/main/java/io/mycat/net/mysql/LongDataPacket.java
================================================
package io.mycat.net.mysql;

import io.mycat.backend.mysql.MySQLMessage;

/**
 * 
 * 
 * 
 * COM_STMT_SEND_LONG_DATA sends the data for a column. Repeating to send it, appends the data to the parameter.
 * No response is sent back to the client.

 * COM_STMT_SEND_LONG_DATA:
 * COM_STMT_SEND_LONG_DATA
 * direction: client -> server
 * response: none

 * payload:
 *   1              [18] COM_STMT_SEND_LONG_DATA
 *   4              statement-id
 *   2              param-id
 *   n              data
 * 
 * 

 * 
 * @see https://dev.mysql.com/doc/internals/en/com-stmt-send-long-data.html
 * 
 * @author CrazyPig
 * @since 2016-09-08
 *
 */
public class LongDataPacket extends MySQLPacket {

	private static final byte PACKET_FALG = (byte) 24;
	private long pstmtId;
	private long paramId;
	private byte[] longData = new byte[0];
	
	public void read(byte[] data) {
		MySQLMessage mm = new MySQLMessage(data);
		packetLength = mm.readUB3();
		packetId = mm.read();
		byte code = mm.read();
		assert code == PACKET_FALG;
		pstmtId = mm.readUB4();
		paramId = mm.readUB2();
		this.longData = mm.readBytes(packetLength - (1 + 4 + 2));
	}

	@Override
	public int calcPacketSize() {
		return 1 + 4 + 2 + this.longData.length;
	}

	@Override
	protected String getPacketInfo() {
		return "MySQL Long Data Packet";
	}

	public long getPstmtId() {
		return pstmtId;
	}

	public long getParamId() {
		return paramId;
	}

	public byte[] getLongData() {
		return longData;
	}

	
}


================================================
FILE: src/main/java/io/mycat/net/mysql/MySQLPacket.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.mysql;

import java.nio.ByteBuffer;

import io.mycat.net.BackendAIOConnection;
import io.mycat.net.FrontendConnection;

/**
 * @author mycat
 */
public abstract class MySQLPacket {
    /**
     * none, this is an internal thread state
     */
    public static final byte COM_SLEEP = 0;

    /**
     * mysql_close
     */
    public static final byte COM_QUIT = 1;

    /**
     * mysql_select_db
     */
    public static final byte COM_INIT_DB = 2;

    /**
     * mysql_real_query
     */
    public static final byte COM_QUERY = 3;

    /**
     * mysql_list_fields
     */
    public static final byte COM_FIELD_LIST = 4;

    /**
     * mysql_create_db (deprecated)
     */
    public static final byte COM_CREATE_DB = 5;

    /**
     * mysql_drop_db (deprecated)
     */
    public static final byte COM_DROP_DB = 6;

    /**
     * mysql_refresh
     */
    public static final byte COM_REFRESH = 7;

    /**
     * mysql_shutdown
     */
    public static final byte COM_SHUTDOWN = 8;

    /**
     * mysql_stat
     */
    public static final byte COM_STATISTICS = 9;

    /**
     * mysql_list_processes
     */
    public static final byte COM_PROCESS_INFO = 10;

    /**
     * none, this is an internal thread state
     */
    public static final byte COM_CONNECT = 11;

    /**
     * mysql_kill
     */
    public static final byte COM_PROCESS_KILL = 12;

    /**
     * mysql_dump_debug_info
     */
    public static final byte COM_DEBUG = 13;

    /**
     * mysql_ping
     */
    public static final byte COM_PING = 14;

    /**
     * none, this is an internal thread state
     */
    public static final byte COM_TIME = 15;

    /**
     * none, this is an internal thread state
     */
    public static final byte COM_DELAYED_INSERT = 16;

    /**
     * mysql_change_user
     */
    public static final byte COM_CHANGE_USER = 17;

    /**
     * used by slave server mysqlbinlog
     */
    public static final byte COM_BINLOG_DUMP = 18;

    /**
     * used by slave server to get master table
     */
    public static final byte COM_TABLE_DUMP = 19;

    /**
     * used by slave to log connection to master
     */
    public static final byte COM_CONNECT_OUT = 20;

    /**
     * used by slave to register to master
     */
    public static final byte COM_REGISTER_SLAVE = 21;

    /**
     * mysql_stmt_prepare
     */
    public static final byte COM_STMT_PREPARE = 22;

    /**
     * mysql_stmt_execute
     */
    public static final byte COM_STMT_EXECUTE = 23;

    /**
     * mysql_stmt_send_long_data
     */
    public static final byte COM_STMT_SEND_LONG_DATA = 24;

    /**
     * mysql_stmt_close
     */
    public static final byte COM_STMT_CLOSE = 25;

    /**
     * mysql_stmt_reset
     */
    public static final byte COM_STMT_RESET = 26;

    /**
     * mysql_set_server_option
     */
    public static final byte COM_SET_OPTION = 27;

    /**
     * mysql_stmt_fetch
     */
    public static final byte COM_STMT_FETCH = 28;

    /**
     * mysql_reset_connection
     */
    public static final byte COM_RESET_CONNECTION = 31;

    /**
     * Mycat heartbeat
     */
    public static final byte COM_HEARTBEAT = 64;
    
    //包头大小
    public static final int packetHeaderSize = 4;


    public int packetLength;
    public byte packetId;

    /**
     * 把数据包写到buffer中，如果buffer满了就把buffer通过前端连接写出 (writeSocketIfFull=true)。
     */
    public ByteBuffer write(ByteBuffer buffer, FrontendConnection c,boolean writeSocketIfFull) {
        throw new UnsupportedOperationException();
    }

    /**
     * 把数据包通过后端连接写出，一般使用buffer机制来提高写的吞吐量。
     */
    public void write(BackendAIOConnection c) {
        throw new UnsupportedOperationException();
    }

    /**
     * 计算数据包大小，不包含包头长度。
     */
    public abstract int calcPacketSize();

    /**
     * 取得数据包信息
     */
    protected abstract String getPacketInfo();

    @Override
    public String toString() {
        return new StringBuilder().append(getPacketInfo()).append("{length=").append(packetLength).append(",id=")
                .append(packetId).append('}').toString();
    }

}

================================================
FILE: src/main/java/io/mycat/net/mysql/OkPacket.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.mysql;

import java.nio.ByteBuffer;

import io.mycat.MycatServer;
import io.mycat.backend.mysql.BufferUtil;
import io.mycat.backend.mysql.MySQLMessage;
import io.mycat.net.FrontendConnection;

/**
 * From server to client in response to command, if no error and no result set.
 * 
 * 
 * Bytes                       Name
 * -----                       ----
 * 1                           field_count, always = 0
 * 1-9 (Length Coded Binary)   affected_rows
 * 1-9 (Length Coded Binary)   insert_id
 * 2                           server_status
 * 2                           warning_count
 * n   (until end of packet)   message fix:(Length Coded String)
 * 
 * @see http://forge.mysql.com/wiki/MySQL_Internals_ClientServer_Protocol#OK_Packet
 * 

 * 
 * @author mycat
 */
public class OkPacket extends MySQLPacket {
	public static final byte FIELD_COUNT = 0x00;
	public static final byte[] OK = new byte[] { 7, 0, 0, 1, 0, 0, 0, 2, 0, 0,
			0 };

	public byte fieldCount = FIELD_COUNT;
	public long affectedRows;
	public long insertId;
	public int serverStatus;
	public int warningCount;
	public byte[] message;

	public void read(BinaryPacket bin) {
		packetLength = bin.packetLength;
		packetId = bin.packetId;
		MySQLMessage mm = new MySQLMessage(bin.data);
		fieldCount = mm.read();
		affectedRows = mm.readLength();
		insertId = mm.readLength();
		serverStatus = mm.readUB2();
		warningCount = mm.readUB2();
		if (mm.hasRemaining()) {
			this.message = mm.readBytesWithLength();
		}
	}

	public void read(byte[] data) {
		MySQLMessage mm = new MySQLMessage(data);
		packetLength = mm.readUB3();
		packetId = mm.read();
		fieldCount = mm.read();
		affectedRows = mm.readLength();
		insertId = mm.readLength();
		serverStatus = mm.readUB2();
		warningCount = mm.readUB2();
		if (mm.hasRemaining()) {
			this.message = mm.readBytesWithLength();
		}
	}

	public byte[] writeToBytes(FrontendConnection c) {
		ByteBuffer buffer = c.allocate();
		this.write(buffer, c);
		buffer.flip();
		byte[] data = new byte[buffer.limit()];
		buffer.get(data);
		c.recycle(buffer);
		return data;
	}

	private ByteBuffer write(ByteBuffer buffer, FrontendConnection c) {

		int size = calcPacketSize();
		buffer = c.checkWriteBuffer(buffer, c.getPacketHeaderSize() + size,
				true);
		BufferUtil.writeUB3(buffer, calcPacketSize());
		buffer.put(packetId);
		buffer.put(fieldCount);
		BufferUtil.writeLength(buffer, affectedRows);
		BufferUtil.writeLength(buffer, insertId);
		BufferUtil.writeUB2(buffer, serverStatus);
		BufferUtil.writeUB2(buffer, warningCount);
		if (message != null) {
			BufferUtil.writeWithLength(buffer, message);
		}

		return buffer;

	}

	public void write(FrontendConnection c) {
		ByteBuffer buffer = write(c.allocate(), c);
		c.write(buffer);
	}

	@Override
	public int calcPacketSize() {
		int i = 1;
		i += BufferUtil.getLength(affectedRows);
		i += BufferUtil.getLength(insertId);
		i += 4;
		if (message != null) {
			i += BufferUtil.getLength(message);
		}
		return i;
	}

	@Override
	protected String getPacketInfo() {
		return "MySQL OK Packet";
	}

	 public byte[] writeToBytes() {
	
	   int totalSize = calcPacketSize() + packetHeaderSize;
        ByteBuffer buffer=MycatServer.getInstance().getBufferPool().allocate(totalSize);
        BufferUtil.writeUB3(buffer, calcPacketSize());
        buffer.put(packetId);
        buffer.put(fieldCount);
        BufferUtil.writeLength(buffer, affectedRows);
        BufferUtil.writeLength(buffer, insertId);
        BufferUtil.writeUB2(buffer, serverStatus);
        BufferUtil.writeUB2(buffer, warningCount);
        if (message != null) {
            BufferUtil.writeWithLength(buffer, message);
        }
        buffer.flip();
        byte[] data = new byte[buffer.limit()];
        buffer.get(data);
		 MycatServer.getInstance().getBufferPool().recycle(buffer);
		 return data;
    }

	public void markMoreResultsExists() {
		serverStatus = serverStatus | StatusFlags.SERVER_MORE_RESULTS_EXISTS;
	}

	public void markNoMoreResultsExists() {
		serverStatus = serverStatus & (~StatusFlags.SERVER_MORE_RESULTS_EXISTS);
	}

	public boolean hasMoreResultsExists() {
		return (serverStatus & StatusFlags.SERVER_MORE_RESULTS_EXISTS) > 0;
	}

}

================================================
FILE: src/main/java/io/mycat/net/mysql/PingPacket.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.mysql;

/**
 * @author mycat
 */
public class PingPacket extends MySQLPacket {
    public static final byte[] PING = new byte[] { 1, 0, 0, 0, 14 };

    @Override
    public int calcPacketSize() {
        return 1;
    }

    @Override
    protected String getPacketInfo() {
        return "MySQL Ping Packet";
    }

}

================================================
FILE: src/main/java/io/mycat/net/mysql/PreparedOkPacket.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.mysql;

import java.nio.ByteBuffer;

import io.mycat.backend.mysql.BufferUtil;
import io.mycat.backend.mysql.MySQLMessage;
import io.mycat.net.FrontendConnection;

/**
 * 
 * From server to client, in response to prepared statement initialization packet. 
 * It is made up of: 
 *   1.a PREPARE_OK packet
 *   2.if "number of parameters" > 0 
 *       (field packets) as in a Result Set Header Packet 
 *       (EOF packet)
 *   3.if "number of columns" > 0 
 *       (field packets) as in a Result Set Header Packet 
 *       (EOF packet)
 *   
 * -----------------------------------------------------------------------------------------
 * 
 *  Bytes              Name
 *  -----              ----
 *  1                  0 - marker for OK packet
 *  4                  statement_handler_id
 *  2                  number of columns in result set
 *  2                  number of parameters in query
 *  1                  filler (always 0)
 *  2                  warning count
 *  
 *  @see http://dev.mysql.com/doc/internals/en/prepared-statement-initialization-packet.html
 * 

 * 
 * @author mycat
 */
public class PreparedOkPacket extends MySQLPacket {
    public static int PACKET_SIZE = 12;
    public byte flag;
    public long statementId;
    public int columnsNumber;
    public int parametersNumber;
    public byte filler;
    public int warningCount;

    public PreparedOkPacket() {
        this.flag = 0;
        this.filler = 0;
    }

    public void read(byte[] data) {
        MySQLMessage mm = new MySQLMessage(data);
        packetLength = mm.readUB3();
        packetId = mm.read();
        flag = mm.read();
        statementId = mm.readUB4();
        columnsNumber = mm.readUB2();
        parametersNumber = mm.readUB2();
        warningCount = mm.read();

    }
    @Override
    public ByteBuffer write(ByteBuffer buffer, FrontendConnection c,boolean writeSocketIfFull) {
        int size = calcPacketSize();
        buffer = c.checkWriteBuffer(buffer, c.getPacketHeaderSize() + size,writeSocketIfFull);
        BufferUtil.writeUB3(buffer, size);
        buffer.put(packetId);
        buffer.put(flag);
        BufferUtil.writeUB4(buffer, statementId);
        BufferUtil.writeUB2(buffer, columnsNumber);
        BufferUtil.writeUB2(buffer, parametersNumber);
        buffer.put(filler);
        BufferUtil.writeUB2(buffer, warningCount);
        return buffer;
    }

    @Override
    public int calcPacketSize() {
        return PACKET_SIZE;
    }

    @Override
    protected String getPacketInfo() {
        return "MySQL PreparedOk Packet";
    }

}

================================================
FILE: src/main/java/io/mycat/net/mysql/QuitPacket.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.mysql;

/**
 * @author mycat
 */
public class QuitPacket extends MySQLPacket {
    public static final byte[] QUIT = new byte[] { 1, 0, 0, 0, 1 };

    @Override
    public int calcPacketSize() {
        return 1;
    }

    @Override
    protected String getPacketInfo() {
        return "MySQL Quit Packet";
    }

}

================================================
FILE: src/main/java/io/mycat/net/mysql/Reply323Packet.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.mysql;

import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;

import io.mycat.backend.mysql.BufferUtil;
import io.mycat.backend.mysql.StreamUtil;
import io.mycat.net.BackendAIOConnection;

/**
 * @author mycat
 */
public class Reply323Packet extends MySQLPacket {

    public byte[] seed;

    public void write(OutputStream out) throws IOException {
        StreamUtil.writeUB3(out, calcPacketSize());
        StreamUtil.write(out, packetId);
        if (seed == null) {
            StreamUtil.write(out, (byte) 0);
        } else {
            StreamUtil.writeWithNull(out, seed);
        }
    }

    @Override
    public void write(BackendAIOConnection c) {
        ByteBuffer buffer = c.allocate();
        BufferUtil.writeUB3(buffer, calcPacketSize());
        buffer.put(packetId);
        if (seed == null) {
            buffer.put((byte) 0);
        } else {
            BufferUtil.writeWithNull(buffer, seed);
        }
        c.write(buffer);
    }

    @Override
    public int calcPacketSize() {
        return seed == null ? 1 : seed.length + 1;
    }

    @Override
    protected String getPacketInfo() {
        return "MySQL Auth323 Packet";
    }

}

================================================
FILE: src/main/java/io/mycat/net/mysql/RequestFilePacket.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.mysql;

import java.nio.ByteBuffer;

import io.mycat.backend.mysql.BufferUtil;
import io.mycat.net.FrontendConnection;

/**
 * load data local infile 向客户端请求发送文件用
 */
public class RequestFilePacket extends MySQLPacket
{
    public static final byte FIELD_COUNT = (byte) 251;
    public byte command = FIELD_COUNT;
    public byte[] fileName;


    @Override
    public ByteBuffer write(ByteBuffer buffer, FrontendConnection c, boolean writeSocketIfFull)
    {
        int size = calcPacketSize();
        buffer = c.checkWriteBuffer(buffer, c.getPacketHeaderSize() + size, writeSocketIfFull);
        BufferUtil.writeUB3(buffer, size);
        buffer.put(packetId);
        buffer.put(command);
        if (fileName != null)
        {

            buffer.put(fileName);

        }

        c.write(buffer);

        return buffer;
    }

    @Override
    public int calcPacketSize()
    {
        return fileName == null ? 1 : 1 + fileName.length;
    }

    @Override
    protected String getPacketInfo()
    {
        return "MySQL Request File Packet";
    }


}

================================================
FILE: src/main/java/io/mycat/net/mysql/ResetPacket.java
================================================
package io.mycat.net.mysql;

import io.mycat.backend.mysql.MySQLMessage;

/**
 * 
 * 
 * COM_STMT_RESET resets the data of a prepared statement which was accumulated with COM_STMT_SEND_LONG_DATA commands and closes the cursor if it was opened with COM_STMT_EXECUTE

 * The server will send a OK_Packet if the statement could be reset, a ERR_Packet if not.
 * 
 * COM_STMT_RESET:
 * COM_STMT_RESET
 * direction: client -> server
 * response: OK or ERR

 * payload:
 *   1              [1a] COM_STMT_RESET
 *   4              statement-id
 * 
 * 

 * 
 * @author CrazyPig
 * @since 2016-09-08
 *
 */
public class ResetPacket extends MySQLPacket {

	private static final byte PACKET_FALG = (byte) 26;
	private long pstmtId;
	
	public void read(byte[] data) {
		MySQLMessage mm = new MySQLMessage(data);
		packetLength = mm.readUB3();
		packetId = mm.read();
		byte code = mm.read();
		assert code == PACKET_FALG;
		pstmtId = mm.readUB4();
	}
	
	@Override
	public int calcPacketSize() {
		return 1 + 4;
	}

	@Override
	protected String getPacketInfo() {
		return "MySQL Reset Packet";
	}

	public long getPstmtId() {
		return pstmtId;
	}

}


================================================
FILE: src/main/java/io/mycat/net/mysql/ResultSetHeaderPacket.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.mysql;

import java.nio.ByteBuffer;

import io.mycat.backend.mysql.BufferUtil;
import io.mycat.backend.mysql.MySQLMessage;
import io.mycat.buffer.BufferArray;
import io.mycat.net.FrontendConnection;

/**
 * From server to client after command, if no error and result set -- that is,
 * if the command was a query which returned a result set. The Result Set Header
 * Packet is the first of several, possibly many, packets that the server sends
 * for result sets. The order of packets for a result set is:
 * 
 * 
 * (Result Set Header Packet)   the number of columns
 * (Field Packets)              column descriptors
 * (EOF Packet)                 marker: end of Field Packets
 * (Row Data Packets)           row contents
 * (EOF Packet)                 marker: end of Data Packets
 * 
 * Bytes                        Name
 * -----                        ----
 * 1-9   (Length-Coded-Binary)  field_count
 * 1-9   (Length-Coded-Binary)  extra
 * 
 * @see http://forge.mysql.com/wiki/MySQL_Internals_ClientServer_Protocol#Result_Set_Header_Packet
 * 

 * 
 * @author mycat
 */
public class ResultSetHeaderPacket extends MySQLPacket {

    public int fieldCount;
    public long extra;

    public void read(byte[] data) {
        MySQLMessage mm = new MySQLMessage(data);
        this.packetLength = mm.readUB3();
        this.packetId = mm.read();
        this.fieldCount = (int) mm.readLength();
        if (mm.hasRemaining()) {
            this.extra = mm.readLength();
        }
    }

    @Override
    public ByteBuffer write(ByteBuffer buffer, FrontendConnection c,boolean writeSocketIfFull) {
        int size = calcPacketSize();
        buffer = c.checkWriteBuffer(buffer, c.getPacketHeaderSize() + size,writeSocketIfFull);
        BufferUtil.writeUB3(buffer, size);
        buffer.put(packetId);
        BufferUtil.writeLength(buffer, fieldCount);
        if (extra > 0) {
            BufferUtil.writeLength(buffer, extra);
        }
        return buffer;
    }
    
    public void write(BufferArray bufferArray) {
		int size = calcPacketSize();
		ByteBuffer buffer = bufferArray
				.checkWriteBuffer(packetHeaderSize + size);
		BufferUtil.writeUB3(buffer, size);
		buffer.put(packetId);
		BufferUtil.writeLength(buffer, fieldCount);
		if (extra > 0) {
			BufferUtil.writeLength(buffer, extra);
		}
	}

    @Override
    public int calcPacketSize() {
        int size = BufferUtil.getLength(fieldCount);
        if (extra > 0) {
            size += BufferUtil.getLength(extra);
        }
        return size;
    }

    @Override
    protected String getPacketInfo() {
        return "MySQL ResultSetHeader Packet";
    }

	

}

================================================
FILE: src/main/java/io/mycat/net/mysql/RowDataPacket.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.mysql;

import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;

import io.mycat.backend.mysql.BufferUtil;
import io.mycat.backend.mysql.MySQLMessage;
import io.mycat.buffer.BufferArray;
import io.mycat.net.FrontendConnection;

/**
 * From server to client. One packet for each row in the result set.
 * 
 * 
 * Bytes                   Name
 * -----                   ----
 * n (Length Coded String) (column value)
 * ...
 * 
 * (column value):         The data in the column, as a character string.
 *                         If a column is defined as non-character, the
 *                         server converts the value into a character
 *                         before sending it. Since the value is a Length
 *                         Coded String, a NULL can be represented with a
 *                         single byte containing 251(see the description
 *                         of Length Coded Strings in section "Elements" above).
 * 
 * @see http://forge.mysql.com/wiki/MySQL_Internals_ClientServer_Protocol#Row_Data_Packet
 * 

 * 
 * @author mycat
 */
public class RowDataPacket extends MySQLPacket {
	private static final byte NULL_MARK = (byte) 251;
    private static final byte EMPTY_MARK = (byte) 0;

	public byte[] value;
	public int fieldCount;
	public final List fieldValues;

	public RowDataPacket(int fieldCount) {
		this.fieldCount = fieldCount;
		this.fieldValues = new ArrayList(fieldCount);
	}

	public void add(byte[] value) {
		//这里应该修改value
		fieldValues.add(value);
	}
	public void addFieldCount(int add) {
		//这里应该修改field
		fieldCount=fieldCount+add;
	}
	
	public void read(byte[] data) {
		value = data;
		MySQLMessage mm = new MySQLMessage(data);
		packetLength = mm.readUB3();
		packetId = mm.read();
		for (int i = 0; i < fieldCount; i++) {
			fieldValues.add(mm.readBytesWithLength());
		}
	}

	@Override
	public ByteBuffer write(ByteBuffer bb, FrontendConnection c,
			boolean writeSocketIfFull) {
		bb = c.checkWriteBuffer(bb, c.getPacketHeaderSize(), writeSocketIfFull);
		BufferUtil.writeUB3(bb, calcPacketSize());
		bb.put(packetId);
		for (int i = 0; i < fieldCount; i++) {
			byte[] fv = fieldValues.get(i);
			if (fv == null ) {
				bb = c.checkWriteBuffer(bb, 1, writeSocketIfFull);
				bb.put(RowDataPacket.NULL_MARK);
			}else if (fv.length == 0) {
                bb = c.checkWriteBuffer(bb, 1, writeSocketIfFull);
                bb.put(RowDataPacket.EMPTY_MARK);
            }
            else {
				bb = c.checkWriteBuffer(bb, BufferUtil.getLength(fv),
						writeSocketIfFull);
				BufferUtil.writeLength(bb, fv.length);
				bb = c.writeToBuffer(fv, bb);
			}
		}
		return bb;
	}

	@Override
	public int calcPacketSize() {
		int size = 0;
		for (int i = 0; i < fieldCount; i++) {
			byte[] v = fieldValues.get(i);
			size += (v == null || v.length == 0) ? 1 : BufferUtil.getLength(v);
		}
		return size;
	}

	@Override
	protected String getPacketInfo() {
		return "MySQL RowData Packet";
	}

	public void write(BufferArray bufferArray) {
		int size = calcPacketSize();
		ByteBuffer buffer = bufferArray.checkWriteBuffer(packetHeaderSize + size);
		BufferUtil.writeUB3(buffer, size);
		buffer.put(packetId);
		for (int i = 0; i < fieldCount; i++) {
			byte[] fv = fieldValues.get(i);
			if (fv == null) {
				buffer = bufferArray.checkWriteBuffer(1);
				buffer.put(RowDataPacket.NULL_MARK);
			} else if (fv.length == 0) {
				buffer = bufferArray.checkWriteBuffer(1);
				buffer.put(RowDataPacket.EMPTY_MARK);
			} else {
				buffer = bufferArray.checkWriteBuffer(BufferUtil
						.getLength(fv.length));
				BufferUtil.writeLength(buffer, fv.length);
				bufferArray.write(fv);
			}
		}
	}

}

================================================
FILE: src/main/java/io/mycat/net/mysql/StatusFlags.java
================================================
package io.mycat.net.mysql;

public final class StatusFlags {
    private StatusFlags() {
    }

    // a transaction is active
    public static final int SERVER_STATUS_IN_TRANS = 0x0001;
    // auto-commit is enabled
    public static final int SERVER_STATUS_AUTOCOMMIT = 0x0002;
    public static final int SERVER_MORE_RESULTS_EXISTS = 0x0008;
    public static final int SERVER_STATUS_NO_GOOD_INDEX_USED = 0x0010;
    public static final int SERVER_STATUS_NO_INDEX_USED = 0x0020;
    // Used by Binary Protocol Resultset to signal that COM_STMT_FETCH has to be
    // used to fetch the row-data.
    public static final int SERVER_STATUS_CURSOR_EXISTS = 0x0040;
    public static final int SERVER_STATUS_LAST_ROW_SENT = 0x0080;
    public static final int SERVER_STATUS_DB_DROPPED = 0x0100;
    public static final int SERVER_STATUS_NO_BACKSLASH_ESCAPES = 0x0200;
    public static final int SERVER_STATUS_METADATA_CHANGED = 0x0400;
    public static final int SERVER_QUERY_WAS_SLOW = 0x0800;
    public static final int SERVER_PS_OUT_PARAMS = 0x1000;
    // in a read-only transaction
    public static final int SERVER_STATUS_IN_TRANS_READONLY = 0x2000;
    public static final int SERVER_SESSION_STATE_CHANGED = 0x4000;
}


================================================
FILE: src/main/java/io/mycat/net/postgres/AuthenticationCleartextPassword.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * AuthenticationCleartextPassword (B)
 * Byte1('R') Identifies the message as an authentication request. 
 * Int32(8) Length of message contents in bytes, including self. 
 * Int32(3) Specifies that a clear-text password is required.
 * 

 * 
 * @author mycat
 */
public class AuthenticationCleartextPassword extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/AuthenticationGSS.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * AuthenticationGSS (B) 
 * Byte1('R') Identifies the message as an authentication request. 
 * Int32(8) Length of message contents in bytes, including self. 
 * Int32(7) Specifies that GSSAPI authentication is required.
 * 

 * 
 * @author mycat
 */
public class AuthenticationGSS extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/AuthenticationGSSContinue.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * AuthenticationGSSContinue (B) 
 * Byte1('R') Identifies the message as an authentication request. 
 * Int32 Length of message contents in bytes, including self. 
 * Int32(8) Specifies that this message contains GSSAPI or SSPI data. 
 * Byten GSSAPI or SSPI authentication data.
 * 

 * 
 * @author mycat
 */
public class AuthenticationGSSContinue extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/AuthenticationKerberosV5.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * AuthenticationKerberosV5 (B) 
 * Byte1('R') Identifies the message as an authentication request. 
 * Int32(8) Length of message contents in bytes, including self. 
 * Int32(2) Specifies that Kerberos V5 authentication is required.
 * 

 * 
 * @author mycat
 */
public class AuthenticationKerberosV5 extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/AuthenticationMD5Password.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * AuthenticationMD5Password (B)
 * Byte1('R') Identifies the message as an authentication request. 
 * Int32(12) Length of message contents in bytes, including self. 
 * Int32(5) Specifies that an MD5-encrypted password is required. 
 * Byte4 The salt to use when encrypting the password.
 * 

 * 
 * @author mycat
 */
public class AuthenticationMD5Password extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/AuthenticationOk.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * AuthenticationOk (B) 
 * Byte1('R') Identifies the message as an authentication request. 
 * Int32(8) Length of message contents in bytes, including self. 
 * Int32(0) Specifies that the authentication was successful.
 * 

 * 
 * @author mycat
 */
public class AuthenticationOk extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/AuthenticationSCMCredential.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * AuthenticationSCMCredential (B) 
 * Byte1('R') Identifies the message as an authentication request. 
 * Int32(8) Length of message contents in bytes, including self. 
 * Int32(6) Specifies that an SCM credentials message is required.
 * 

 * 
 * @author mycat
 */
public class AuthenticationSCMCredential extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/AuthenticationSSPI.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * AuthenticationSSPI (B) 
 * Byte1('R') Identifies the message as an authentication request. 
 * Int32(8) Length of message contents in bytes, including self. 
 * Int32(9) Specifies that SSPI authentication is required.
 * 

 * 
 * @author mycat
 */
public class AuthenticationSSPI extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/BackendKeyData.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * BackendKeyData (B) 
 * Byte1('K') Identifies the message as cancellation key data. 
 *            The frontend must save these values if it wishes to be able to
 *            issue CancelRequest messages later. 
 * Int32(12) Length of message contents in bytes, including self. 
 * Int32 The process ID of this backend. 
 * Int32 The secret key of this backend.
 * 

 * 
 * @author mycat
 */
public class BackendKeyData extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/Bind.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * Bind (F) 
 * Byte1('B') Identifies the message as a Bind command. 
 * Int32 Length of message contents in bytes, including self. 
 * String The name of the destination portal (an empty string selects the unnamed portal).
 * String The name of the source prepared statement (an empty string selects the unnamed 
 *        prepared statement). 
 * Int16 The number of parameter format codes that follow (denoted C below). 
 *       This can be zero to indicate that there are no parameters or that the parameters 
 *       all use the default format(text); or one, in which case the specified format code 
 *       is applied to all parameters; or it can equal the actual number of parameters. 
 * Int16[C] The parameter format codes. Each must presently be zero (text) or one(binary). 
 * Int16 The number of parameter values that follow (possibly zero). This must match the 
 *       number of parameters needed by the query. Next, the following pair of fields appear 
 *       for each parameter: 
 * Int32 The length of the parameter value, in bytes (this count does not include
 *       itself). Can be zero. As a special case, -1 indicates a NULL parameter
 *       value. No value bytes follow in the NULL case. 
 * Byten The value of the parameter, in the format indicated by the associated format code. 
 *       n is the above length. After the last parameter, the following fields appear:
 * Int16 The number of result-column format codes that follow (denoted R
 *       below). This can be zero to indicate that there are no result columns or
 *       that the result columns should all use the default format (text); or one,
 *       in which case the specified format code is applied to all result columns
 *       (if any); or it can equal the actual number of result columns of the query. 
 * Int16[R] The result-column format codes. Each must presently be zero (text) or one (binary).
 * 

 * 
 * @author mycat
 */
public class Bind extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/BindComplete.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * BindComplete (B) 
 * Byte1('2') Identifies the message as a Bind-complete indicator. 
 * Int32(4) Length of message contents in bytes, including self.
 * 

 * 
 * @author mycat
 */
public class BindComplete extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/CancelRequest.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * CancelRequest (F) 
 * Int32(16) Length of message contents in bytes,including self. 
 * Int32(80877102) The cancel request code. The value is chosen to 
 *                 contain 1234 in the most significant 16 bits, and 
 *                 5678 in the least 16 significant bits. (To avoid 
 *                 confusion, this code must not be the same as any 
 *                 protocol version number.) 
 * Int32 The process ID of the target backend. 
 * Int32 The secret key for the target backend.
 * 

 * 
 * @author mycat
 */
public class CancelRequest extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/Close.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * Close (F) 
 * Byte1('C') Identifies the message as a Close command. 
 * Int32 Length of message contents in bytes, including self. 
 * Byte1 'S' to close a prepared statement; or 'P' to close a portal. 
 * String The name of the prepared statement or portal to close (an 
 *        empty string selects the unnamed prepared statement or portal).
 * 

 * 
 * @author mycat
 */
public class Close extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/CloseComplete.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * CloseComplete (B) 
 * Byte1('3') Identifies the message as a Close-complete indicator. 
 * Int32(4) Length of message contents in bytes, including self.
 * 

 * 
 * @author mycat
 */
public class CloseComplete extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/CommandComplete.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * CommandComplete (B)
 * Byte1('C') Identifies the message as a command-completed response.     
 * Int32 Length of message contents in bytes, including self.     
 * String The command tag. This is usually a single word that identifies which SQL command was completed. 
 *        For an INSERT command, the tag is INSERT oid rows, where rows is the number of rows inserted. 
 *        oid is the object ID of the inserted row if rows is 1 and the target table has OIDs; otherwise oid is 0.  
 *        For a DELETE command, the tag is DELETE rows where rows is the number of rows deleted. 
 *        For an UPDATE command, the tag is UPDATE rows where rows is the number of rows updated. 
 *        For a SELECT or CREATE TABLE AS command, the tag is SELECT rows where rows is the number of rows retrieved. 
 *        For a MOVE command, the tag is MOVE rows where rows is the number of rows the cursor's position has been changed by. 
 *        For a FETCH command, the tag is FETCH rows where rows is the number of rows that have been retrieved from the cursor. 
 *        For a COPY command, the tag is COPY rows where rows is the number of rows copied. 
 *        (Note: the row count appears only in PostgreSQL 8.2 and later.)
 * 

 * 
 * @author mycat
 */
public class CommandComplete extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/CopyBothResponse.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * CopyBothResponse (B) 
 * Byte1('W') Identifies the message as a Start Copy Both response. 
 *            This message is used only for Streaming Replication. 
 * Int32 Length of message contents in bytes, including self. 
 * Int8 0 indicates the overall COPY format is textual (rows separated 
 *      by newlines, columns separated by separator characters, etc). 
 *      1 indicates the overall copy format is binary (similar to DataRow 
 *      format). See COPY for more information. 
 * Int16 The number of columns in the data to be copied(denoted N below). 
 * Int16[N] The format codes to be used for each column. Each must presently 
 *          be zero (text) or one (binary). All must be zero if the overall 
 *          copy format is textual.
 * 

 * 
 * @author mycat
 */
public class CopyBothResponse extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/CopyData.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * CopyData (F & B) 
 * Byte1('d') Identifies the message as COPY data. 
 * Int32 Length of message contents in bytes, including self. 
 * Byten Data that forms part of a COPY data stream. Messages sent from the backend will
 *       always correspond to single data rows, but messages sent by frontends
 *       might divide the data stream arbitrarily.
 * 

 * 
 * @author mycat
 */
public class CopyData extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/CopyDone.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * CopyDone (F & B) 
 * Byte1('c') Identifies the message as a COPY-complete indicator. 
 * Int32(4) Length of message contents in bytes, including self.
 * 

 * 
 * @author mycat
 */
public class CopyDone extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/CopyFail.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * CopyFail (F) 
 * Byte1('f') Identifies the message as a COPY-failure indicator. 
 * Int32 Length of message contents in bytes, including self.
 * String An error message to report as the cause of failure.
 * 

 * 
 * @author mycat
 */
public class CopyFail extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/CopyInResponse.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * CopyInResponse (B)     
 * Byte1('G') Identifies the message as a Start Copy In response. 
 *            The frontend must now send copy-in data (if not prepared 
 *            to do so, send a CopyFail message).
 * Int32 Length of message contents in bytes, including self.
 * Int8 0 indicates the overall COPY format is textual (rows separated 
 *      by newlines, columns separated by separator characters, etc). 
 *      1 indicates the overall copy format is binary (similar to DataRow 
 *      format). See COPY for more information.
 * Int16 The number of columns in the data to be copied (denoted N below).
 * Int16[N] The format codes to be used for each column. Each must presently 
 *          be zero (text) or one (binary). All must be zero if the overall 
 *          copy format is textual.
 * 

 * 
 * @author mycat
 */
public class CopyInResponse extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/CopyOutResponse.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * CopyOutResponse (B) 
 * Byte1('H') Identifies the message as a Start Copy Out response. 
 *            This message will be followed by copy-out data. Int32 Length of
 *            message contents in bytes, including self. 
 * Int8 0 indicates the overall COPY format is textual (rows separated by 
 *      newlines, columns separated by separator characters, etc). 1 indicates 
 *      the overall copy format is binary(similar to DataRow format). 
 *      See COPY for more information. 
 * Int16 The number of columns in the data to be copied (denoted N below). 
 * Int16[N] The format codes to be used for each column. Each must presently 
 *          be zero(text) or one (binary). All must be zero if the overall 
 *          copy format is textual.
 * 

 * 
 * @author mycat
 */
public class CopyOutResponse extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/DataRow.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * DataRow (B) 
 * Byte1('D') Identifies the message as a data row. 
 * Int32 Length of message contents in bytes, including self. 
 * Int16 The number of column values that follow (possibly zero). 
 *       Next, the following pair of fields appear for each column: 
 * Int32 The length of the column value, in bytes(this count does not 
 *       include itself). Can be zero. As a special case, -1 indicates 
 *       a NULL column value. No value bytes follow in the NULL case.
 * Byten The value of the column, in the format indicated by the associated
 *       format code. n is the above length.
 * 

 * 
 * @author mycat
 */
public class DataRow extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/Describe.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * Describe (F) 
 * Byte1('D') Identifies the message as a Describe command.
 * Int32 Length of message contents in bytes, including self. 
 * Byte1 'S' to describe a prepared statement; or 'P' to describe a portal. 
 * String The name of the prepared statement or portal to describe (an empty 
 *        string selects the unnamed prepared statement or portal).
 * 

 * 
 * @author mycat
 */
public class Describe extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/EmptyQueryResponse.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * EmptyQueryResponse (B) 
 * Byte1('I') Identifies the message as a response to an empty query 
 *            string. (This substitutes for CommandComplete.) 
 * Int32(4) Length of message contents in bytes, including self.
 * 

 * 
 * @author mycat
 */
public class EmptyQueryResponse extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/ErrorResponse.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * ErrorResponse (B) 
 * Byte1('E') Identifies the message as an error. 
 * Int32 Length of message contents in bytes, including self. 
 *       The message body consists of one or more identified fields, 
 *       followed by a zero byte as a terminator. Fields can appear 
 *       in any order. For each field there is the following: 
 * Byte1 A code identifying the field type; if zero, this is the
 *       message terminator and no string follows. The presently defined 
 *       field types are listed in Section 46.6. Since more field types 
 *       might be added in future, frontends should silently ignore 
 *       fields of unrecognized type.
 * String The field value.
 * 

 * 
 * @author mycat
 */
public class ErrorResponse extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/Execute.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * Execute (F) 
 * Byte1('E') Identifies the message as an Execute command.
 * Int32 Length of message contents in bytes, including self. 
 * String The name of the portal to execute (an empty string 
 *        selects the unnamed portal). 
 * Int32 Maximum number of rows to return, if portal contains a
 *       query that returns rows (ignored otherwise). 
 *       Zero denotes "no limit".
 * 

 * 
 * @author mycat
 */
public class Execute extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/Flush.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * Flush (F) 
 * Byte1('H') Identifies the message as a Flush command. 
 * Int32(4) Length of message contents in bytes, including self.
 * 

 * 
 * @author mycat
 */
public class Flush extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/FunctionCall.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * FunctionCall (F) 
 * Byte1('F') Identifies the message as a function call.
 * Int32 Length of message contents in bytes, including self. 
 * Int32 Specifies the object ID of the function to call. 
 * Int16 The number of argument format codes that follow (denoted C below). 
 *       This can be zero to indicate that there are no arguments or that 
 *       the arguments all use the default format (text); or one, in which 
 *       case the specified format code is applied to all arguments; or it 
 *       can equal the actual number of arguments.
 * Int16[C] The argument format codes. Each must presently be zero (text) or
 *          one (binary). 
 * Int16 Specifies the number of arguments being supplied to the function. 
 *       Next, the following pair of fields appear for each argument: 
 * Int32 The length of the argument value, in bytes (this count does not include 
 *       itself). Can be zero. As a special case, -1 indicates a NULL argument 
 *       value. No value bytes follow in the NULL case. 
 * Byten The value of the argument, in the format indicated by the associated 
 *       format code. n is the above length. After the last argument, the 
 *       following field appears: 
 * Int16 The format code for the function result. Must presently be zero (text) 
 *       or one (binary).
 * 

 * 
 * @author mycat
 */
public class FunctionCall extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/FunctionCallResponse.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * FunctionCallResponse (B) 
 * Byte1('V') Identifies the message as a function call result. 
 * Int32 Length of message contents in bytes, including self.
 * Int32 The length of the function result value, in bytes (this count does
 *       not include itself). Can be zero. As a special case, -1 indicates a 
 *       NULL function result. No value bytes follow in the NULL case. 
 * Byten The value of the function result, in the format indicated by the 
 *       associated format code. n is the above length.
 * 

 * 
 * @author mycat
 */
public class FunctionCallResponse extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/NoData.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * NoData (B) 
 * Byte1('n') Identifies the message as a no-data indicator.
 * Int32(4) Length of message contents in bytes, including self.
 * 

 * 
 * @author mycat
 */
public class NoData extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/NoticeResponse.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * NoticeResponse (B) 
 * Byte1('N') Identifies the message as a notice. 
 * Int32 Length of message contents in bytes, including self. The message 
 *       body consists of one or more identified fields, followed by a zero 
 *       byte as a terminator. Fields can appear in any order. For each 
 *       field there is the following: 
 * Byte1 A code identifying the field type; if zero, this is the message 
 *       terminator and no string follows. The presently defined field types 
 *       are listed in Section 46.6. Since more field types might be added
 *       in future, frontends should silently ignore fields of unrecognized 
 *       type.
 * String The field value.
 * 

 * 
 * @author mycat
 */
public class NoticeResponse extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/NotificationResponse.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * NotificationResponse (B) 
 * Byte1('A') Identifies the message as a notification response. 
 * Int32 Length of message contents in bytes,including self. 
 * Int32 The process ID of the notifying backend process.
 * String The name of the channel that the notify has been raised on. 
 * String The "payload" string passed from the notifying process.
 * 

 * 
 * @author mycat
 */
public class NotificationResponse extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/ParameterDescription.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * ParameterDescription (B) 
 * Byte1('t') Identifies the message as a parameter description. 
 * Int32 Length of message contents in bytes, including self.
 * Int16 The number of parameters used by the statement (can be zero). 
 *       Then,for each parameter, there is the following: 
 * Int32 Specifies the object ID of the parameter data type.
 * 

 * 
 * @author mycat
 */
public class ParameterDescription extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/ParameterStatus.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * ParameterStatus (B) 
 * Byte1('S') Identifies the message as a run-time parameter status report. 
 * Int32 Length of message contents in bytes,including self. 
 * String The name of the run-time parameter being reported.
 * String The current value of the parameter.
 * 

 * 
 * @author mycat
 */
public class ParameterStatus extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/Parse.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * Parse (F) 
 * Byte1('P') Identifies the message as a Parse command. 
 * Int32 Length of message contents in bytes, including self. 
 * String The name of the destination prepared statement (an empty string 
 *        selects the unnamed prepared statement). 
 * String The query string to be parsed. 
 * Int16 The number of parameter data types specified (can be zero). Note 
 *       that this is not an indication of the number of parameters that 
 *       might appear in the query string, only the number that the frontend 
 *       wants to prespecify types for. Then, for each parameter, there is 
 *       the following: 
 * Int32 Specifies the object ID of the parameter data type. Placing a zero 
 *       here is equivalent to leaving the type unspecified.
 * 

 * 
 * @author mycat
 */
public class Parse extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/ParseComplete.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * ParseComplete (B) 
 * Byte1('1') Identifies the message as a Parse-complete indicator. 
 * Int32(4) Length of message contents in bytes, including self.
 * 

 * 
 * @author mycat
 */
public class ParseComplete extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/PasswordMessage.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * PasswordMessage (F) 
 * Byte1('p') Identifies the message as a password response. Note that this is 
 *            also used for GSSAPI and SSPI response messages (which is really
 *            a design error, since the contained data is not a null-terminated 
 *            string in that case, but can be arbitrary binary data).
 * Int32 Length of message contents in bytes, including self. 
 * String The password(encrypted, if requested).
 * 

 * 
 * @author mycat
 */
public class PasswordMessage extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/PortalSuspended.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * PortalSuspended (B) 
 * Byte1('s') Identifies the message as a portal-suspended indicator. Note this 
 *            only appears if an Execute message's row-count limit was reached. 
 * Int32(4) Length of message contents in bytes, including self.
 * 

 * 
 * @author mycat
 */
public class PortalSuspended extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/PostgresPacket.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * @see http://www.postgresql.org/docs/9.1/interactive/protocol.html
 * @author mycat
 */
public abstract class PostgresPacket {
    /**
     * 
     * AuthenticationOk (B)   
     * AuthenticationKerberosV5 (B)       
     * AuthenticationCleartextPassword (B)    
     * AuthenticationMD5Password (B)    
     * AuthenticationSCMCredential (B)     
     * AuthenticationGSS (B)     
     * AuthenticationSSPI (B)      
     * AuthenticationGSSContinue (B)
     * 

     */
    public static final byte AUTHENTICATION = (byte) 'R';

    /**
     * BackendKeyData (B)
     */
    public static final byte BACKEND_KEY_DATA = (byte) 'K';

    /**
     * Bind (F)
     */
    public static final byte BIND = (byte) 'B';

    /**
     * BindComplete (B)
     */
    public static final byte BIND_COMPLETE = (byte) '2';

    /**
     * CancelRequest (F)
     */

    /**
     * Close (F)
     */
    public static final byte CLOSE = (byte) 'C';

    /**
     * CloseComplete (B)
     */
    public static final byte CLOSE_COMPLETE = (byte) '3';

    /**
     * CommandComplete (B)
     */
    public static final byte COMMAND_COMPLETE = (byte) 'C';

    /**
     * CopyData (F & B)
     */
    public static final byte COPY_DATA = (byte) 'd';

    /**
     * CopyDone (F & B)
     */
    public static final byte COPY_DONE = (byte) 'c';

    /**
     * CopyFail (F)
     */
    public static final byte COPY_FAIL = (byte) 'f';

    /**
     * CopyInResponse (B)
     */
    public static final byte COPY_IN_RESPONSE = (byte) 'G';

    /**
     * CopyOutResponse (B)
     */
    public static final byte COPY_OUT_RESPONSE = (byte) 'H';

    /**
     * CopyBothResponse (B)
     */
    public static final byte COPY_BOTH_RESPONSE = (byte) 'W';

    /**
     * DataRow (B)
     */
    public static final byte DATA_ROW = (byte) 'D';

    /**
     * Describe (F)
     */
    public static final byte DESCRIBE = (byte) 'D';

    /**
     * EmptyQueryResponse (B)
     */
    public static final byte EMPTY_QUERY_RESPONSE = (byte) 'I';

    /**
     * ErrorResponse (B)
     */
    public static final byte ERROR_RESPONSE = (byte) 'E';

    /**
     * Execute (F)
     */
    public static final byte EXECUTE = (byte) 'E';

    /**
     * Flush (F)
     */
    public static final byte FLUSH = (byte) 'H';

    /**
     * FunctionCall (F)
     */
    public static final byte FUNCTION_CALL = (byte) 'F';

    /**
     * FunctionCallResponse (B)
     */
    public static final byte FUNCTION_CALL_RESPONSE = (byte) 'V';

    /**
     * NoData (B)
     */
    public static final byte NO_DATA = (byte) 'n';

    /**
     * NoticeResponse (B)
     */
    public static final byte NOTICE_RESPONSE = (byte) 'N';

    /**
     * NotificationResponse (B)
     */
    public static final byte NOTIFICATION_RESPONSE = (byte) 'A';

    /**
     * ParameterDescription (B)
     */
    public static final byte PARAMETER_DESCRIPTION = (byte) 't';

    /**
     * ParameterStatus (B)
     */
    public static final byte PARAMETER_STATUS = (byte) 'S';

    /**
     * Parse (F)
     */
    public static final byte PARSE = (byte) 'P';

    /**
     * ParseComplete (B)
     */
    public static final byte PARSE_COMPLETE = (byte) '1';

    /**
     * PasswordMessage (F)
     */
    public static final byte PASSWORD_MESSAGE = (byte) 'p';

    /**
     * PortalSuspended (B)
     */
    public static final byte PORTAL_SUSPENDED = (byte) 's';

    /**
     * Query (F)
     */
    public static final byte QUERY = (byte) 'Q';

    /**
     * ReadyForQuery (B)
     */
    public static final byte READY_FOR_QUERY = (byte) 'Z';

    /**
     * RowDescription (B)
     */
    public static final byte ROW_DESCRIPTION = (byte) 'T';

    /**
     * SSLRequest (F)
     */

    /**
     * StartupMessage (F)
     */

    /**
     * Sync (F)
     */
    public static final byte SYNC = (byte) 'S';

    /**
     * Terminate (F)
     */
    public static final byte TERMINATE = (byte) 'X';

    private byte             type;
    private int              length;
    
    public byte getType() {
        return type;
    }
    
    public void setType(byte type) {
        this.type = type;
    }
    
    public int getLength() {
        return length;
    }
    
    public void setLength(int length) {
        this.length = length;
    }
    
    

}

================================================
FILE: src/main/java/io/mycat/net/postgres/Query.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * Query (F) 
 * Byte1('Q') Identifies the message as a simple query. 
 * Int32 Length of message contents in bytes, including self. 
 * String The query string itself.
 * 

 * 
 * @author mycat
 */
public class Query extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/ReadyForQuery.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * ReadyForQuery (B) 
 * Byte1('Z') Identifies the message type. ReadyForQuery is sent whenever the 
 *            backend is ready for a new query cycle. 
 * Int32(5) Length of message contents in bytes, including self. 
 * Byte1 Current backend transaction status indicator. Possible values are 'I' 
 *       if idle(not in a transaction block); 'T' if in a transaction block; 
 *       or 'E' if in a failed transaction block (queries will be rejected until
 *       block is ended).
 * 

 * 
 * @author mycat
 */
public class ReadyForQuery extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/RowDescription.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * RowDescription (B) 
 * Byte1('T') Identifies the message as a row description. 
 * Int32 Length of message contents in bytes, including self.
 * Int16 Specifies the number of fields in a row (can be zero). Then, for
 *       each field,there is the following: String The field name. 
 * Int32 If the field can be identified as a column of a specific table, 
 *       the object ID of the table; otherwise zero. 
 * Int16 If the field can be identified as a column of a specific table, the 
 *       attribute number of the column; otherwise zero. 
 * Int32 The object ID of the field's data type. 
 * Int16 The data type size (see pg_type.typlen). Note that negative values 
 *       denote variable-width types. 
 * Int32 The type modifier (see pg_attribute.atttypmod). The meaning of the 
 *       modifier is type-specific.
 * Int16 The format code being used for the field. Currently will be zero
 *       (text) or one (binary). In a RowDescription returned from the 
 *       statement variant of Describe, the format code is not yet known and 
 *       will always be zero.
 * 

 * 
 * @author mycat
 */
public class RowDescription extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/SSLRequest.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * SSLRequest (F) 
 * Int32(8) Length of message contents in bytes, including self. 
 * Int32(80877103) The SSL request code. The value is chosen to contain 1234 in 
 *                 the most significant 16 bits, and 5679 in the least 16 significant 
 *                 bits. (To avoid confusion, this code must not be the same as any 
 *                 protocol version number.)
 * 

 * 
 * @author mycat
 */
public class SSLRequest extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/StartupMessage.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * StartupMessage (F) 
 * Int32 Length of message contents in bytes, including self. 
 * Int32(196608) The protocol version number. The most significant 16 bits are 
 *               the major version number (3 for the protocol described here).
 *               The least significant 16 bits are the minor version number (0 
 *               for the protocol described here). The protocol version number 
 *               is followed by one or more pairs of parameter name and value 
 *               strings. A zero byte is required as a terminator after the 
 *               last name/value pair. Parameters can appear in any order. user 
 *               is required, others are optional. Each parameter is specified as: 
 * String The parameter name. Currently recognized names are: 
 *        user The database user name to connect as. Required; there is no default. 
 *        database The database to connect to. Defaults to the user name. 
 *        options Command-line arguments for the backend. (This is deprecated in 
 *                favor of setting individual run-time parameters.) In addition to
 *                the above, any run-time parameter that can be set at backend start 
 *                time might be listed. Such settings will be applied during backend 
 *                start (after parsing the command-line options if any). The values 
 *                will act as session defaults. 
 * String The parameter value.
 * 

 * 
 * @author mycat
 */
public class StartupMessage extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/Sync.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * Sync (F) 
 * Byte1('S') Identifies the message as a Sync command. 
 * Int32(4) Length of message contents in bytes, including self.
 * 

 * 
 * @author mycat
 */
public class Sync extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/net/postgres/Terminate.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.net.postgres;

/**
 * 
 * Terminate (F) 
 * Byte1('X') Identifies the message as a termination.
 * Int32(4) Length of message contents in bytes, including self.
 * 

 * 
 * @author mycat
 */
public class Terminate extends PostgresPacket {

}

================================================
FILE: src/main/java/io/mycat/route/MyCATSequnceProcessor.java
================================================
package io.mycat.route;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import io.mycat.MycatServer;
import io.mycat.config.ErrorCode;
import io.mycat.route.parser.druid.DruidSequenceHandler;

public class MyCATSequnceProcessor {
	private static final Logger LOGGER = LoggerFactory.getLogger(MyCATSequnceProcessor.class);
	
	//使用Druid解析器实现sequence处理  @兵临城下
	private static final DruidSequenceHandler sequenceHandler = new DruidSequenceHandler(MycatServer
			.getInstance().getConfig().getSystem().getSequenceHandlerType(),MycatServer.getInstance().getConfig().getSystem().getSequnceHandlerPattern());
	
	private static class InnerMyCATSequnceProcessor{
		private static MyCATSequnceProcessor INSTANCE = new MyCATSequnceProcessor();
	}
	
	public static MyCATSequnceProcessor getInstance(){
		return InnerMyCATSequnceProcessor.INSTANCE;
	}
	
	private MyCATSequnceProcessor() {
	}

	/**
	 *  锁的粒度控制到序列级别.一个序列一把锁.
	 *  如果是 db 方式, 可以 给 mycat_sequence表的 name 列 加索引.可以借助mysql 行级锁 提高并发 
	 * @param pair
	 */
	public void executeSeq(SessionSQLPair pair) {
		try {
			/*// @micmiu 扩展NodeToString实现自定义全局序列号
			NodeToString strHandler = new ExtNodeToString4SEQ(MycatServer
					.getInstance().getConfig().getSystem()
					.getSequenceHandlerType());
			// 如果存在sequence 转化sequence为实际数值
			String charset = pair.session.getSource().getCharset();
			QueryTreeNode ast = SQLParserDelegate.parse(pair.sql,
					charset == null ? "utf-8" : charset);
			String sql = strHandler.toString(ast);
			if (sql.toUpperCase().startsWith("SELECT")) {
				String value=sql.substring("SELECT".length()).trim();
				outRawData(pair.session.getSource(),value);
				return;
			}*/
			
			String charset = pair.session.getSource().getCharset();
			String executeSql = sequenceHandler.getExecuteSql(pair,charset == null ? "utf-8":charset);
			
			pair.session.getSource().routeEndExecuteSQL(executeSql, pair.type,pair.schema);
		} catch (Exception e) {
			LOGGER.error("MyCATSequenceProcessor.executeSeq(SesionSQLPair)",e);
			pair.session.getSource().writeErrMessage(ErrorCode.ER_YES,"mycat sequnce err." + e);
			return;
		}
	}
}


================================================
FILE: src/main/java/io/mycat/route/Procedure.java
================================================
package io.mycat.route;

import com.google.common.base.*;

import java.io.Serializable;
import java.sql.Types;
import java.util.*;

/**
 * Created by magicdoom on 2016/3/24.
 *
 *
 * 1.no return

 ok


 2.simple

 ok
 row
 eof


 3.list


 row
 row
 row
 row
 eof
 ok

 */
public class Procedure implements Serializable
{
    private String originSql;
    private String name;
    private String callSql;
    private String setSql ;
    private String selectSql;
    private Set selectColumns=new LinkedHashSet<>();
    private Set listFields=new LinkedHashSet<>();
    private boolean isResultList=false;

    public boolean isResultList()
    {
        return isResultList;
    }
    public boolean isResultSimpleValue()
    {
        return selectSql!=null&&!isResultList;
    }
    public boolean isResultNothing()
    {
        return selectSql==null&&!isResultList;
    }
    public void setResultList(boolean resultList)
    {
        isResultList = resultList;
    }

    public String toPreCallSql(String dbType)
    {
        StringBuilder sb=new StringBuilder();
        sb.append("{ call ")  ;
        sb.append(this.getName()).append("(") ;
        Collection paramters=    this.getParamterMap().values();
        int j=0;
        for (ProcedureParameter paramter : paramters)
        {

            String name="?";
            String joinStr=  j==this.getParamterMap().size()-1?name:name+"," ;
            sb.append(joinStr);
            j++;
        }
        sb.append(")}")  ;
        return sb.toString();
    }

    public String toChangeCallSql(String dbType)
    {
        StringBuilder sb=new StringBuilder();
        sb.append("call ")  ;
        sb.append(this.getName()).append("(") ;
        Collection paramters=    this.getParamterMap().values();
        int j=0;
        for (ProcedureParameter paramter : paramters)
        {
            Object value=paramter.getValue()!=null&& Types.VARCHAR==paramter.getJdbcType() ?"'"+paramter.getValue()+"'":paramter.getValue();
             String name=paramter.getValue()==null?paramter.getName():String.valueOf(value);
            String joinStr=  j==this.getParamterMap().size()-1?name:name+"," ;
            sb.append(joinStr);
            j++;
        }
        sb.append(")")  ;
        if(isResultSimpleValue())
        {
            sb.append(";select ");
          sb.append(  Joiner.on(",").join(selectColumns)  );
        }
        return sb.toString();
    }

    public Set getListFields()
    {
        return listFields;
    }

    public void setListFields(Set listFields)
    {
        this.listFields = listFields;
    }

    public Set getSelectColumns()
    {
        return selectColumns;
    }

    public String getSetSql()
    {
        return setSql;
    }

    public void setSetSql(String setSql)
    {
        this.setSql = setSql;
    }

    public String getSelectSql()
    {
        return selectSql;
    }

    public void setSelectSql(String selectSql)
    {
        this.selectSql = selectSql;
    }

    private Map paramterMap=new LinkedHashMap<>();

    public String getOriginSql()
    {
        return originSql;
    }

    public void setOriginSql(String originSql)
    {
        this.originSql = originSql;
    }

    public Map getParamterMap()
    {
        return paramterMap;
    }

    public void setParamterMap(Map paramterMap)
    {
        this.paramterMap = paramterMap;
    }

    public String getName()
    {
        return name;
    }

    public void setName(String name)
    {
        this.name = name;
    }

    public String getCallSql()
    {
        return callSql;
    }

    public void setCallSql(String callSql)
    {
        this.callSql = callSql;
    }
}


================================================
FILE: src/main/java/io/mycat/route/ProcedureParameter.java
================================================
package io.mycat.route;

import java.io.Serializable;
import java.sql.Types;

/**
 * Created by magicdoom on 2016/3/24.
 */
public class ProcedureParameter implements Serializable
{
    public static final String IN="in";
    public static final String OUT="out";
    public static final String INOUT="inout";


   private int index;
    private String name;

    //in out inout
    private String parameterType;

    //java.sql.Types
    private int jdbcType= Types.VARCHAR;

    private Object value;


    public Object getValue()
    {
        return value;
    }

    public void setValue(Object value)
    {
        this.value = value;
    }

    public int getIndex()
    {
        return index;
    }

    public void setIndex(int index)
    {
        this.index = index;
    }

    public String getName()
    {
        return name;
    }

    public void setName(String name)
    {
        this.name = name;
    }

    public String getParameterType()
    {
        return parameterType;
    }

    public void setParameterType(String parameterType)
    {
        this.parameterType = parameterType;
    }

    public int getJdbcType()
    {
        return jdbcType;
    }

    public void setJdbcType(int jdbcType)
    {
        this.jdbcType = jdbcType;
    }
}


================================================
FILE: src/main/java/io/mycat/route/RouteCheckRule.java
================================================
package io.mycat.route;

import io.mycat.route.function.PartitionByCRC32PreSlot;
import io.mycat.route.function.PartitionByCRC32PreSlot.Range;

import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;

/**
 *    迁移切换时准备切换阶段需要禁止写操作和读所有分片的sql
 */
public class RouteCheckRule {
    public static ConcurrentMap>> migrateRuleMap=new ConcurrentHashMap<>();

}


================================================
FILE: src/main/java/io/mycat/route/RouteResultset.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese
 * opensource volunteers. you can redistribute it and/or modify it under the
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Any questions about this component can be directed to it's project Web address
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route;

import com.alibaba.druid.sql.ast.SQLStatement;
import io.mycat.config.model.SchemaConfig;
import io.mycat.route.parser.util.PageSQLUtil;
import io.mycat.sqlengine.mpp.HavingCols;
import io.mycat.util.FormatUtil;

import java.io.Serializable;
import java.util.*;
import java.util.Map.Entry;

/**
 * @author mycat
 */
public final class RouteResultset implements Serializable {
    private String statement; // 原始语句
    private final int sqlType;
    private RouteResultsetNode[] nodes; // 路由结果节点
    private Set subTables;
    private SQLStatement sqlStatement;


    private int limitStart;
    private boolean cacheAble;
    // used to store table's ID->datanodes cache
    // format is table.primaryKey
    private String primaryKey;
    // limit output total
    private int limitSize;
    private SQLMerge sqlMerge;

    private boolean callStatement = false; // 处理call关键字

    // 是否为全局表，只有在insert、update、delete、ddl里会判断并修改。默认不是全局表，用于修正全局表修改数据的反馈。
    private boolean globalTableFlag = false;

    //是否完成了路由
    private boolean isFinishedRoute = false;

    //是否自动提交，此属性主要用于记录ServerConnection上的autocommit状态
    private boolean autocommit = true;

    private boolean isLoadData=false;

    //是否可以在从库运行,此属性主要供RouteResultsetNode获取
    private Boolean canRunInReadDB;

    // 强制走 master，可以通过 RouteResultset的属性canRunInReadDB=false
    // 传给 RouteResultsetNode 来实现，但是 强制走 slave需要增加一个属性来实现:
    private Boolean runOnSlave = null;	// 默认null表示不施加影响

    //key=dataNode    value=slot
    private Map   dataNodeSlotMap=new HashMap<>();

    private boolean selectForUpdate;
    private boolean autoIncrement;
    private Map> subTableMaps;

    public boolean isSelectForUpdate() {
        return selectForUpdate;
    }

    public void setSelectForUpdate(boolean selectForUpdate) {
        this.selectForUpdate = selectForUpdate;
    }


    private List tables;

    public List getTables() {
        return tables;
    }

    public void setTables(List tables) {
        this.tables = tables;
    }

    public Map getDataNodeSlotMap() {
        return dataNodeSlotMap;
    }

    public void setDataNodeSlotMap(Map dataNodeSlotMap) {
        this.dataNodeSlotMap = dataNodeSlotMap;
    }

    public Boolean getRunOnSlave() {
        return runOnSlave;
    }
    public String getRunOnSlaveDebugInfo() {
        return runOnSlave == null?"default":Boolean.toString(runOnSlave);
    }
    public void setRunOnSlave(Boolean runOnSlave) {
        this.runOnSlave = runOnSlave;
    }
    private Procedure procedure;

    public Procedure getProcedure()
    {
        return procedure;
    }

    public void setProcedure(Procedure procedure)
    {
        this.procedure = procedure;
    }

    public boolean isLoadData()
    {
        return isLoadData;
    }

    public void setLoadData(boolean isLoadData)
    {
        this.isLoadData = isLoadData;
    }

    public boolean isFinishedRoute() {
        return isFinishedRoute;
    }

    public void setFinishedRoute(boolean isFinishedRoute) {
        this.isFinishedRoute = isFinishedRoute;
    }

    public boolean isGlobalTable() {
        return globalTableFlag;
    }

    public void setGlobalTable(boolean globalTableFlag) {
        this.globalTableFlag = globalTableFlag;
    }

    public RouteResultset(String stmt, int sqlType) {
        this.statement = stmt;
        this.limitSize = -1;
        this.sqlType = sqlType;
    }

    public void resetNodes() {
        if (nodes != null) {
            for (RouteResultsetNode node : nodes) {
                node.resetStatement();
            }
        }
    }

    public void copyLimitToNodes() {

        if(nodes!=null)
        {
            for (RouteResultsetNode node : nodes)
            {
                if(node.getLimitSize()==-1&&node.getLimitStart()==0)
                {
                    node.setLimitStart(limitStart);
                    node.setLimitSize(limitSize);
                }
            }

        }
    }


    public SQLMerge getSqlMerge() {
        return sqlMerge;
    }

    public boolean isCacheAble() {
        return cacheAble;
    }

    public void setCacheAble(boolean cacheAble) {
        this.cacheAble = cacheAble;
    }

    public boolean needMerge() {
        return limitSize > 0 || sqlMerge != null;
    }

    public int getSqlType() {
        return sqlType;
    }

    public boolean isHasAggrColumn() {
        return (sqlMerge != null) && sqlMerge.isHasAggrColumn();
    }

    public int getLimitStart() {
        return limitStart;
    }

    public String[] getGroupByCols() {
        return (sqlMerge != null) ? sqlMerge.getGroupByCols() : null;
    }

    private SQLMerge createSQLMergeIfNull() {
        if (sqlMerge == null) {
            sqlMerge = new SQLMerge();
        }
        return sqlMerge;
    }

    public Map getMergeCols() {
        return (sqlMerge != null) ? sqlMerge.getMergeCols() : null;
    }

    public void setLimitStart(int limitStart) {
        this.limitStart = limitStart;
    }

    public String getPrimaryKey() {
        return primaryKey;
    }

    public boolean hasPrimaryKeyToCache() {
        return primaryKey != null;
    }

    public void setPrimaryKey(String primaryKey) {
        if (!primaryKey.contains(".")) {
            throw new java.lang.IllegalArgumentException(
                "must be table.primarykey fomat :" + primaryKey);
        }
        this.primaryKey = primaryKey;
    }

    /**
     * return primary key items ,first is table name ,seconds is primary key
     *
     * @return
     */
    public String[] getPrimaryKeyItems() {
        return primaryKey.split("\\.");
    }

    public void setOrderByCols(LinkedHashMap orderByCols) {
        if (orderByCols != null && !orderByCols.isEmpty()) {
            createSQLMergeIfNull().setOrderByCols(orderByCols);
        }
    }

    public void setHasAggrColumn(boolean hasAggrColumn) {
        if (hasAggrColumn) {
            createSQLMergeIfNull().setHasAggrColumn(true);
        }
    }

    public void setGroupByCols(String[] groupByCols) {
        if (groupByCols != null && groupByCols.length > 0) {
            createSQLMergeIfNull().setGroupByCols(groupByCols);
        }
    }

    public void setMergeCols(Map mergeCols) {
        if (mergeCols != null && !mergeCols.isEmpty()) {
            createSQLMergeIfNull().setMergeCols(mergeCols);
        }

    }

    public LinkedHashMap getOrderByCols() {
        return (sqlMerge != null) ? sqlMerge.getOrderByCols() : null;

    }

    public String getStatement() {
        return statement;
    }

    public RouteResultsetNode[] getNodes() {
        return nodes;
    }

    public void setNodes(RouteResultsetNode[] nodes) {
        if(nodes!=null)
        {
            int nodeSize=nodes.length;
            for (RouteResultsetNode node : nodes)
            {
                node.setTotalNodeSize(nodeSize);
            }

        }
        this.nodes = nodes;
    }

    /**
     * @return -1 if no limit
     */
    public int getLimitSize() {
        return limitSize;
    }

    public void setLimitSize(int limitSize) {
        this.limitSize = limitSize;
    }

    public void setStatement(String statement) {
        this.statement = statement;
    }

    public boolean isCallStatement() {
        return callStatement;
    }

    public void setCallStatement(boolean callStatement) {
        this.callStatement = callStatement;
        if(nodes!=null)
        {
            for (RouteResultsetNode node : nodes)
            {
                node.setCallStatement(callStatement);
            }

        }
    }

    public void changeNodeSqlAfterAddLimit(SchemaConfig schemaConfig, String sourceDbType, String sql, int offset, int count, boolean isNeedConvert) {
        if (nodes != null)
        {

            Map dataNodeDbTypeMap = schemaConfig.getDataNodeDbTypeMap();
            Map sqlMapCache = new HashMap<>();
            for (RouteResultsetNode node : nodes)
            {
                String dbType = dataNodeDbTypeMap.get(node.getName());
                if (dbType.equalsIgnoreCase("mysql"))
                {
                    node.setStatement(sql);   //mysql之前已经加好limit
                } else if (sqlMapCache.containsKey(dbType))
                {
                    node.setStatement(sqlMapCache.get(dbType));
                } else if(isNeedConvert)
                {
                    String nativeSql = PageSQLUtil.convertLimitToNativePageSql(dbType, sql, offset, count);
                    sqlMapCache.put(dbType, nativeSql);
                    node.setStatement(nativeSql);
                }  else {
                    node.setStatement(sql);
                }

                node.setLimitStart(offset);
                node.setLimitSize(count);
            }


        }
    }

    public boolean isAutocommit() {
        return autocommit;
    }

    public void setAutocommit(boolean autocommit) {
        this.autocommit = autocommit;
    }

    public Boolean getCanRunInReadDB() {
        return canRunInReadDB;
    }

    public void setCanRunInReadDB(Boolean canRunInReadDB) {
        this.canRunInReadDB = canRunInReadDB;
    }

    public HavingCols getHavingCols() {
        return (sqlMerge != null) ? sqlMerge.getHavingCols() : null;
    }

    public void setSubTables(Set subTables) {
        this.subTables = subTables;
    }

    public void setHavings(HavingCols havings) {
        if (havings != null) {
            createSQLMergeIfNull().setHavingCols(havings);
        }
    }

    // Added by winbill, 20160314, for having clause, Begin ==>
    public void setHavingColsName(Object[] names) {
        if (names != null && names.length > 0) {
            createSQLMergeIfNull().setHavingColsName(names);
        }
    }
    // Added by winbill, 20160314, for having clause, End  <==

    public SQLStatement getSqlStatement() {
        return this.sqlStatement;
    }

    public void setSqlStatement(SQLStatement sqlStatement) {
        this.sqlStatement = sqlStatement;
    }

    public Set getSubTables() {
        return this.subTables;
    }

    public boolean isDistTable(){
        if(this.getSubTables()!=null && !this.getSubTables().isEmpty() ){
            return true;
        }
        return false;
    }

    @Override
    public String toString() {
        StringBuilder s = new StringBuilder();
        s.append(statement).append(", route={");
        if (nodes != null) {
            for (int i = 0; i < nodes.length; ++i) {
                s.append("\n ").append(FormatUtil.format(i + 1, 3));
                s.append(" -> ").append(nodes[i]);
            }
        }
        s.append("\n}");
        return s.toString();
    }

    public void setAutoIncrement(boolean b) {
        autoIncrement = b;
    }

    public boolean getAutoIncrement() {
        return autoIncrement;
    }

    public Map> getSubTableMaps() {
        return subTableMaps;
    }

    public void setSubTableMaps(Map> subTableMaps) {
        this.subTableMaps = subTableMaps;
    }

	/**
	 * 合并路由节点相同的节点
	 */
	public void mergeSameNode() {
		Map mapNodes = new HashMap<>(64);
		for (RouteResultsetNode node : nodes) {
			if (mapNodes.containsKey(node.getName())) {
				// merge node
				RouteResultsetNode tmpNode = mapNodes.get(node.getName());
				tmpNode.setStatement(tmpNode.getStatement() + ";" + node.getStatement());
			} else {
				mapNodes.put(node.getName(), node);
			}
		}

		RouteResultsetNode[] newNodes = new RouteResultsetNode[mapNodes.size()];
		int i = 0;
		for (Entry entry : mapNodes.entrySet()) {
			newNodes[i++] = entry.getValue();
		}
		this.setNodes(newNodes);
	}

}


================================================
FILE: src/main/java/io/mycat/route/RouteResultsetNode.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese
 * opensource volunteers. you can redistribute it and/or modify it under the
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Any questions about this component can be directed to it's project Web address
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route;

import java.io.Serializable;
import java.util.Map;

import com.google.common.base.Strings;

import io.mycat.server.parser.ServerParse;
import io.mycat.sqlengine.mpp.LoadData;

/**
 * @author mycat
 */
public final class RouteResultsetNode implements Serializable , Comparable {
	/**
	 *
	 */
	private static final long serialVersionUID = 1L;
	private final String name; // 数据节点名称
	private String statement; // 执行的语句
	private final String srcStatement;
	private final int sqlType;
	private volatile boolean canRunInReadDB;
	private final boolean hasBlanceFlag;
    private boolean callStatement = false; // 处理call关键字
	private int limitStart;
	private int limitSize;
	private int totalNodeSize =0; //方便后续jdbc批量获取扩展
   private Procedure procedure;
	private LoadData loadData;
	private RouteResultset source;

	// 强制走 master，可以通过 RouteResultset的属性canRunInReadDB(false)
	// 传给 RouteResultsetNode 来实现，但是 强制走 slave需要增加一个属性来实现:
	private Boolean runOnSlave = null;	// 默认null表示不施加影响, true走slave,false走master

	private String subTableName; // 分表的表名
	private Map subTableNames;//分表的表名集合

	//迁移算法用     -2代表不是slot分片  ，-1代表扫描所有分片
	private int slot=-2;

	public RouteResultsetNode(String name, int sqlType, String srcStatement) {
		this.name = name;
		limitStart=0;
		this.limitSize = -1;
		this.sqlType = sqlType;
		this.srcStatement = srcStatement;
		this.statement = srcStatement;
		canRunInReadDB = (sqlType == ServerParse.SELECT || sqlType == ServerParse.SHOW);
		hasBlanceFlag = (statement != null)
				&& statement.startsWith("/*balance*/");
	}

	public Map getSubTableNames() {
      return subTableNames;
    }

    public void setSubTableNames(Map subTableNames) {
      this.subTableNames = subTableNames;
    }

    public Boolean getRunOnSlave() {
		return runOnSlave;
	}
	public String getRunOnSlaveDebugInfo() {
		return runOnSlave == null?" default ":Boolean.toString(runOnSlave);
	}
	public boolean isUpdateSql() {
		int type=sqlType;
		return ServerParse.INSERT==type||ServerParse.UPDATE==type||ServerParse.DELETE==type||ServerParse.DDL==type;
	}
	public void setRunOnSlave(Boolean runOnSlave) {
		this.runOnSlave = runOnSlave;
	}
	  private Map hintMap;

    public Map getHintMap()
    {
        return hintMap;
    }

    public void setHintMap(Map hintMap)
    {
        this.hintMap = hintMap;
    }

	public void setStatement(String statement) {
		this.statement = statement;
	}

	public void setCanRunInReadDB(boolean canRunInReadDB) {
		this.canRunInReadDB = canRunInReadDB;
	}

	public boolean getCanRunInReadDB() {
		return this.canRunInReadDB;
	}

	public void resetStatement() {
		this.statement = srcStatement;
	}

	/**
	 * 这里的逻辑是为了优化，实现：非业务sql可以在负载均衡走slave的效果。因为业务sql一般是非自动提交，
	 * 而非业务sql一般默认是自动提交，比如mysql client，还有SQLJob, heartbeat都可以使用
	 * 了Leader-us优化的query函数，该函数实现为自动提交；
	 *
	 * 在非自动提交的情况下(有事物)，除非使用了  balance 注解的情况下，才可以走slave.
	 *
	 * 当然还有一个大前提，必须是 select 或者 show 语句(canRunInReadDB=true)
	 * @param autocommit
	 * @return
	 */
	public boolean canRunnINReadDB(boolean autocommit) {
		return canRunInReadDB && ( autocommit || (!autocommit && hasBlanceFlag) );
	}

//	public boolean canRunnINReadDB(boolean autocommit) {
//		return canRunInReadDB && autocommit && !hasBlanceFlag
//			|| canRunInReadDB && !autocommit && hasBlanceFlag;
//	}
  public Procedure getProcedure()
    {
        return procedure;
    }

	public int getSlot() {
		return slot;
	}

	public void setSlot(int slot) {
		this.slot = slot;
	}

	public void setProcedure(Procedure procedure)
    {
        this.procedure = procedure;
    }

    public boolean isCallStatement()
    {
        return callStatement;
    }

    public void setCallStatement(boolean callStatement)
    {
        this.callStatement = callStatement;
    }
	public String getName() {
		return name;
	}

	public int getSqlType() {
		return sqlType;
	}

	public String getStatement() {
		return statement;
	}

	public int getLimitStart()
	{
		return limitStart;
	}

	public void setLimitStart(int limitStart)
	{
		this.limitStart = limitStart;
	}

	public int getLimitSize()
	{
		return limitSize;
	}

	public void setLimitSize(int limitSize)
	{
		this.limitSize = limitSize;
	}

	public int getTotalNodeSize()
	{
		return totalNodeSize;
	}

	public void setTotalNodeSize(int totalNodeSize)
	{
		this.totalNodeSize = totalNodeSize;
	}

	public LoadData getLoadData()
	{
		return loadData;
	}

	public void setLoadData(LoadData loadData)
	{
		this.loadData = loadData;
	}

	@Override
	public int hashCode() {
		return name.hashCode();
	}

	@Override
	public boolean equals(Object obj) {
		if (this == obj) {
			return true;
		}
		if (obj instanceof RouteResultsetNode) {
			RouteResultsetNode rrn = (RouteResultsetNode) obj;
			if(subTableName!=null){
				if (equals(name, rrn.getName()) && equals(subTableName, rrn.getSubTableName())) {
					return true;
				}
			}else{
				if (equals(name, rrn.getName())) {
					return true;
				}
			}
		}
		return false;
	}

	@Override
	public String toString() {
		StringBuilder s = new StringBuilder();
		s.append(name);
		s.append('{').append(statement).append('}');
		return s.toString();
	}

	private static boolean equals(String str1, String str2) {
		if (str1 == null) {
			return str2 == null;
		}
		return str1.equals(str2);
	}

	public String getSubTableName() {
		return this.subTableName;
	}

	public void setSubTableName(String subTableName) {
		this.subTableName = subTableName;
	}

	public boolean isModifySQL() {
		return !canRunInReadDB;
	}

    /**
     * 非DDL的修改语句，如INSERT,DELETE,UPDATE
     * @return
     */
    public boolean isModifySQLExceptDDL() {
        boolean isDDL = (sqlType == ServerParse.DDL);
        return !canRunInReadDB && !isDDL;
    }

	public boolean isDisctTable() {
		if(subTableName!=null && !subTableName.equals("")){
			return true;
		};
		return false;
	}


	@Override
	public int compareTo(RouteResultsetNode obj) {
		if(obj == null) {
			return 1;
		}
		if(this.name == null) {
			return -1;
		}
		if(obj.name == null) {
			return 1;
		}
		int c = this.name.compareTo(obj.name);
		if(!this.isDisctTable()||obj.subTableName == null){
			return c;
		}else{
			if(c==0){
			    if (Strings.isNullOrEmpty(obj.subTableName)) {
			        return 1;
			    }
				return this.subTableName.compareTo(obj.subTableName);
			}
			return c;
		}
	}

	public boolean isHasBlanceFlag() {
		return hasBlanceFlag;
	}

	public RouteResultset getSource() {
		return source;
	}

	public void setSource(RouteResultset source) {
		this.source = source;
	}
}


================================================
FILE: src/main/java/io/mycat/route/RouteService.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route;

import java.sql.SQLNonTransientException;
import java.sql.SQLSyntaxErrorException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentMap;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import io.mycat.cache.CachePool;
import io.mycat.cache.CacheService;
import io.mycat.cache.LayerCachePool;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.SystemConfig;
import io.mycat.route.factory.RouteStrategyFactory;
import io.mycat.route.function.PartitionByCRC32PreSlot;
import io.mycat.route.handler.HintHandler;
import io.mycat.route.handler.HintHandlerFactory;
import io.mycat.route.handler.HintSQLHandler;
import io.mycat.server.ServerConnection;
import io.mycat.server.parser.ServerParse;

public class RouteService {
    private static final Logger LOGGER = LoggerFactory
            .getLogger(RouteService.class);
    public static final String MYCAT_HINT_TYPE = "_mycatHintType";
    private final CachePool sqlRouteCache;
	private final LayerCachePool tableId2DataNodeCache;	

	private final String OLD_MYCAT_HINT = "/*!mycat:"; 	// 处理自定义分片注解, 注解格式：/*!mycat: type = value */ sql
	private final String NEW_MYCAT_HINT = "/*#mycat:"; 	// 新的注解格式:/* !mycat: type = value */ sql，oldMycatHint的格式不兼容直连mysql
    private final String HINT_SPLIT = "=";

	public RouteService(CacheService cachService) {
		sqlRouteCache = cachService.getCachePool("SQLRouteCache");
		tableId2DataNodeCache = (LayerCachePool) cachService
				.getCachePool("TableID2DataNodeCache");
	}

	public LayerCachePool getTableId2DataNodeCache() {
		return tableId2DataNodeCache;
	}

	public RouteResultset route(SystemConfig sysconf, SchemaConfig schema,
			int sqlType, String stmt, String charset, ServerConnection sc)
			throws SQLNonTransientException {
		stmt = stmt.trim();
		RouteResultset rrs = null;
		String cacheKey = null;

		/**
		 *  SELECT 类型的SQL, 检测
		 */
		if (sqlType == ServerParse.SELECT) {
			cacheKey = schema.getName() + stmt;			
			rrs = (RouteResultset) sqlRouteCache.get(cacheKey);
			if (rrs != null) {
				checkMigrateRule(schema.getName(),rrs,sqlType);
				return rrs;
			}
		}
        
		/*!mycat: sql = select name from aa */
        /*!mycat: schema = test */
//      boolean isMatchOldHint = stmt.startsWith(OLD_MYCAT_HINT);
//      boolean isMatchNewHint = stmt.startsWith(NEW_MYCAT_HINT);
//		if (isMatchOldHint || isMatchNewHint ) {
		int hintLength = RouteService.isHintSql(stmt);
		if(hintLength != -1){
			int endPos = stmt.indexOf("*/");
			if (endPos > 0) {				
				// 用!mycat:内部的语句来做路由分析
//				int hintLength = isMatchOldHint ? OLD_MYCAT_HINT.length() : NEW_MYCAT_HINT.length();
				String hint = stmt.substring(hintLength, endPos).trim();	
				
                int firstSplitPos = hint.indexOf(HINT_SPLIT);                
                if(firstSplitPos > 0 ){
                    Map hintMap = new HashMap<>(); // parseHint(hint);
                    parseKeyValue(hintMap, hint);
                	String hintType = (String) hintMap.get(MYCAT_HINT_TYPE);
                    String hintSql = (String) hintMap.get(hintType);
                    if( hintSql.length() == 0 ) {
                    	LOGGER.warn("comment int sql must meet :/*!mycat:type=value*/ or /*#mycat:type=value*/ or /*mycat:type=value*/: "+stmt);
                    	throw new SQLSyntaxErrorException("comment int sql must meet :/*!mycat:type=value*/ or /*#mycat:type=value*/ or /*mycat:type=value*/: "+stmt);
                    }
                    String realSQL = stmt.substring(endPos + "*/".length()).trim();

                    HintHandler hintHandler = HintHandlerFactory.getHintHandler(hintType);
                    if( hintHandler != null ) {    

                    	if ( hintHandler instanceof  HintSQLHandler) {                    		
                          	/**
                        	 * 修复 注解SQL的 sqlType 与 实际SQL的 sqlType 不一致问题， 如： hint=SELECT，real=INSERT
                        	 * fixed by zhuam
                        	 */
                    		int hintSqlType = ServerParse.parse( hintSql ) & 0xff;     
                    		rrs = hintHandler.route(sysconf, schema, sqlType, realSQL, charset, sc, tableId2DataNodeCache, hintSql,hintSqlType,hintMap);
                    		
                    	} else {                    		
                    		rrs = hintHandler.route(sysconf, schema, sqlType, realSQL, charset, sc, tableId2DataNodeCache, hintSql,sqlType,hintMap);
                    	}
 
                    }else{
                        LOGGER.warn("TODO , support hint sql type : " + hintType);
                    }
                    
                }else{//fixed by runfriends@126.com
                	LOGGER.warn("comment in sql must meet :/*!mycat:type=value*/ or /*#mycat:type=value*/ or /*mycat:type=value*/: "+stmt);
                	throw new SQLSyntaxErrorException("comment in sql must meet :/*!mcat:type=value*/ or /*#mycat:type=value*/ or /*mycat:type=value*/: "+stmt);
                }
			}
		} else {
			stmt = stmt.trim();
			rrs = RouteStrategyFactory.getRouteStrategy().route(sysconf, schema, sqlType, stmt,
					charset, sc, tableId2DataNodeCache);
		}

		if (rrs != null && sqlType == ServerParse.SELECT && rrs.isCacheAble()) {
			sqlRouteCache.putIfAbsent(cacheKey, rrs);
		}
		checkMigrateRule(schema.getName(),rrs,sqlType);
		return rrs;
	}

	 //数据迁移的切换准备阶段，需要拒绝写操作和所有的跨多节点写操作
		private   void checkMigrateRule(String schemal,RouteResultset rrs,int sqlType ) throws SQLNonTransientException {
		if(rrs!=null&&rrs.getTables()!=null){
			boolean isUpdate=isUpdateSql(sqlType);
			if(!isUpdate)return;
			ConcurrentMap> tableRules= RouteCheckRule.migrateRuleMap.get(schemal.toUpperCase()) ;
			if(tableRules!=null){
			for (String table : rrs.getTables()) {
				List rangeList= tableRules.get(table.toUpperCase()) ;
				if(rangeList!=null&&!rangeList.isEmpty()){
					if(rrs.getNodes().length>1&&isUpdate){
						throw new   SQLNonTransientException ("schema:"+schemal+",table:"+table+",sql:"+rrs.getStatement()+" is not allowed,because table is migrate switching,please wait for a moment");
					}
					for (PartitionByCRC32PreSlot.Range range : rangeList) {
						RouteResultsetNode[] routeResultsetNodes=	rrs.getNodes();
						for (RouteResultsetNode routeResultsetNode : routeResultsetNodes) {
							int slot=routeResultsetNode.getSlot();
							if(isUpdate&&slot>=range.start&&slot<=range.end){
								throw new   SQLNonTransientException ("schema:"+schemal+",table:"+table+",sql:"+rrs.getStatement()+" is not allowed,because table is migrate switching,please wait for a moment");

							}
						}
					}
				}
			}
			}
		}
	}


	private boolean isUpdateSql(int type) {
		return ServerParse.INSERT==type||ServerParse.UPDATE==type||ServerParse.DELETE==type||ServerParse.DDL==type;
	}

	public static int isHintSql(String sql){
		int j = 0;
		int len = sql.length();
		if(sql.charAt(j++) == '/' && sql.charAt(j++) == '*'){
			char c = sql.charAt(j);
			// 过滤掉 空格 和 * 两种字符, 支持： "/** !mycat: */" 和 "/** #mycat: */" 形式的注解
			while(j < len && c != '!' && c != '#' && (c == ' ' || c == '*')){
				c = sql.charAt(++j);
			}
			//注解支持的'!'不被mysql单库兼容，
			//注解支持的'#'不被mybatis兼容
			//注解支持的':'不被hibernate兼容
			//考虑用mycat字符前缀标志Hintsql:"/** mycat: */"
			if(sql.charAt(j)=='m'){
				j--;
			}
			if(j + 6 >= len)	{// prevent the following sql.charAt overflow
				return -1;        // false
			}
			if(sql.charAt(++j) == 'm' && sql.charAt(++j) == 'y' && sql.charAt(++j) == 'c'
				&& sql.charAt(++j) == 'a' && sql.charAt(++j) == 't' && (sql.charAt(++j) == ':' || sql.charAt(j) == '#' || sql.charAt(j) == '-')) {
				return j + 1;    // true，同时返回注解部分的长度
			}
		}
		return -1;	// false
	}
	
    /**
      最原始的代码，不知道要实现什么功能，注释掉。会有#2754 bug
     */
    @Deprecated
	 private   Map parseHint( String sql)
    {
        Map map=new HashMap();
        int y=0;
        int begin=0;
        for(int i=0;i orderByCols;
	private HavingCols havingCols;
	private Object[] havingColsName;			// Added by winbill, 20160314, for having clause
	private Map mergeCols;
	private String[] groupByCols;
	private boolean hasAggrColumn;

	public LinkedHashMap getOrderByCols() {
		return orderByCols;
	}

	public void setOrderByCols(LinkedHashMap orderByCols) {
		this.orderByCols = orderByCols;
	}

	public Map getMergeCols() {
		return mergeCols;
	}

	public void setMergeCols(Map mergeCols) {
		this.mergeCols = mergeCols;
	}

	public String[] getGroupByCols() {
		return groupByCols;
	}

	public void setGroupByCols(String[] groupByCols) {
		this.groupByCols = groupByCols;
	}

	public boolean isHasAggrColumn() {
		return hasAggrColumn;
	}

	public void setHasAggrColumn(boolean hasAggrColumn) {
		this.hasAggrColumn = hasAggrColumn;
	}

	public HavingCols getHavingCols() {
		return havingCols;
	}

	public void setHavingCols(HavingCols havingCols) {
		this.havingCols = havingCols;
	}

	public Object[] getHavingColsName() {
		return havingColsName;
	}

	public void setHavingColsName(Object[] havingColsName) {
		this.havingColsName = havingColsName;
	}
}

================================================
FILE: src/main/java/io/mycat/route/SessionSQLPair.java
================================================
package io.mycat.route;

import io.mycat.config.model.SchemaConfig;
import io.mycat.server.NonBlockingSession;

public class SessionSQLPair {
	public final NonBlockingSession session;
	
	public final SchemaConfig schema;
	public final String sql;
	public final int type;

	public SessionSQLPair(NonBlockingSession session, SchemaConfig schema,
			String sql,int type) {
		super();
		this.session = session;
		this.schema = schema;
		this.sql = sql;
		this.type=type;
	}

}


================================================
FILE: src/main/java/io/mycat/route/factory/RouteStrategyFactory.java
================================================
package io.mycat.route.factory;

import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;

import io.mycat.MycatServer;
import io.mycat.config.model.SystemConfig;
import io.mycat.route.RouteStrategy;
import io.mycat.route.impl.DruidMycatRouteStrategy;

/**
 * 路由策略工厂类
 * @author wang.dw
 *
 */
public class RouteStrategyFactory {
	private static RouteStrategy defaultStrategy = null;
	private static volatile boolean isInit = false;
	private static ConcurrentMap strategyMap = new ConcurrentHashMap();
	public static void init() {
		SystemConfig config = MycatServer.getInstance().getConfig().getSystem();

		String defaultSqlParser = config.getDefaultSqlParser();
		defaultSqlParser = defaultSqlParser == null ? "" : defaultSqlParser;
		//修改为ConcurrentHashMap，避免并发问题
		strategyMap.putIfAbsent("druidparser", new DruidMycatRouteStrategy());

		defaultStrategy = strategyMap.get(defaultSqlParser);
		if(defaultStrategy == null) {
			defaultStrategy = strategyMap.get("druidparser");
			defaultSqlParser = "druidparser";
		}
		config.setDefaultSqlParser(defaultSqlParser);
		isInit = true;
	}
	private RouteStrategyFactory() {
	    
	}

	
	public static RouteStrategy getRouteStrategy() {
//		if(!isInit) {
//			synchronized(RouteStrategyFactory.class){
//				if(!isInit){
//					init();
//				}
//			}
//		}
		return defaultStrategy;
	}
	
	public static RouteStrategy getRouteStrategy(String parserType) {
//		if(!isInit) {
//			synchronized(RouteStrategyFactory.class){
//				if(!isInit){
//					init();
//				}
//			}
//		}
		return strategyMap.get(parserType);
	}
}


================================================
FILE: src/main/java/io/mycat/route/function/AbstractPartitionAlgorithm.java
================================================
package io.mycat.route.function;

import io.mycat.config.model.TableConfig;
import io.mycat.config.model.rule.RuleAlgorithm;

import io.mycat.util.StringUtil;
import java.io.Serializable;
import java.util.List;

/**
 * 路由分片函数抽象类
 * 为了实现一个默认的支持范围分片的函数 calcualteRange
 * 重写它以实现自己的范围路由规则
 * @author lxy
 *
 */
public abstract class AbstractPartitionAlgorithm implements RuleAlgorithm ,Serializable {

	@Override
	public void init() {
	}

	/**
	 * 返回所有被路由到的节点的编号
	 * 返回长度为0的数组表示所有节点都被路由（默认）
	 * 返回null表示没有节点被路由到
	 */
	@Override
	public Integer[] calculateRange(String beginValue, String endValue)  {
		return new Integer[0];
	}
	
	/**
	 * 对于存储数据按顺序存放的字段做范围路由，可以使用这个函数
	 * @param algorithm
	 * @param beginValue
	 * @param endValue
	 * @return
	 */
	public static Integer[] calculateSequenceRange(AbstractPartitionAlgorithm algorithm, String beginValue, String endValue)  {
		Integer begin = 0, end = 0;
		begin = algorithm.calculate(StringUtil.removeBackquote(beginValue));
		end = algorithm.calculate(StringUtil.removeBackquote(endValue));

		if(begin == null || end == null){
			return new Integer[0];
		}
		
		if (end >= begin) {
			int len = end-begin+1;
			Integer [] re = new Integer[len];
			
			for(int i =0;i rule function partition size
	 */
	public final int suitableFor(TableConfig tableConf) {
		int nPartition = getPartitionNum();
		if(nPartition > 0) { // 对于有限制分区数的规则,进行检查
			int dnSize = tableConf.getDataNodes().size();
			boolean  distTable = tableConf.isDistTable();
			List tables = tableConf.getDistTables();
			if(distTable){
				if(tables.size() < nPartition){
					return  -1;
				} else if(dnSize > nPartition) {
					return 1;
				}
			}else{
				if(dnSize < nPartition) {
					return  -1;
				} else if(dnSize > nPartition) {
					return 1;
				}
			}
		}
		return 0;
	}
	
	/**
	 * 返回分区数, 返回-1表示分区数没有限制
	 * @return
	 */
	public int getPartitionNum() {
		return -1; // 表示没有限制
	}
	
}


================================================
FILE: src/main/java/io/mycat/route/function/AutoPartitionByLong.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.function;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.Set;

import io.mycat.config.model.rule.RuleAlgorithm;

/**
 * auto partition by Long ,can be used in auto increment primary key partition
 * 
 * @author wuzhi
 */
public class AutoPartitionByLong extends AbstractPartitionAlgorithm implements RuleAlgorithm{

	private String mapFile;
	private LongRange[] longRongs;
	
	private int defaultNode = -1;
	@Override
	public void init() {

		initialize();
	}

	public void setMapFile(String mapFile) {
		this.mapFile = mapFile;
	}

	@Override
	public Integer calculate(String columnValue)  {
//		columnValue = NumberParseUtil.eliminateQoute(columnValue);
		try {
			long value = Long.parseLong(columnValue);
			Integer rst = null;
			for (LongRange longRang : this.longRongs) {
				if (value <= longRang.valueEnd && value >= longRang.valueStart) {
					return longRang.nodeIndx;
				}
			}
			//数据超过范围，暂时使用配置的默认节点
			if (rst == null && defaultNode >= 0) {
				return defaultNode;
			}
			return rst;
		} catch (NumberFormatException e){
			throw new IllegalArgumentException(new StringBuilder().append("columnValue:").append(columnValue).append(" Please eliminate any quote and non number within it.").toString(),e);
		}
	}
	
	@Override
	public Integer[] calculateRange(String beginValue, String endValue)  {
		return AbstractPartitionAlgorithm.calculateSequenceRange(this, beginValue, endValue);
	}

	@Override
	public int getPartitionNum() {
//		int nPartition = longRongs.length;
		
		/*
		 * fix #1284 这里的统计应该统计Range的nodeIndex的distinct总数
		 */
		Set distNodeIdxSet = new HashSet();
		for(LongRange range : longRongs) {
			distNodeIdxSet.add(range.nodeIndx);
		}
		int nPartition = distNodeIdxSet.size();
		return nPartition;
	}

	private void initialize() {
		BufferedReader in = null;
		try {
			// FileInputStream fin = new FileInputStream(new File(fileMapPath));
			InputStream fin = this.getClass().getClassLoader()
					.getResourceAsStream(mapFile);
			if (fin == null) {
				throw new RuntimeException("can't find class resource file "
						+ mapFile);
			}
			in = new BufferedReader(new InputStreamReader(fin));
			LinkedList longRangeList = new LinkedList();

			for (String line = null; (line = in.readLine()) != null;) {
				line = line.trim();
				if (line.startsWith("#") || line.startsWith("//")) {
					continue;
				}
				int ind = line.indexOf('=');
				if (ind < 0) {
					System.out.println(" warn: bad line int " + mapFile + " :"
							+ line);
					continue;
				}
					String pairs[] = line.substring(0, ind).trim().split("-");
					long longStart = NumberParseUtil.parseLong(pairs[0].trim());
					long longEnd = NumberParseUtil.parseLong(pairs[1].trim());
					int nodeId = Integer.parseInt(line.substring(ind + 1)
							.trim());
					longRangeList
							.add(new LongRange(nodeId, longStart, longEnd));

			}
			longRongs = longRangeList.toArray(new LongRange[longRangeList
					.size()]);
		} catch (Exception e) {
			if (e instanceof RuntimeException) {
				throw (RuntimeException) e;
			} else {
				throw new RuntimeException(e);
			}

		} finally {
			try {
				in.close();
			} catch (Exception e2) {
			}
		}
	}
	
	public int getDefaultNode() {
		return defaultNode;
	}

	public void setDefaultNode(int defaultNode) {
		this.defaultNode = defaultNode;
	}

	static class LongRange {
		public final int nodeIndx;
		public final long valueStart;
		public final long valueEnd;

		public LongRange(int nodeIndx, long valueStart, long valueEnd) {
			super();
			this.nodeIndx = nodeIndx;
			this.valueStart = valueStart;
			this.valueEnd = valueEnd;
		}

	}
}

================================================
FILE: src/main/java/io/mycat/route/function/LatestMonthPartion.java
================================================
package io.mycat.route.function;

/**
 * Latest one month data partions ,only reserve data of latest 31 days and one
 * day is partioned into N slide (splitOneDay), so total datanode is M*N table's
 * partion column must be int type and it's value format should be yyyyMMddHH
 * fomat for example colmn=2014050115 means: 15 clock of april 5 ,2014
 * 
 * @author wuzhih
 * 
 */
public class LatestMonthPartion extends AbstractPartitionAlgorithm {
	private int splitOneDay = 24;
	private int hourSpan;
	private String[] dataNodes;

	public String[] getDataNodes() {
		return dataNodes;
	}

	/**
	 * @param dataNodeExpression
	 */
	public void setSplitOneDay(int split) {
		splitOneDay = split;
		hourSpan = 24 / splitOneDay;
		if (hourSpan * 24 < 24) {
			throw new java.lang.IllegalArgumentException(
					"invalid splitOnDay param:"
							+ splitOneDay
							+ " should be an even number and less or equals than 24");
		}
	}

	@Override
	public Integer calculate(String columnValue)  {
		try {
			int valueLen = columnValue.length();
			int day = Integer.parseInt(columnValue.substring(valueLen - 4,
					valueLen - 2));
			int hour = Integer.parseInt(columnValue.substring(valueLen - 2));
			int dnIndex = (day - 1) * splitOneDay + hour / hourSpan;
			return dnIndex;
		}catch (NumberFormatException e){
			throw new IllegalArgumentException(new StringBuilder().append("columnValue:").append(columnValue).append(" Please check if the format satisfied.").toString(),e);
		}
	}

	public Integer[] calculateRange(String beginValue, String endValue)  {
		return calculateSequenceRange(this,beginValue, endValue);
	}

}


================================================
FILE: src/main/java/io/mycat/route/function/NumberParseUtil.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.function;

public class NumberParseUtil {
	/**
	 * 只去除开头结尾的引号，而且是结对去除，语法不对的话通不过
	 * @param number
	 * @return
     */
	public static String eliminateQoute(String number){
		number = number.trim();
		if(number.contains("\"")){
			if(number.charAt(0)=='\"'){
				number = number.substring(1);
				if(number.charAt(number.length()-1)=='\"'){
					number = number.substring(0,number.length()-1);
				}
			}
		}else if(number.contains("\'")){
			if(number.charAt(0)=='\''){
				number = number.substring(1);
				if(number.charAt(number.length()-1)=='\''){
					number = number.substring(0,number.length()-1);
				}
			}
		}
		return number;
	}

	/**
	 * can parse values like 200M ,200K,200M1(2000001)
	 * 
	 * @param val
	 * @return
	 */
	public static long parseLong(String val) {
		val = val.toUpperCase();
		int indx = val.indexOf("M");

		int plus = 10000;
		if (indx < 0) {
			indx = val.indexOf("K");
			plus = 1000;
		}
		if (indx > 0) {
			String longVal = val.substring(0, indx);

			long theVale = Long.parseLong(longVal) * plus;
			String remain = val.substring(indx + 1);
			if (remain.length() > 0) {
				theVale += Integer.parseInt(remain);
			}
			return theVale;
		} else {
			return Long.parseLong(val);
		}

	}
}

================================================
FILE: src/main/java/io/mycat/route/function/PartitionByCRC32PreSlot.java
================================================
package io.mycat.route.function;

import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.io.Files;
import io.mycat.config.model.SystemConfig;
import io.mycat.config.model.TableConfig;
import io.mycat.config.model.rule.RuleAlgorithm;
import io.mycat.config.model.rule.RuleConfig;
import io.mycat.util.StringUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.nio.charset.Charset;
import java.util.*;

/**
 * 自动迁移御用分片算法，预分slot 102400个，映射到dn上，再conf下会保存映射文件，请不要修改
 *
 * @author nange magicdoom@gmail.com
 */
public class PartitionByCRC32PreSlot extends AbstractPartitionAlgorithm
        implements RuleAlgorithm, TableRuleAware, SlotFunction, ReloadFunction {

    private static final Logger LOGGER = LoggerFactory.getLogger("PartitionByCRC32PreSlot");

    public static final int DEFAULT_SLOTS_NUM = 102400;

    private static final Charset DEFAULT_CHARSET = Charset.forName("UTF-8");
    private Map> rangeMap = new TreeMap<>();

    //slot:index
    private int[] rangeMap2 = new int[DEFAULT_SLOTS_NUM];
    private int slot = -1;

    public Map> getRangeMap() {
        return rangeMap;
    }

    public void saveSlotMapping(Map> rangeMap) {
        this.rangeMap = rangeMap;

        Properties prop = new Properties();
        File file = getFile();
        if (file.exists())
            file.delete();
        for (Map.Entry> integerListEntry : rangeMap.entrySet()) {
            String key = String.valueOf(integerListEntry.getKey());
            List values = new ArrayList<>();
            for (Range range : integerListEntry.getValue()) {
                values.add(range.start + "-" + range.end);
            }
            prop.setProperty(key, Joiner.on(",").join(values));
        }
        try {
            Files.createParentDirs(file);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        try (FileOutputStream out = new FileOutputStream(file)) {
            prop.store(out, "WARNING   !!!Please do not modify or delete this file!!!");
        } catch (IOException e) {
            throw new RuntimeException(e);
        }

    }

    private Properties loadProps(String name, boolean forceNew) {
        Properties prop = new Properties();
        File file = getFile();
        if (file.exists() && forceNew)
            file.delete();
        if (!file.exists()) {
            prop = genarateProperties();
            try {
                Files.createParentDirs(file);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
            try (FileOutputStream out = new FileOutputStream(file)) {
                prop.store(out, "WARNING   !!!Please do not modify or delete this file!!!");
                out.flush();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
            return prop;
        }

        try (FileInputStream filein = new FileInputStream(file)) {
            prop.load(filein);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        return prop;
    }

    private File getFile() {
        return new File(SystemConfig.getHomePath(), "conf" + File.separator + "ruledata" + File.separator + ruleName + ".properties");
    }

    /**
     * 首次构造ruledata,根据table的dataNode数量构建Properties的分片范围
     * @cjw
     * @return
     */
    private Properties genarateProperties() {
        int count = getCount();
        int slotSize = DEFAULT_SLOTS_NUM / count;
        Properties prop = new Properties();
        for (int i = 0; i < count; i++) {
            if (i == count - 1) {
                prop.put(String.valueOf(i), i * slotSize + "-" + (DEFAULT_SLOTS_NUM - 1));
            } else {
                prop.put(String.valueOf(i), i * slotSize + "-" + ((i + 1) * slotSize - 1));
            }
        }

        return prop;
    }

    private Map> convertToMap(Properties p) {
        Map> map = new TreeMap<>();
        for (Object o : p.keySet()) {
            String k = (String) o;
            String v = p.getProperty(k);
            List ranges = Splitter.on(",").omitEmptyStrings().trimResults().splitToList(v);
            List rangeList = new ArrayList<>();
            for (String range : ranges) {
                List vv = Splitter.on("-").omitEmptyStrings().trimResults().splitToList(range);
                if (vv.size() == 2) {
                    Range ran = new Range(Integer.parseInt(vv.get(0)), Integer.parseInt(vv.get(1)));
                    rangeList.add(ran);

                } else if (vv.size() == 1) {
                    Range ran = new Range(Integer.parseInt(vv.get(0)), Integer.parseInt(vv.get(0)));
                    rangeList.add(ran);

                } else {
                    throw new RuntimeException("load crc32slot datafile error:dn=" + k + ",value=" + range);
                }
            }
            map.put(Integer.parseInt(k), rangeList);
        }

        return map;
    }

    @Override
    public void init() {

        super.init();
        if (ruleName != null) {
            Properties p = loadProps(ruleName, false);
            rangeMap = convertToMap(p);
            checkSize();
            hack();
        }
    }

    private void checkSize(){
        if (this.getCount() != this.rangeMap.size()){
            throw new RuntimeException(ruleName + "数量与dataNode数量不符");
        }
    }

    public void reInit() {

        if (ruleName != null) {
            Properties p = loadProps(ruleName, true);
            rangeMap = convertToMap(p);
            checkSize();
            hack();
        }
    }


    private void hack() {
        //todo   优化
        Iterator>> iterator = rangeMap.entrySet().iterator();
        while (iterator.hasNext()) {
            Map.Entry> rangeEntry = iterator.next();
            List range = rangeEntry.getValue();
            for (Range range1 : range) {
                for (int i = range1.start; i <= range1.end; i++) {
                    rangeMap2[i] = rangeEntry.getKey();
                }
            }

        }
    }

    @Override
    public Integer calculate(String columnValue) {
        if (ruleName == null)
            throw new RuntimeException();
        PureJavaCrc32 crc32 = new PureJavaCrc32();
        byte[] bytes = columnValue.getBytes(DEFAULT_CHARSET);
        crc32.update(bytes, 0, bytes.length);
        long x = crc32.getValue();
        int slot = (int) (x % DEFAULT_SLOTS_NUM);
        this.slot = slot;
        return rangeMap2[slot];
//        //todo   优化
//        for (Map.Entry> rangeEntry : rangeMap.entrySet()) {
//            List range = rangeEntry.getValue();
//            for (Range range1 : range) {
//                if (slot >= range1.start && slot <= range1.end) {
//                    this.slot = slot;
//                    return rangeEntry.getKey();
//                }
//            }
//
//        }
//        this.slot = slot;
//        int slotSize = DEFAULT_SLOTS_NUM / count;
//
//        int index = slot / slotSize;
//        if (slotSize * count != DEFAULT_SLOTS_NUM && index > count - 1) {
//            index = (count - 1);
//        }
//        return index;
    }

    @Override
    public int getPartitionNum() {
        int count = getCount();
        return count;
    }

    private static void hashTest() throws IOException {
        PartitionByCRC32PreSlot hash = new PartitionByCRC32PreSlot();
        hash.setRuleName("test");
        RuleConfig rule = new RuleConfig("id", "crc32slot");
        //考虑myccat1.65还有用户使用jdk7,故
        int count = 1024;
        String sb = genDataNodesString(count);
        TableConfig tableConf = new TableConfig("test", "id", true, false, -1, sb,
                null, rule, true, null, false, null, null, null, false);

        hash.setTableConfig(tableConf);
        hash.reInit();
        long start = System.currentTimeMillis();
        int[] bucket = new int[hash.getCount()];

        Map> hashed = new HashMap<>();

        int total = 1000_0000;//数据量
        int c = 0;
        for (int i = 100_0000; i < total + 100_0000; i++) {//假设分片键从100万开始
            c++;
            int h = hash.calculate(StringUtil.removeBackquote(Integer.toString(i)));
            if (h >= count) {
                System.out.println("error:" + h);
            }
            bucket[h]++;
            List list = hashed.get(h);
            if (list == null) {
                list = new ArrayList<>();
                hashed.put(h, list);
            }
            list.add(i);
        }
        System.out.println(c + "   " + total);
        double d = 0;
        c = 0;
        int idx = 0;
        System.out.println("index    bucket   ratio");
        for (int i : bucket) {
            d += i / (double) total;
            c += i;
            System.out.println(idx++ + "  " + i + "   " + (i / (double) total));
        }
        System.out.println(d + "  " + c);

        long used = System.currentTimeMillis() - start;

        System.out.println("tps " + total * 1000.0 / used);
        System.out.println("****************************************************");

    }

    public static String genDataNodesString(int count) {
        StringBuilder sb = new StringBuilder();
        for (int i = 0; i < count; i++) {//1024分片数
            sb.append("db").append(String.valueOf(i)).append(",");
        }
        sb.deleteCharAt(sb.length()-1);//cut last one ,
        return sb.toString();
    }

    public static void main(String[] args) throws IOException {
        hashTest();
    }

    private TableConfig tableConfig;
    private String ruleName;

    private int getCount() {
        if (isIstance()){
            return tableConfig.getDataNodes().size();
        }
        return 0;
    }

    @Override
    public void setTableConfig(TableConfig tableConfig) {
        this.tableConfig = tableConfig;
    }

    @Override
    public void setRuleName(String ruleName) {
        this.ruleName = ruleName;
    }

    @Override
    public TableConfig getTableConfig() {
        return this.tableConfig;
    }

    @Override
    public String getRuleName() {
        return ruleName;
    }

    @Override
    public int slotValue() {
        return slot;
    }

    @Override
    public void reload() {
        init();
    }

    @Override
    public boolean isIstance() {
        return this.tableConfig != null;
    }

    public static class Range implements Serializable {
        public Range(int start, int end) {
            this.start = start;
            this.end = end;
            size = end - start + 1;
        }

        public Range() {
        }

        public int start;
        public int end;

        public int size;
    }
}


================================================
FILE: src/main/java/io/mycat/route/function/PartitionByDate.java
================================================
package io.mycat.route.function;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;

import org.slf4j.Logger; import org.slf4j.LoggerFactory;

import io.mycat.config.model.rule.RuleAlgorithm;

/**
 * 例子 按日期列分区  格式 between操作解析的范例
 *
 * @author lxy
 * @author Sean xiaoyouyy
 */
public class PartitionByDate extends AbstractPartitionAlgorithm implements RuleAlgorithm {
	private static final Logger LOGGER = LoggerFactory.getLogger(PartitionByDate.class);

	private String sBeginDate;
	private String sEndDate;
	private String sPartionDay;
	private String dateFormat;

	private long beginDate;
	private long partionTime;
	private long endDate;
	private int nCount;

	private ThreadLocal formatter;

	private static final long oneDay = 86400000;
	//支持自然日分区属性
	private String sNaturalDay;
	//是否自然日分区
	private boolean bNaturalDay;
	//自然日差额最少是28天
	private static final int naturalLimitDay =28;
	//开启自然日模式
	private static final String  naturalDayOpen ="1";

	private String oldsPartionDay;
	@Override
	public void init() {
		try {
			//Support  Natural Day
			if(naturalDayOpen.equals(sNaturalDay)){
				bNaturalDay =true;
				oldsPartionDay=sPartionDay;
				sPartionDay="1";
			}
			if(sBeginDate!=null&&!sBeginDate.equals("")) {
				partionTime = Integer.parseInt(sPartionDay) * oneDay;
				beginDate = new SimpleDateFormat(dateFormat).parse(sBeginDate).getTime();
			}
			if(sEndDate!=null&&!sEndDate.equals("")&&beginDate>0){
				endDate = new SimpleDateFormat(dateFormat).parse(sEndDate).getTime();
				nCount = (int) ((endDate - beginDate) / partionTime) + 1;
				if(bNaturalDay&&nCount() {
				@Override
				protected SimpleDateFormat initialValue() {
					return new SimpleDateFormat(dateFormat);
				}
			};
		} catch (ParseException e) {
			throw new java.lang.IllegalArgumentException(e);
		}
	}

	@Override
	public Integer calculate(String columnValue)  {
		try {
			int targetPartition ;
			if(bNaturalDay){
				Calendar curTime = Calendar.getInstance();
				curTime.setTime(formatter.get().parse(columnValue));
				targetPartition = curTime.get(Calendar.DAY_OF_MONTH);
				return  targetPartition-1;
			}
			long targetTime = formatter.get().parse(columnValue).getTime();
			targetPartition = (int) ((targetTime - beginDate) / partionTime);
			if(targetTime>endDate && nCount!=0) {
				targetPartition = targetPartition % nCount;
			}
			return targetPartition;

		} catch (ParseException e) {
			throw new IllegalArgumentException(new StringBuilder().append("columnValue:").append(columnValue).append(" Please check if the format satisfied.").toString(),e);
		}
	}

	@Override
	public Integer[] calculateRange(String beginValue, String endValue)  {
		SimpleDateFormat format = new SimpleDateFormat(this.dateFormat);
		try {
			Date beginDate = format.parse(beginValue);
			Date endDate = format.parse(endValue);
			Calendar cal = Calendar.getInstance();
			List list = new ArrayList();
			while(beginDate.getTime() <= endDate.getTime()){
				Integer nodeValue = this.calculate(format.format(beginDate));
				if(Collections.frequency(list, nodeValue) < 1) list.add(nodeValue);
				cal.setTime(beginDate);
				cal.add(Calendar.DATE, 1);
				beginDate = cal.getTime();
			}

			Integer[] nodeArray = new Integer[list.size()];
			for (int i=0;i 0 ? count : -1;
	}

	public void setsBeginDate(String sBeginDate) {
		this.sBeginDate = sBeginDate;
	}

	public void setsPartionDay(String sPartionDay) {
		this.sPartionDay = sPartionDay;
	}

	public void setDateFormat(String dateFormat) {
		this.dateFormat = dateFormat;
	}
	public String getsEndDate() {
		return this.sEndDate;
	}
	public void setsEndDate(String sEndDate) {
		this.sEndDate = sEndDate;
	}

	public String getsNaturalDay() {
		return sNaturalDay;
	}

	public void setsNaturalDay(String sNaturalDay) {
		this.sNaturalDay = sNaturalDay;
	}
}

================================================
FILE: src/main/java/io/mycat/route/function/PartitionByFileMap.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.function;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import io.mycat.config.model.rule.RuleAlgorithm;

/**
 * 
 * @author mycat
 */
public class PartitionByFileMap extends AbstractPartitionAlgorithm implements RuleAlgorithm {

	private String mapFile;
	private Map app2Partition;
	/**
	 * Map app2Partition中key值的类型：默认值为0，0表示Integer，非零表示String
	 */
	private int type;
	
	/**
	 * 默认节点在map中的key
	 */
	private static final String DEFAULT_NODE = "DEFAULT_NODE";
	
	/**
	 * 默认节点:小于0表示不设置默认节点，大于等于0表示设置默认节点
	 * 
	 * 默认节点的作用：枚举分片时，如果碰到不识别的枚举值，就让它路由到默认节点
	 *                如果不配置默认节点（defaultNode值小于0表示不配置默认节点），碰到
	 *                不识别的枚举值就会报错，
	 *                like this：can't find datanode for sharding column:column_name val:ffffffff    
	 */
	private int defaultNode = -1;

	@Override
	public void init() {

		initialize();
	}

	public void setMapFile(String mapFile) {
		this.mapFile = mapFile;
	}
	
	public void setType(int type) {
		this.type = type;
	}

	public void setDefaultNode(int defaultNode) {
		this.defaultNode = defaultNode;
	}

	@Override
	public Integer calculate(String columnValue)  {
		try {
			Object value = columnValue;
			if (type == 0) {
				value = Integer.valueOf(columnValue);
			}
			Integer rst = null;
			Integer pid = app2Partition.get(value);
			if (pid != null) {
				rst = pid;
			} else {
				rst = app2Partition.get(DEFAULT_NODE);
			}
			return rst;
		} catch (NumberFormatException e){
			throw new IllegalArgumentException(new StringBuilder().append("columnValue:").append(columnValue).append(" Please check if the format satisfied.").toString(),e);
		}
	}
	
	@Override
	public int getPartitionNum() {
		Set set = new HashSet(app2Partition.values());
		int count = set.size();
		return count;
	}

	private void initialize() {
		BufferedReader in = null;
		try {
			// FileInputStream fin = new FileInputStream(new File(fileMapPath));
			InputStream fin = this.getClass().getClassLoader()
					.getResourceAsStream(mapFile);
			if (fin == null) {
				throw new RuntimeException("can't find class resource file "
						+ mapFile);
			}
			in = new BufferedReader(new InputStreamReader(fin));
			
			app2Partition = new HashMap();
			
			for (String line = null; (line = in.readLine()) != null;) {
				line = line.trim();
				if (line.startsWith("#") || line.startsWith("//")) {
					continue;
				}
				int ind = line.indexOf('=');
				if (ind < 0) {
					continue;
				}
				try {
					String key = line.substring(0, ind).trim();
					int pid = Integer.parseInt(line.substring(ind + 1).trim());
					if(type == 0) {
						app2Partition.put(Integer.parseInt(key), pid);
					} else {
						app2Partition.put(key, pid);
					}
				} catch (Exception e) {
				}
			}
			//设置默认节点
			if(defaultNode >= 0) {
				app2Partition.put(DEFAULT_NODE, defaultNode);
			}
		} catch (Exception e) {
			if (e instanceof RuntimeException) {
				throw (RuntimeException) e;
			} else {
				throw new RuntimeException(e);
			}

		} finally {
			try {
				in.close();
			} catch (Exception e2) {
			}
		}
	}
}

================================================
FILE: src/main/java/io/mycat/route/function/PartitionByHashMod.java
================================================
package io.mycat.route.function;

import io.mycat.config.model.rule.RuleAlgorithm;

import java.math.BigInteger;

/**
 * 哈希值取模
 * 根据分片列的哈希值对分片个数取模，哈希算法为Wang/Jenkins
 * 用法和简单取模相似，规定分片个数和分片列即可。
 *
 * @author Hash Zhang
 */
public class PartitionByHashMod extends AbstractPartitionAlgorithm implements RuleAlgorithm {
    private boolean watch = false;
    private int count;

    public void setCount(int count) {
        this.count = count;
        if ((count & (count - 1)) == 0) {
            watch = true;
        }
    }

    /**
     * Using Wang/Jenkins Hash
     *
     * @param key
     * @return hash value
     */
    protected int hash(int key) {
        key = (~key) + (key << 21); // key = (key << 21) - key - 1;
        key = key ^ (key >> 24);
        key = (key + (key << 3)) + (key << 8); // key * 265
        key = key ^ (key >> 14);
        key = (key + (key << 2)) + (key << 4); // key * 21
        key = key ^ (key >> 28);
        key = key + (key << 31);
        return key;
    }

    @Override
    public Integer calculate(String columnValue) {
//        columnValue = columnValue.replace("\'", " ");
//        columnValue = columnValue.trim();
        BigInteger bigNum = new BigInteger(hash(columnValue.hashCode()) + "").abs();
        // if count==2^n, then m%count == m&(count-1)
        if (watch) {
            return bigNum.intValue() & (count - 1);
        }
        return (bigNum.mod(BigInteger.valueOf(count))).intValue();
    }

    @Override
    public void init() {
        super.init();
    }

	@Override
	public int getPartitionNum() {
		int count = this.count;
		return count;
	}

}


================================================
FILE: src/main/java/io/mycat/route/function/PartitionByHotDate.java
================================================
package io.mycat.route.function;

import io.mycat.util.StringUtil;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import io.mycat.config.model.rule.RuleAlgorithm;

/**
 * 根据日期查询日志数据 冷热数据分布 ，最近n个月的到实时交易库查询，超过n个月的按照m天分片
 * 
 * @author sw
 * 
 * 
      
        create_time
        sharding-by-hotdate
      
     

    yyyy-MM-dd
    10
    30
  
 */
public class PartitionByHotDate extends AbstractPartitionAlgorithm implements RuleAlgorithm {
	private static final Logger LOGGER = LoggerFactory.getLogger(PartitionByHotDate.class);

	private String dateFormat;
	private String sLastDay;
	private String sPartionDay;

	private long sLastTime;
	private long partionTime;
	private ThreadLocal formatter;
	
	private long beginDate;

	private static final long oneDay = 86400000;

	@Override
	public void init() {
		try {
			formatter = new ThreadLocal() {
				@Override
				protected SimpleDateFormat initialValue() {
					return new SimpleDateFormat(dateFormat);
				}
			};
			sLastTime = Integer.valueOf(sLastDay);
            partionTime = Integer.parseInt(sPartionDay) * oneDay;
		} catch (Exception e) {
			throw new java.lang.IllegalArgumentException(e);
		}
	}

	@Override
	public Integer calculate(String columnValue)  {
		Integer targetPartition = -1;
		try {
			long targetTime = formatter.get().parse(columnValue).getTime();
			Calendar now = Calendar.getInstance();
			long nowTime = now.getTimeInMillis();
			
			beginDate = nowTime - sLastTime * oneDay;
			
			long diffDays = (nowTime - targetTime) / (1000 * 60 * 60 * 24) + 1;
			if(diffDays-sLastTime <= 0 || diffDays<0 ){
				targetPartition = 0;
			}else{
				targetPartition = (int) ((beginDate - targetTime) / partionTime) + 1;
			}
			
		    LOGGER.debug("PartitionByHotDate calculate for " + columnValue + " return " + targetPartition);
			return targetPartition;
		} catch (ParseException e) {
			throw new IllegalArgumentException(new StringBuilder().append("columnValue:").append(columnValue).append(" Please check if the format satisfied.").toString(),e);
		}
	}

	@Override
	public Integer[] calculateRange(String beginValue, String endValue)  {
		Integer[] targetPartition = null;
		try {
			long startTime = formatter.get().parse(beginValue).getTime();
			long endTime = formatter.get().parse(endValue).getTime();
			Calendar now = Calendar.getInstance();
			long nowTime = now.getTimeInMillis();
			
			long limitDate = nowTime - sLastTime * oneDay;
			long diffDays = (nowTime - startTime) / (1000 * 60 * 60 * 24) + 1;
			if(diffDays-sLastTime <= 0 || diffDays<0 ){
				Integer [] re = new Integer[1];
				re[0] = 0;
				targetPartition = re ;
			}else{
				Integer [] re = null;
				Integer begin = 0, end = 0;
				end = this.calculate(StringUtil.removeBackquote(beginValue));
				boolean hasLimit = false;
				if(endTime-limitDate > 0){
					endTime = limitDate;
					hasLimit = true;
				}
				begin = this.calculate(StringUtil.removeBackquote(formatter.get().format(endTime)));
				if(begin == null || end == null){
					return re;
				}
				if (end >= begin) {
					int len = end-begin+1;
					if(hasLimit){
						re = new Integer[len+1];
						re[0] = 0;
						for(int i =0;i= 1.8, just use Long.parseUnsignedLong("2862933555777941757") instead.
    private static final long CONSTANT = Long.parseLong("286293355577794175", 10) * 10 + 7;

    private int totalBuckets;

    @Override
    public Integer calculate(String columnValue) {
        return jumpConsistentHash(columnValue.hashCode(), totalBuckets);
    }
    
	@Override
	public int getPartitionNum() {
		int nPartition = this.totalBuckets;
		return nPartition;
	}

	public static int jumpConsistentHash(final long key, final int buckets) {
        checkBuckets(buckets);
        long k = key;
        long b = -1;
        long j = 0;

        while (j < buckets) {
            b = j;
            k = k * CONSTANT + 1L;

            j = (long) ((b + 1L) * (JUMP / toDouble((k >>> 33) + 1L)));
        }
        return (int) b;
    }

    private static void checkBuckets(final int buckets) {
        if (buckets < 0) {
            throw new IllegalArgumentException("Buckets cannot be less than 0");
        }
    }

    private static double toDouble(final long n) {
        double d = n & UNSIGNED_MASK;
        if (n < 0) {
            d += 0x1.0p63;
        }
        return d;
    }

    public void setTotalBuckets(int totalBuckets) {
        this.totalBuckets = totalBuckets;
    }

    public int getTotalBuckets() {
        return totalBuckets;
    }
}


================================================
FILE: src/main/java/io/mycat/route/function/PartitionByLong.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.function;

import io.mycat.config.model.rule.RuleAlgorithm;
import io.mycat.route.util.PartitionUtil;

public final class PartitionByLong extends AbstractPartitionAlgorithm implements RuleAlgorithm {
	protected int[] count;
	protected int[] length;
	protected PartitionUtil partitionUtil;

	private static int[] toIntArray(String string) {
		String[] strs = io.mycat.util.SplitUtil.split(string, ',', true);
		int[] ints = new int[strs.length];
		for (int i = 0; i < strs.length; ++i) {
			ints[i] = Integer.parseInt(strs[i]);
		}
		return ints;
	}

	public void setPartitionCount(String partitionCount) {
		this.count = toIntArray(partitionCount);
	}

	public void setPartitionLength(String partitionLength) {
		this.length = toIntArray(partitionLength);
	}

	@Override
	public void init() {
		partitionUtil = new PartitionUtil(count, length);

	}

	@Override
	public Integer calculate(String columnValue)  {
//		columnValue = NumberParseUtil.eliminateQoute(columnValue);
		try {
			long key = Long.parseLong(columnValue);
			key = (key >>> 32) ^ key;
			return partitionUtil.partition(key);
		} catch (NumberFormatException e){
			throw new IllegalArgumentException(new StringBuilder().append("columnValue:").append(columnValue).append(" Please eliminate any quote and non number within it.").toString(),e);
		}
	}
	
	@Override
	public Integer[] calculateRange(String beginValue, String endValue)  {
		return AbstractPartitionAlgorithm.calculateSequenceRange(this, beginValue, endValue);
	}

//	@Override
//	public int getPartitionCount() {
//		int nPartition = 0;
//		for(int i = 0; i < count.length; i++) {
//			nPartition += count[i];
//		}
//		return nPartition;
//	}
	
}

================================================
FILE: src/main/java/io/mycat/route/function/PartitionByMod.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.function;

import io.mycat.util.StringUtil;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import io.mycat.config.model.rule.RuleAlgorithm;

/**
 * number column partion by Mod operator
 * if count is 10 then 0 to 0,21 to 1 (21 % 10 =1)
 * @author wuzhih
 *
 */
public class PartitionByMod extends AbstractPartitionAlgorithm implements RuleAlgorithm  {

	private int count;
	@Override
	public void init() {
	
		
	}



	public void setCount(int count) {
		this.count = count;
	}

	@Override
	public Integer calculate(String columnValue)  {
//		columnValue = NumberParseUtil.eliminateQoute(columnValue);
		try {
			BigInteger bigNum = new BigInteger(columnValue).abs();
			return (bigNum.mod(BigInteger.valueOf(count))).intValue();
		} catch (NumberFormatException e){
            // throw new IllegalArgumentException(new
            // StringBuilder().append("columnValue:").append(columnValue).append(" Please
            // eliminate any quote and non number within it.").toString(),e);
 //           int value = Math.abs((columnValue).hashCode());
//            return value % count;
			/**
 			* hashcode后的值使用int类型超出数值范围时，返回负数，报数组下标越界
 			* date 2020/12/21
 			*/
			long value = Math.abs(Long.valueOf((columnValue).hashCode()));
			return (int)(value%count);
		}

	}
	

	@Override
	public int getPartitionNum() {
		int nPartition = this.count;
		return nPartition;
	}

	private static void hashTest()  {
		PartitionByMod hash=new PartitionByMod();
		hash.setCount(11);
		hash.init();
		
		int[] bucket=new int[hash.count];
		
		Map> hashed=new HashMap<>();
		
		int total=1000_0000;//数据量
		int c=0;
		for(int i=100_0000;i list=hashed.get(h);
			if(list==null){
				list=new ArrayList<>();
				hashed.put(h, list);
			}
			list.add(i);
		}
		System.out.println(c+"   "+total);
		double d=0;
		c=0;
		int idx=0;
		System.out.println("index    bucket   ratio");
		for(int i:bucket){
			d+=i/(double)total;
			c+=i;
			System.out.println(idx+++"  "+i+"   "+(i/(double)total));
		}
		System.out.println(d+"  "+c);
		
		System.out.println("****************************************************");
		rehashTest(hashed.get(0));
	}
	private static void rehashTest(List partition)  {
		PartitionByMod hash=new PartitionByMod();
		hash.count=110;//分片数
		hash.init();
		
		int[] bucket=new int[hash.count];
		
		int total=partition.size();//数据量
		int c=0;
		for(int i:partition){//假设分片键从100万开始
			c++;
			int h=hash.calculate(StringUtil.removeBackquote(Integer.toString(i)));
			bucket[h]++;
		}
		System.out.println(c+"   "+total);
		c=0;
		int idx=0;
		System.out.println("index    bucket   ratio");
		for(int i:bucket){
			c+=i;
			System.out.println(idx+++"  "+i+"   "+(i/(double)total));
		}
	}
	public static void main(String[] args)  {
//		hashTest();
		PartitionByMod partitionByMod = new PartitionByMod();
		partitionByMod.count=8;
		partitionByMod.calculate("\"6\"");
		partitionByMod.calculate("\'6\'");
	}
}

================================================
FILE: src/main/java/io/mycat/route/function/PartitionByMonth.java
================================================
package io.mycat.route.function;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.List;

import org.apache.log4j.Logger;

import io.mycat.config.model.rule.RuleAlgorithm;
import io.mycat.util.StringUtil;

/**
 * 例子 按月份列分区 ，每个自然月一个分片，格式 between操作解析的范例
 *
 * @author wzh
 *
 */
public class PartitionByMonth extends AbstractPartitionAlgorithm implements
    RuleAlgorithm {

    private static final Logger LOGGER = Logger.getLogger(PartitionByDate.class);
    private String sBeginDate;
    /** 默认格式 */
    private String dateFormat = "yyyy-MM-dd";
    /** 场景 */
    private int scene = -1;
    private String sEndDate;
    private Calendar beginDate;
    private Calendar endDate;
    private int nPartition;

    private ThreadLocal formatter;

    @Override
    public void init() {
        try {
            if (StringUtil.isEmpty(sBeginDate) && StringUtil.isEmpty(sEndDate)) {
                nPartition = 12;
                scene = 1;
                initFormatter();
                beginDate = Calendar.getInstance();
                beginDate.set(Calendar.MONTH, 0);
                endDate = Calendar.getInstance();
                endDate.set(Calendar.MONTH, 11);
                return;
            }
            beginDate = Calendar.getInstance();
            beginDate.setTime(new SimpleDateFormat(dateFormat)
                .parse(sBeginDate));
            initFormatter();
            if (sEndDate != null && !sEndDate.equals("")) {
                endDate = Calendar.getInstance();
                endDate.setTime(new SimpleDateFormat(dateFormat).parse(sEndDate));
                nPartition = ((endDate.get(Calendar.YEAR) - beginDate.get(Calendar.YEAR)) * 12
                    + endDate.get(Calendar.MONTH) - beginDate.get(Calendar.MONTH)) + 1;

                if (nPartition <= 0) {
                    throw new java.lang.IllegalArgumentException("Incorrect time range for month partitioning!");
                }
            } else {
                nPartition = -1;
            }
        } catch (ParseException e) {
            throw new java.lang.IllegalArgumentException(e);
        }
    }

    private void initFormatter() {
        formatter = new ThreadLocal() {
            @Override
            protected SimpleDateFormat initialValue() {
                return new SimpleDateFormat(dateFormat);
            }
        };
    }

    /**
     * For circulatory partition, calculated value of target partition needs to be
     * rotated to fit the partition range
     */
    private int reCalculatePartition(int targetPartition) {
        // 没有指定end_date，不是循环使用的情况，直接返回对应的targetPartition
        if (nPartition == -1) {
            return targetPartition;
        }
        /**
         * If target date is previous of start time of partition setting, shift
         * the delta range between target and start date to be positive value
         */
        if (targetPartition < 0) {
            targetPartition = nPartition - (-targetPartition) % nPartition;
        }

        if (targetPartition >= nPartition) {
            targetPartition = targetPartition % nPartition;
        }

        return targetPartition;
    }

    @Override
    public Integer calculate(String columnValue) {
        try {
            if (scene == 1) {
                Calendar curTime = Calendar.getInstance();
                curTime.setTime(formatter.get().parse(columnValue));
                return curTime.get(Calendar.MONTH);
            }
            int targetPartition;
            Calendar curTime = Calendar.getInstance();
            curTime.setTime(formatter.get().parse(columnValue));
            targetPartition = ((curTime.get(Calendar.YEAR) - beginDate.get(Calendar.YEAR))
                * 12 + curTime.get(Calendar.MONTH)
                - beginDate.get(Calendar.MONTH));

            /**
             * For circulatory partition, calculated value of target partition needs to be
             * rotated to fit the partition range
             */
            if (nPartition > 0) {
                targetPartition = reCalculatePartition(targetPartition);
            }
            // 防止越界的情况
            if (targetPartition < 0) {
                targetPartition = 0;
            }
            return targetPartition;

        } catch (ParseException e) {
            throw new IllegalArgumentException(new StringBuilder().append("columnValue:").append(columnValue)
                .append(" Please check if the format satisfied.").toString(), e);
        }
    }

    @Override
    public Integer[] calculateRange(String beginValue, String endValue) {
        try {
            return doCalculateRange(beginValue, endValue, beginDate);
        } catch (ParseException e) {
            LOGGER.error("error", e);
            return new Integer[0];
        }
    }

    private Integer[] doCalculateRange(String beginValue, String endValue, Calendar beginDate) throws ParseException {
        int startPartition, endPartition;
        Calendar partitionTime = Calendar.getInstance();
        SimpleDateFormat format = new SimpleDateFormat(dateFormat);
        partitionTime.setTime(format.parse(beginValue));
        startPartition = ((partitionTime.get(Calendar.YEAR) - beginDate.get(Calendar.YEAR))
            * 12 + partitionTime.get(Calendar.MONTH)
            - beginDate.get(Calendar.MONTH));
        partitionTime.setTime(format.parse(endValue));
        endPartition = ((partitionTime.get(Calendar.YEAR) - beginDate.get(Calendar.YEAR))
            * 12 + partitionTime.get(Calendar.MONTH)
            - beginDate.get(Calendar.MONTH));

        List list = new ArrayList<>();

        while (startPartition <= endPartition) {
            Integer nodeValue = reCalculatePartition(startPartition);
            if (nodeValue < 0) {
                nodeValue = 0;
            }
            if (Collections.frequency(list, nodeValue) < 1) {
                list.add(nodeValue);
            }
            startPartition++;
        }
        int size = list.size();
        // 当在场景1： "2015-01-01", "2014-04-03" 范围出现的时候
        // 是应该返回null 还是返回 [] ?
        return (list.toArray(new Integer[size]));
    }

    @Override
    public int getPartitionNum() {
        int nPartition = this.nPartition;
        return nPartition;
    }

    public void setsBeginDate(String sBeginDate) {
        this.sBeginDate = sBeginDate;
    }

    public void setDateFormat(String dateFormat) {
        this.dateFormat = dateFormat;
    }

    public void setsEndDate(String sEndDate) {
        this.sEndDate = sEndDate;
    }

}


================================================
FILE: src/main/java/io/mycat/route/function/PartitionByMonthAndHistory.java
================================================
package io.mycat.route.function;


import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.List;

import org.apache.log4j.Logger;

import io.mycat.config.model.rule.RuleAlgorithm;

/**
 * 例子 按月份列分区 ，每个自然月一个分片，格式 between操作解析的范例
 * 
 * @author wzh
 * 
 */
public class PartitionByMonthAndHistory extends AbstractPartitionAlgorithm implements
        RuleAlgorithm {
    private static final Logger LOGGER = Logger.getLogger(PartitionByMonth.class);
    private String sBeginDate;
    private String dateFormat;
    private String sEndDate;
    private Calendar beginDate;
    private Calendar endDate;
    private int nPartition;

    private ThreadLocal formatter;

    @Override
    public void init() {
        try {
            beginDate = Calendar.getInstance();
            beginDate.setTime(new SimpleDateFormat(dateFormat)
                    .parse(sBeginDate));
            formatter = new ThreadLocal() {
                @Override
                protected SimpleDateFormat initialValue() {
                    return new SimpleDateFormat(dateFormat);
                }
            };
            if(sEndDate!=null&&!sEndDate.equals("")) {
                endDate = Calendar.getInstance();
                endDate.setTime(new SimpleDateFormat(dateFormat).parse(sEndDate));
                nPartition = ((endDate.get(Calendar.YEAR) - beginDate.get(Calendar.YEAR)) * 12
                                + endDate.get(Calendar.MONTH) - beginDate.get(Calendar.MONTH)) + 1;

                if (nPartition <= 0) {
                    throw new java.lang.IllegalArgumentException("Incorrect time range for month partitioning!");
                }
            } else {
                nPartition = -1;
            }
        } catch (ParseException e) {
            throw new java.lang.IllegalArgumentException(e);
        }
    }

    /**
     * For circulatory partition, calculated value of target partition needs to be
     * rotated to fit the partition range
     */
    private int reCalculatePartition(int targetPartition) {
        /**
         * If target date is previous of start time of partition setting, shift
         * the delta range between target and start date to be positive value
         */
        if (targetPartition < 0) {
            targetPartition = nPartition - (-targetPartition) % nPartition;
        }

        if (targetPartition >= nPartition) {
            targetPartition =  targetPartition % nPartition;
        }
        LOGGER.debug("partition is:" + targetPartition);
        return targetPartition;
    }

    @Override
    public Integer calculate(String columnValue)  {
        try {
            int targetPartition;
            Calendar curTime = Calendar.getInstance();
            curTime.setTime(formatter.get().parse(columnValue));
            targetPartition = ((curTime.get(Calendar.YEAR) - beginDate.get(Calendar.YEAR))
                    * 12 + curTime.get(Calendar.MONTH)
                    - beginDate.get(Calendar.MONTH));

            /**
             * For circulatory partition, calculated value of target partition needs to be
             * rotated to fit the partition range
             */
            if (nPartition > 0) {
                targetPartition = reCalculatePartition(targetPartition);
            }
            return targetPartition;

        } catch (ParseException e) {
            throw new IllegalArgumentException(new StringBuilder().append("columnValue:").append(columnValue).append(" Please check if the format satisfied.").toString(),e);
        }
    }

    @Override
    public Integer[] calculateRange(String beginValue, String endValue) {
        try {
            int startPartition, endPartition;
            Calendar partitionTime = Calendar.getInstance();
            SimpleDateFormat format = new SimpleDateFormat(dateFormat);
            partitionTime.setTime(format.parse(beginValue));
            startPartition = ((partitionTime.get(Calendar.YEAR) - beginDate.get(Calendar.YEAR))
                    * 12 + partitionTime.get(Calendar.MONTH)
                    - beginDate.get(Calendar.MONTH));
            partitionTime.setTime(format.parse(endValue));
            endPartition = ((partitionTime.get(Calendar.YEAR) - beginDate.get(Calendar.YEAR))
                    * 12 + partitionTime.get(Calendar.MONTH)
                    - beginDate.get(Calendar.MONTH));

            List list = new ArrayList<>();

            while (startPartition <= endPartition) {
                Integer nodeValue = reCalculatePartition(startPartition);
                if (Collections.frequency(list, nodeValue) < 1)
                    list.add(nodeValue);
                startPartition++;
            }
            int size = list.size();
            return (list.toArray(new Integer[size]));
        } catch (ParseException e) {
            LOGGER.error(e);
            return new Integer[0];
        }
    }

    public void setsBeginDate(String sBeginDate) {
        this.sBeginDate = sBeginDate;
    }

    public void setDateFormat(String dateFormat) {
        this.dateFormat = dateFormat;
    }

    public void setsEndDate(String sEndDate) {
        this.sEndDate = sEndDate;
    }

}

================================================
FILE: src/main/java/io/mycat/route/function/PartitionByMurmurHash.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.function;

import io.mycat.util.StringUtil;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.SortedMap;
import java.util.TreeMap;

import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;

import io.mycat.config.model.rule.RuleAlgorithm;
import io.mycat.util.exception.MurmurHashException;

/**
 * consistancy hash, murmur hash
 * implemented by Guava
 * @author wuzhih
 *
 */
public class PartitionByMurmurHash extends AbstractPartitionAlgorithm implements RuleAlgorithm  {
	private static final int DEFAULT_VIRTUAL_BUCKET_TIMES=160;
	private static final int DEFAULT_WEIGHT=1;
	private static final Charset DEFAULT_CHARSET=Charset.forName("UTF-8");
	
	private int seed;
	private int count;
	private int virtualBucketTimes=DEFAULT_VIRTUAL_BUCKET_TIMES;
	private Map weightMap=new HashMap<>();
//	private String bucketMapPath;
	
	private HashFunction hash;
	
	private SortedMap bucketMap;
	@Override
	public void init()  {
		try{
			bucketMap=new TreeMap<>();
//			boolean serializableBucketMap=bucketMapPath!=null && bucketMapPath.length()>0;
//			if(serializableBucketMap){
//				File bucketMapFile=new File(bucketMapPath);
//				if(bucketMapFile.exists() && bucketMapFile.length()>0){
//					loadBucketMapFile();
//					return;
//				}
//			}
			generateBucketMap();
//			if(serializableBucketMap){
//				storeBucketMap();
//			}
		}catch(Exception e){
			throw new MurmurHashException(e);
		}
	}

	private void generateBucketMap(){
		hash=Hashing.murmur3_32(seed);//计算一致性哈希的对象
		for(int i=0;i0?weight:1);
			}
		}
	}
//	/**
//	 * 保存一致性hash的虚拟节点文件路径。
//	 * 如果这个文件不存在或是空文件就按照指定的count, weightMapFile等构造新的MurmurHash数据结构并保存到这个路径的文件里。
//	 * 如果这个文件已存在且不是空文件就加载这个文件里的内容作为MurmurHash数据结构，此时其它参数都忽略。
//	 * 除第一次以外在之后增加节点时可以直接修改这个文件，不过不推荐这么做。如果节点数量变化了，推荐删除这个文件。
//	 * 可以不指定这个路径，不指定路径时不会保存murmur hash
//	 * @param bucketMapPath
//	 */
//	public void setBucketMapPath(String bucketMapPath){
//		this.bucketMapPath=bucketMapPath;
//	}
	@Override
	public Integer calculate(String columnValue) {
		SortedMap tail = bucketMap.tailMap(hash.hashUnencodedChars(columnValue).asInt());
		if (tail.isEmpty()) {
		    return bucketMap.get(bucketMap.firstKey());
		}
		return tail.get(tail.firstKey());
	}

	@Override
	public int getPartitionNum() {
		int nPartition = this.count;
		return nPartition;
	}

	private static void hashTest() throws IOException{
		PartitionByMurmurHash hash=new PartitionByMurmurHash();
		hash.count=10;//分片数
		hash.init();

		int[] bucket=new int[hash.count];

		Map> hashed=new HashMap<>();

		int total=1000_0000;//数据量
		int c=0;
		for(int i=100_0000;i list=hashed.get(h);
			if(list==null){
				list=new ArrayList<>();
				hashed.put(h, list);
			}
			list.add(i);
		}
		System.out.println(c+"   "+total);
		double d=0;
		c=0;
		int idx=0;
		System.out.println("index    bucket   ratio");
		for(int i:bucket){
			d+=i/(double)total;
			c+=i;
			System.out.println(idx+++"  "+i+"   "+(i/(double)total));
		}
		System.out.println(d+"  "+c);

		Properties props=new Properties();
		for(Map.Entry entry:hash.bucketMap.entrySet()){
			props.setProperty(entry.getKey().toString(), entry.getValue().toString());
		}
		ByteArrayOutputStream out=new ByteArrayOutputStream();
		props.store(out, null);

		props.clear();
		props.load(new ByteArrayInputStream(out.toByteArray()));
		System.out.println(props);
		System.out.println("****************************************************");
//		rehashTest(hashed.get(0));
	}
	private static void rehashTest(List partition){
		PartitionByMurmurHash hash=new PartitionByMurmurHash();
		hash.count=12;//分片数
		hash.init();
		
		int[] bucket=new int[hash.count];
		
		int total=partition.size();//数据量
		int c=0;
		for(int i:partition){//假设分片键从100万开始
			c++;
			int h=hash.calculate(StringUtil.removeBackquote(Integer.toString(i)));
			bucket[h]++;
		}
		System.out.println(c+"   "+total);
		c=0;
		int idx=0;
		System.out.println("index    bucket   ratio");
		for(int i:bucket){
			c+=i;
			System.out.println(idx+++"  "+i+"   "+(i/(double)total));
		}
	}
	public static void main(String[] args) throws IOException {
		hashTest();
	}
}


================================================
FILE: src/main/java/io/mycat/route/function/PartitionByPattern.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.function;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.Set;
import java.util.regex.Pattern;

import io.mycat.config.model.rule.RuleAlgorithm;
import io.mycat.route.function.PartitionByPrefixPattern.LongRange;

/**
 * auto partition by Long
 * 
 * @author hexiaobin
 */
public class PartitionByPattern extends AbstractPartitionAlgorithm implements RuleAlgorithm {
	private static final int PARTITION_LENGTH = 1024;
	private int patternValue = PARTITION_LENGTH;// 分区长度，取模数值
	private String mapFile;
	private LongRange[] longRongs;
	private int defaultNode = 0;// 包含非数值字符，默认存储节点
    private static final  Pattern pattern = Pattern.compile("[0-9]*");;

	@Override
	public void init() {

		initialize();
	}

	public void setMapFile(String mapFile) {
		this.mapFile = mapFile;
	}

	public void setPatternValue(int patternValue) {
		this.patternValue = patternValue;
	}

	public void setDefaultNode(int defaultNode) {
		this.defaultNode = defaultNode;
	}

	@Override
	public Integer calculate(String columnValue) {
		if (!isNumeric(columnValue)) {
			return defaultNode;
		}
		long value = Long.parseLong(columnValue);
		Integer rst = null;
		for (LongRange longRang : this.longRongs) {
			long hash = value % patternValue;
			if (hash <= longRang.valueEnd && hash >= longRang.valueStart) {
				return longRang.nodeIndx;
			}
		}
		return rst;
	}
	
	@Override
	public int getPartitionNum() {
//		int nPartition = this.longRongs.length;
		/*
		 * fix #1284 这里的统计应该统计Range的nodeIndex的distinct总数
		 */
		Set distNodeIdxSet = new HashSet();
		for(LongRange range : longRongs) {
			distNodeIdxSet.add(range.nodeIndx);
		}
		int nPartition = distNodeIdxSet.size();
		return nPartition;
	}

	public static boolean isNumeric(String str) {
		return pattern.matcher(str).matches();
	}

	private void initialize() {
		BufferedReader in = null;
		try {
			// FileInputStream fin = new FileInputStream(new File(fileMapPath));
			InputStream fin = this.getClass().getClassLoader()
					.getResourceAsStream(mapFile);
			if (fin == null) {
				throw new RuntimeException("can't find class resource file "
						+ mapFile);
			}
			in = new BufferedReader(new InputStreamReader(fin));
			LinkedList longRangeList = new LinkedList();

			for (String line = null; (line = in.readLine()) != null;) {
				line = line.trim();
				if (line.startsWith("#") || line.startsWith("//")) {
					continue;
				}
				int ind = line.indexOf('=');
				if (ind < 0) {
					System.out.println(" warn: bad line int " + mapFile + " :"
							+ line);
					continue;
				}
					String pairs[] = line.substring(0, ind).trim().split("-");
					long longStart = Long.parseLong(pairs[0].trim());
					long longEnd = Long.parseLong(pairs[1].trim());
					int nodeId = Integer.parseInt(line.substring(ind + 1)
							.trim());
					longRangeList
							.add(new LongRange(nodeId, longStart, longEnd));

			}
			longRongs = longRangeList.toArray(new LongRange[longRangeList
					.size()]);
		} catch (Exception e) {
			if (e instanceof RuntimeException) {
				throw (RuntimeException) e;
			} else {
				throw new RuntimeException(e);
			}

		} finally {
			try {
				in.close();
			} catch (Exception e2) {
			}
		}
	}

	static class LongRange {
		public final int nodeIndx;
		public final long valueStart;
		public final long valueEnd;

		public LongRange(int nodeIndx, long valueStart, long valueEnd) {
			super();
			this.nodeIndx = nodeIndx;
			this.valueStart = valueStart;
			this.valueEnd = valueEnd;
		}

	}
}

================================================
FILE: src/main/java/io/mycat/route/function/PartitionByPrefixPattern.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.function;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.Set;

import io.mycat.config.model.rule.RuleAlgorithm;
import io.mycat.route.function.AutoPartitionByLong.LongRange;

/**
 * partition by Prefix length ,can be used in String partition
 * 
 * @author hexiaobin
 */
public class PartitionByPrefixPattern extends AbstractPartitionAlgorithm implements RuleAlgorithm {
	private static final int PARTITION_LENGTH = 1024;
	private int patternValue = PARTITION_LENGTH;// 分区长度，取模数值(默认为1024)
	private int prefixLength;// 字符前几位进行ASCII码取和
	private String mapFile;
	private LongRange[] longRongs;

	@Override
	public void init() {

		initialize();
	}

	public void setMapFile(String mapFile) {
		this.mapFile = mapFile;
	}

	public void setPatternValue(int patternValue) {
		this.patternValue = patternValue;
	}

	public void setPrefixLength(int prefixLength) {
		this.prefixLength = prefixLength;
	}

	@Override
	public Integer calculate(String columnValue)  {
		try {
			int Length = Integer.valueOf(prefixLength);

			Length = columnValue.length() < Length ? columnValue.length() : Length;
			int sum = 0;
			for (int i = 0; i < Length; i++) {
				sum = sum + columnValue.charAt(i);
			}
			Integer rst = null;
			for (LongRange longRang : this.longRongs) {
				long hash = sum % patternValue;
				if (hash <= longRang.valueEnd && hash >= longRang.valueStart) {
					return longRang.nodeIndx;
				}
			}
			return rst;
		} catch (NumberFormatException e){
			throw new IllegalArgumentException(new StringBuilder().append("columnValue:").append(columnValue).append(" Please eliminate any quote and non number within it.").toString(),e);
		}
	}
	
	@Override
	public int getPartitionNum() {
//		int nPartition = this.longRongs.length;
		/*
		 * fix #1284 这里的统计应该统计Range的nodeIndex的distinct总数
		 */
		Set distNodeIdxSet = new HashSet();
		for(LongRange range : longRongs) {
			distNodeIdxSet.add(range.nodeIndx);
		}
		int nPartition = distNodeIdxSet.size();
		return nPartition;
	}

	private void initialize() {
		BufferedReader in = null;
		try {
			// FileInputStream fin = new FileInputStream(new File(fileMapPath));
			InputStream fin = this.getClass().getClassLoader()
					.getResourceAsStream(mapFile);
			if (fin == null) {
				throw new RuntimeException("can't find class resource file "
						+ mapFile);
			}
			in = new BufferedReader(new InputStreamReader(fin));
			LinkedList longRangeList = new LinkedList();

			for (String line = null; (line = in.readLine()) != null;) {
				line = line.trim();
				if (line.startsWith("#") || line.startsWith("//")) {
					continue;
				}
				int ind = line.indexOf('=');
				if (ind < 0) {
					System.out.println(" warn: bad line int " + mapFile + " :"
							+ line);
					continue;
				}
					String pairs[] = line.substring(0, ind).trim().split("-");
					long longStart = NumberParseUtil.parseLong(pairs[0].trim());
					long longEnd = NumberParseUtil.parseLong(pairs[1].trim());
					int nodeId = Integer.parseInt(line.substring(ind + 1)
							.trim());
					longRangeList
							.add(new LongRange(nodeId, longStart, longEnd));

			}
			longRongs = longRangeList.toArray(new LongRange[longRangeList
					.size()]);
		} catch (Exception e) {
			if (e instanceof RuntimeException) {
				throw (RuntimeException) e;
			} else {
				throw new RuntimeException(e);
			}

		} finally {
			try {
				in.close();
			} catch (Exception e2) {
			}
		}
	}

	static class LongRange {
		public final int nodeIndx;
		public final long valueStart;
		public final long valueEnd;

		public LongRange(int nodeIndx, long valueStart, long valueEnd) {
			super();
			this.nodeIndx = nodeIndx;
			this.valueStart = valueStart;
			this.valueEnd = valueEnd;
		}

	}
}

================================================
FILE: src/main/java/io/mycat/route/function/PartitionByRangeDateHash.java
================================================
package io.mycat.route.function;

import com.google.common.hash.Hashing;

import io.mycat.config.model.rule.RuleAlgorithm;

import org.slf4j.Logger; import org.slf4j.LoggerFactory;

import java.text.ParseException;
import java.text.SimpleDateFormat;

/**
 * 先根据日期分组，再根据时间hash使得短期内数据分布的更均匀
 * 优点可以避免扩容时的数据迁移，又可以一定程度上避免范围分片的热点问题
 * 要求日期格式尽量精确些，不然达不到局部均匀的目的
 *
 *
 */
public class PartitionByRangeDateHash extends AbstractPartitionAlgorithm implements RuleAlgorithm
{
    private static final Logger LOGGER = LoggerFactory
            .getLogger(PartitionByRangeDateHash.class);

    private String sBeginDate;
    private String sPartionDay;
    private String dateFormat;

    private long beginDate;
    private long partionTime;

    private static final long oneDay = 86400000;

    private String groupPartionSize;
    private int intGroupPartionSize;

    private ThreadLocal formatter;

    @Override
    public void init()
    {
        try
        {
            beginDate = new SimpleDateFormat(dateFormat).parse(sBeginDate)
                    .getTime();
            intGroupPartionSize = Integer.parseInt(groupPartionSize);
            formatter = new ThreadLocal() {
                @Override
                protected SimpleDateFormat initialValue() {
                    return new SimpleDateFormat(dateFormat);
                }
            };
            if (intGroupPartionSize <= 0)
            {
                throw new RuntimeException("groupPartionSize must >0,but cur is " + intGroupPartionSize);
            }
        } catch (ParseException e)
        {
            throw new IllegalArgumentException(e);
        }
        partionTime = Integer.parseInt(sPartionDay) * oneDay;
    }

    @Override
    public Integer calculate(String columnValue)  {
        try
        {
            long targetTime = formatter.get().parse(
                    columnValue).getTime();
            int targetPartition = (int) ((targetTime - beginDate) / partionTime);
            int innerIndex =  Hashing.consistentHash(targetTime,intGroupPartionSize);
            return targetPartition * intGroupPartionSize + innerIndex;

        } catch (ParseException e)
        {
            throw new IllegalArgumentException(new StringBuilder().append("columnValue:").append(columnValue).append(" Please check if the format satisfied.").toString(),e);
        }
    }

    public Integer calculateStart(String columnValue)
    {
        try
        {
            long targetTime = new SimpleDateFormat(dateFormat).parse(
                    columnValue).getTime();
            int targetPartition = (int) ((targetTime - beginDate) / partionTime);
            return targetPartition * intGroupPartionSize;

        } catch (ParseException e)
        {
            throw new IllegalArgumentException(e);

        }
    }

    public Integer calculateEnd(String columnValue)
    {
        try
        {
            long targetTime = new SimpleDateFormat(dateFormat).parse(
                    columnValue).getTime();
            int targetPartition = (int) ((targetTime - beginDate) / partionTime);
            return (targetPartition+1) * intGroupPartionSize  - 1;

        } catch (ParseException e)
        {
            throw new IllegalArgumentException(e);

        }
    }

    @Override
    public Integer[] calculateRange(String beginValue, String endValue)
    {
        Integer begin = 0, end = 0;
        begin = calculateStart(beginValue);
        end = calculateEnd(endValue);

        if (begin == null || end == null)
        {
            return new Integer[0];
        }

        if (end >= begin)
        {
            int len = end - begin + 1;
            Integer[] re = new Integer[len];

            for (int i = 0; i < len; i++)
            {
                re[i] = begin + i;
            }

            return re;
        } else
        {
            return null;
        }
    }

    public long getBeginDate()
    {
        return beginDate;
    }

    public void setsBeginDate(String sBeginDate)
    {
        this.sBeginDate = sBeginDate;
    }

    public void setsPartionDay(String sPartionDay)
    {
        this.sPartionDay = sPartionDay;
    }

    public void setDateFormat(String dateFormat)
    {
        this.dateFormat = dateFormat;
    }

    public String getGroupPartionSize()
    {
        return groupPartionSize;
    }

    public void setGroupPartionSize(String groupPartionSize)
    {
        this.groupPartionSize = groupPartionSize;
    }
}


================================================
FILE: src/main/java/io/mycat/route/function/PartitionByRangeMod.java
================================================
/*
 * Copyright (c) 2015, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://github.com/MyCATApache/Mycat-Server.
 *
 */
package io.mycat.route.function;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.math.BigInteger;
import java.util.LinkedList;

import io.mycat.config.model.rule.RuleAlgorithm;

/**
 * 先进行范围分片计算出分片组，组内再取模
 * 优点可以避免扩容时的数据迁移，又可以一定程度上避免范围分片的热点问题
 * 
 * @author wuzhi
 */
public class PartitionByRangeMod extends AbstractPartitionAlgorithm implements RuleAlgorithm{

	private String mapFile;
	private LongRange[] longRanges;
	
	private int defaultNode = -1;
	@Override
	public void init() {

		initialize();
	}

	public void setMapFile(String mapFile) {
		this.mapFile = mapFile;
	}

	@Override
	public Integer calculate(String columnValue)  {
//		columnValue = NumberParseUtil.eliminateQoute(columnValue);
		try {
			long value = Long.parseLong(columnValue);
			Integer rst = null;
			int nodeIndex = 0;
			for (LongRange longRang : this.longRanges) {
				if (value <= longRang.valueEnd && value >= longRang.valueStart) {
					BigInteger bigNum = new BigInteger(columnValue).abs();
					int innerIndex = (bigNum.mod(BigInteger.valueOf(longRang.groupSize))).intValue();
					return nodeIndex + innerIndex;
				} else {
					nodeIndex += longRang.groupSize;
				}
			}
			//数据超过范围，暂时使用配置的默认节点
			if (rst == null && defaultNode >= 0) {
				return defaultNode;
			}
			return rst;
		} catch (NumberFormatException e) {
			throw new IllegalArgumentException(new StringBuilder().append("columnValue:").append(columnValue).append(" Please eliminate any quote and non number within it.").toString(), e);
		}
	}
    
	@Override
	public int getPartitionNum() {
		int nPartition = 0;
		for(LongRange longRange : this.longRanges) {
			nPartition += longRange.groupSize;
		}
		return nPartition;
	}

	public Integer calculateStart(String columnValue) {
        long value = Long.parseLong(columnValue);
        Integer rst = null;
        int nodeIndex=0;
        for (LongRange longRang : this.longRanges) {
            if (value <= longRang.valueEnd && value >= longRang.valueStart) {

                return nodeIndex;
            }    else
            {
                nodeIndex+= longRang.groupSize;
            }
        }
        // 数据超过范围，暂时使用配置的默认节点
        if(rst ==null && defaultNode>=0){
            return defaultNode ;
        }
        return rst;
    }
    public Integer calculateEnd(String columnValue) {
        long value = Long.parseLong(columnValue);
        Integer rst = null;
        int nodeIndex=0;
        for (LongRange longRang : this.longRanges) {
            if (value <= longRang.valueEnd && value >= longRang.valueStart) {

                return nodeIndex+longRang.groupSize -1;
            }    else
            {
                nodeIndex+= longRang.groupSize;
            }
        }
        // 数据超过范围，暂时使用配置的默认节点
        if(rst ==null && defaultNode>=0){
            return defaultNode ;
        }
        return rst;
    }
	
	@Override
	public Integer[] calculateRange(String beginValue, String endValue) {
        Integer begin = 0, end = 0;
        begin = calculateStart(beginValue);
        end = calculateEnd(endValue);

        if(begin == null || end == null){
            return new Integer[0];
        }

        if (end >= begin) {
            int len = end-begin+1;
            Integer [] re = new Integer[len];

            for(int i =0;i longRangeList = new LinkedList();

			for (String line = null; (line = in.readLine()) != null;) {
				line = line.trim();
				if (line.startsWith("#") || line.startsWith("//")) {
					continue;
				}
				int ind = line.indexOf('=');
				if (ind < 0) {
					System.out.println(" warn: bad line int " + mapFile + " :"
							+ line);
					continue;
				}
					String pairs[] = line.substring(0, ind).trim().split("-");
					long longStart = NumberParseUtil.parseLong(pairs[0].trim());
					long longEnd = NumberParseUtil.parseLong(pairs[1].trim());
					int nodeId = Integer.parseInt(line.substring(ind + 1)
							.trim());
					longRangeList
							.add(new LongRange(nodeId, longStart, longEnd));

			}
			longRanges = longRangeList.toArray(new LongRange[longRangeList.size()]);
		} catch (Exception e) {
			if (e instanceof RuntimeException) {
				throw (RuntimeException) e;
			} else {
				throw new RuntimeException(e);
			}

		} finally {
			try {
				in.close();
			} catch (Exception e2) {
			}
		}
	}
	
	public int getDefaultNode() {
		return defaultNode;
	}

	public void setDefaultNode(int defaultNode) {
		this.defaultNode = defaultNode;
	}

	static class LongRange {
		public final int groupSize;
		public final long valueStart;
		public final long valueEnd;

		public LongRange(int groupSize, long valueStart, long valueEnd) {
			super();
			this.groupSize = groupSize;
			this.valueStart = valueStart;
			this.valueEnd = valueEnd;
		}

	}
}


================================================
FILE: src/main/java/io/mycat/route/function/PartitionByString.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.function;

import io.mycat.config.model.rule.RuleAlgorithm;
import io.mycat.route.parser.util.Pair;
import io.mycat.route.util.PartitionUtil;
import io.mycat.util.StringUtil;

/**
 * @author yangwenx
 */
public final class PartitionByString extends AbstractPartitionAlgorithm implements RuleAlgorithm  {
  
    private int hashSliceStart = 0;
    /** 0 means str.length(), -1 means str.length()-1 */
    private int hashSliceEnd = 8;
    protected int[] count;
    protected int[] length;
    protected PartitionUtil partitionUtil;

    public void setPartitionCount(String partitionCount) {
        this.count = toIntArray(partitionCount);
    }

    public void setPartitionLength(String partitionLength) {
        this.length = toIntArray(partitionLength);
    }


	public void setHashLength(int hashLength) {
        setHashSlice(String.valueOf(hashLength));
    }

    public void setHashSlice(String hashSlice) {
        Pair p = sequenceSlicing(hashSlice);
        hashSliceStart = p.getKey();
        hashSliceEnd = p.getValue();
    }


    /**
     * "2" -> (0,2)

     * "1:2" -> (1,2)

     * "1:" -> (1,0)

     * "-1:" -> (-1,0)

     * ":-1" -> (0,-1)

     * ":" -> (0,0)

     */
    public static Pair sequenceSlicing(String slice) {
        int ind = slice.indexOf(':');
        if (ind < 0) {
            int i = Integer.parseInt(slice.trim());
            if (i >= 0) {
                return new Pair(0, i);
            } else {
                return new Pair(i, 0);
            }
        }
        String left = slice.substring(0, ind).trim();
        String right = slice.substring(1 + ind).trim();
        int start, end;
        if (left.length() <= 0) {
            start = 0;
        } else {
            start = Integer.parseInt(left);
        }
        if (right.length() <= 0) {
            end = 0;
        } else {
            end = Integer.parseInt(right);
        }
        return new Pair(start, end);
    }

	@Override
	public void init() {
		partitionUtil = new PartitionUtil(count,length);
		
	}
	private static int[] toIntArray(String string) {
		String[] strs = io.mycat.util.SplitUtil.split(string, ',', true);
		int[] ints = new int[strs.length];
		for (int i = 0; i < strs.length; ++i) {
			ints[i] = Integer.parseInt(strs[i]);
		}
		return ints;
	}
	@Override
	public Integer calculate(String key) {
        int start = hashSliceStart >= 0 ? hashSliceStart : key.length() + hashSliceStart;
        int end = hashSliceEnd > 0 ? hashSliceEnd : key.length() + hashSliceEnd;
        long hash = StringUtil.hash(key, start, end);
        return partitionUtil.partition(hash);
	}

	@Override
	public int getPartitionNum() {
		int nPartition = 0;
		for(int i = 0; i < count.length; i++) {
			nPartition += count[i];
		}
		return nPartition;
	}
	
}

================================================
FILE: src/main/java/io/mycat/route/function/PartitionDirectBySubString.java
================================================
package io.mycat.route.function;

import io.mycat.config.model.rule.RuleAlgorithm;

/**
 * 直接根据字符子串（必须是数字）计算分区号（由应用传递参数，显式指定分区号）。
 * 
 * 9 
 * 2
 * 8
 * 0
 * 
 */
public class PartitionDirectBySubString extends AbstractPartitionAlgorithm implements RuleAlgorithm {
    // 字符子串起始索引（zero-based)
    private int startIndex;
    // 字串长度
    private int size;
    // 分区数量
    private int partitionCount;
    // 默认分区（在分区数量定义时，字串标示的分区编号不在分区数量内时，使用默认分区）
    private int defaultPartition;

    public void setStartIndex(String str) {
        startIndex = Integer.parseInt(str);
    }

    public void setSize(String str) {
        size = Integer.parseInt(str);
    }

    public void setPartitionCount(String partitionCount) {
        this.partitionCount = Integer.parseInt(partitionCount);
    }

    public void setDefaultPartition(String defaultPartition) {
        this.defaultPartition = Integer.parseInt(defaultPartition);
    }

    @Override
    public void init() {

    }

    @Override
    public Integer calculate(String columnValue) {
        String partitionSubString = columnValue.substring(startIndex, startIndex + size);
        int partition = Integer.parseInt(partitionSubString, 10);
        return partitionCount > 0 && partition >= partitionCount
                ? defaultPartition : partition;
    }

	@Override
	public int getPartitionNum() {
		int nPartition = this.partitionCount;
		return nPartition;
	}
	
}


================================================
FILE: src/main/java/io/mycat/route/function/PureJavaCrc32.java
================================================
package io.mycat.route.function;

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

        import java.util.HashSet;
        import java.util.Set;
        import java.util.zip.Checksum;

/**
 * A pure-java implementation of the CRC32 checksum that uses
 * the same polynomial as the built-in native CRC32.
 *
 * This is to avoid the JNI overhead for certain uses of Checksumming
 * where many small pieces of data are checksummed in succession.
 *
 * The current version is ~10x to 1.8x as fast as Sun's native
 * java.util.zip.CRC32 in Java 1.6
 *
 * @see java.util.zip.CRC32
 *
 * This class is copied from hadoop-commons project.
 * (The initial patch added PureJavaCrc32 was HADOOP-6148)
 */
public class PureJavaCrc32 implements Checksum {

    /** the current CRC value, bit-flipped */
    private int crc;

    /** Create a new PureJavaCrc32 object. */
    public PureJavaCrc32() {
        reset();
    }

    @Override
    public long getValue() {
        return (~crc) & 0xffffffffL;
    }

    @Override
    public void reset() {
        crc = 0xffffffff;
    }

    @Override
    public void update(byte[] b, int off, int len) {
        int localCrc = crc;

        while(len > 7) {
            final int c0 =(b[off+0] ^ localCrc) & 0xff;
            final int c1 =(b[off+1] ^ (localCrc >>>= 8)) & 0xff;
            final int c2 =(b[off+2] ^ (localCrc >>>= 8)) & 0xff;
            final int c3 =(b[off+3] ^ (localCrc >>>= 8)) & 0xff;
            localCrc = (T[T8_7_start + c0] ^ T[T8_6_start + c1])
                    ^ (T[T8_5_start + c2] ^ T[T8_4_start + c3]);

            final int c4 = b[off+4] & 0xff;
            final int c5 = b[off+5] & 0xff;
            final int c6 = b[off+6] & 0xff;
            final int c7 = b[off+7] & 0xff;

            localCrc ^= (T[T8_3_start + c4] ^ T[T8_2_start + c5])
                    ^ (T[T8_1_start + c6] ^ T[T8_0_start + c7]);

            off += 8;
            len -= 8;
        }

    /* loop unroll - duff's device style */
        switch(len) {
            case 7: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
            case 6: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
            case 5: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
            case 4: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
            case 3: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
            case 2: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
            case 1: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
            default:
        /* nothing */
        }

        // Publish crc out to object
        crc = localCrc;
    }

    @Override
    final public void update(int b) {
        crc = (crc >>> 8) ^ T[T8_0_start + ((crc ^ b) & 0xff)];
    }

    /*
     * CRC-32 lookup tables generated by the polynomial 0xEDB88320.
     * See also TestPureJavaCrc32.Table.
     */
    private static final int T8_0_start = 0*256;
    private static final int T8_1_start = 1*256;
    private static final int T8_2_start = 2*256;
    private static final int T8_3_start = 3*256;
    private static final int T8_4_start = 4*256;
    private static final int T8_5_start = 5*256;
    private static final int T8_6_start = 6*256;
    private static final int T8_7_start = 7*256;

    private static final int[] T = new int[] {
    /* T8_0 */
            0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA,
            0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3,
            0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988,
            0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91,
            0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE,
            0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7,
            0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC,
            0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5,
            0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172,
            0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B,
            0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940,
            0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59,
            0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116,
            0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F,
            0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924,
            0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D,
            0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A,
            0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433,
            0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818,
            0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01,
            0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E,
            0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457,
            0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C,
            0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65,
            0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2,
            0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB,
            0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0,
            0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9,
            0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086,
            0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F,
            0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4,
            0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD,
            0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A,
            0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683,
            0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8,
            0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1,
            0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE,
            0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7,
            0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC,
            0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5,
            0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252,
            0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B,
            0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60,
            0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79,
            0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236,
            0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F,
            0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04,
            0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D,
            0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A,
            0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713,
            0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38,
            0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21,
            0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E,
            0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777,
            0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C,
            0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45,
            0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2,
            0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB,
            0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0,
            0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9,
            0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6,
            0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF,
            0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94,
            0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D,
    /* T8_1 */
            0x00000000, 0x191B3141, 0x32366282, 0x2B2D53C3,
            0x646CC504, 0x7D77F445, 0x565AA786, 0x4F4196C7,
            0xC8D98A08, 0xD1C2BB49, 0xFAEFE88A, 0xE3F4D9CB,
            0xACB54F0C, 0xB5AE7E4D, 0x9E832D8E, 0x87981CCF,
            0x4AC21251, 0x53D92310, 0x78F470D3, 0x61EF4192,
            0x2EAED755, 0x37B5E614, 0x1C98B5D7, 0x05838496,
            0x821B9859, 0x9B00A918, 0xB02DFADB, 0xA936CB9A,
            0xE6775D5D, 0xFF6C6C1C, 0xD4413FDF, 0xCD5A0E9E,
            0x958424A2, 0x8C9F15E3, 0xA7B24620, 0xBEA97761,
            0xF1E8E1A6, 0xE8F3D0E7, 0xC3DE8324, 0xDAC5B265,
            0x5D5DAEAA, 0x44469FEB, 0x6F6BCC28, 0x7670FD69,
            0x39316BAE, 0x202A5AEF, 0x0B07092C, 0x121C386D,
            0xDF4636F3, 0xC65D07B2, 0xED705471, 0xF46B6530,
            0xBB2AF3F7, 0xA231C2B6, 0x891C9175, 0x9007A034,
            0x179FBCFB, 0x0E848DBA, 0x25A9DE79, 0x3CB2EF38,
            0x73F379FF, 0x6AE848BE, 0x41C51B7D, 0x58DE2A3C,
            0xF0794F05, 0xE9627E44, 0xC24F2D87, 0xDB541CC6,
            0x94158A01, 0x8D0EBB40, 0xA623E883, 0xBF38D9C2,
            0x38A0C50D, 0x21BBF44C, 0x0A96A78F, 0x138D96CE,
            0x5CCC0009, 0x45D73148, 0x6EFA628B, 0x77E153CA,
            0xBABB5D54, 0xA3A06C15, 0x888D3FD6, 0x91960E97,
            0xDED79850, 0xC7CCA911, 0xECE1FAD2, 0xF5FACB93,
            0x7262D75C, 0x6B79E61D, 0x4054B5DE, 0x594F849F,
            0x160E1258, 0x0F152319, 0x243870DA, 0x3D23419B,
            0x65FD6BA7, 0x7CE65AE6, 0x57CB0925, 0x4ED03864,
            0x0191AEA3, 0x188A9FE2, 0x33A7CC21, 0x2ABCFD60,
            0xAD24E1AF, 0xB43FD0EE, 0x9F12832D, 0x8609B26C,
            0xC94824AB, 0xD05315EA, 0xFB7E4629, 0xE2657768,
            0x2F3F79F6, 0x362448B7, 0x1D091B74, 0x04122A35,
            0x4B53BCF2, 0x52488DB3, 0x7965DE70, 0x607EEF31,
            0xE7E6F3FE, 0xFEFDC2BF, 0xD5D0917C, 0xCCCBA03D,
            0x838A36FA, 0x9A9107BB, 0xB1BC5478, 0xA8A76539,
            0x3B83984B, 0x2298A90A, 0x09B5FAC9, 0x10AECB88,
            0x5FEF5D4F, 0x46F46C0E, 0x6DD93FCD, 0x74C20E8C,
            0xF35A1243, 0xEA412302, 0xC16C70C1, 0xD8774180,
            0x9736D747, 0x8E2DE606, 0xA500B5C5, 0xBC1B8484,
            0x71418A1A, 0x685ABB5B, 0x4377E898, 0x5A6CD9D9,
            0x152D4F1E, 0x0C367E5F, 0x271B2D9C, 0x3E001CDD,
            0xB9980012, 0xA0833153, 0x8BAE6290, 0x92B553D1,
            0xDDF4C516, 0xC4EFF457, 0xEFC2A794, 0xF6D996D5,
            0xAE07BCE9, 0xB71C8DA8, 0x9C31DE6B, 0x852AEF2A,
            0xCA6B79ED, 0xD37048AC, 0xF85D1B6F, 0xE1462A2E,
            0x66DE36E1, 0x7FC507A0, 0x54E85463, 0x4DF36522,
            0x02B2F3E5, 0x1BA9C2A4, 0x30849167, 0x299FA026,
            0xE4C5AEB8, 0xFDDE9FF9, 0xD6F3CC3A, 0xCFE8FD7B,
            0x80A96BBC, 0x99B25AFD, 0xB29F093E, 0xAB84387F,
            0x2C1C24B0, 0x350715F1, 0x1E2A4632, 0x07317773,
            0x4870E1B4, 0x516BD0F5, 0x7A468336, 0x635DB277,
            0xCBFAD74E, 0xD2E1E60F, 0xF9CCB5CC, 0xE0D7848D,
            0xAF96124A, 0xB68D230B, 0x9DA070C8, 0x84BB4189,
            0x03235D46, 0x1A386C07, 0x31153FC4, 0x280E0E85,
            0x674F9842, 0x7E54A903, 0x5579FAC0, 0x4C62CB81,
            0x8138C51F, 0x9823F45E, 0xB30EA79D, 0xAA1596DC,
            0xE554001B, 0xFC4F315A, 0xD7626299, 0xCE7953D8,
            0x49E14F17, 0x50FA7E56, 0x7BD72D95, 0x62CC1CD4,
            0x2D8D8A13, 0x3496BB52, 0x1FBBE891, 0x06A0D9D0,
            0x5E7EF3EC, 0x4765C2AD, 0x6C48916E, 0x7553A02F,
            0x3A1236E8, 0x230907A9, 0x0824546A, 0x113F652B,
            0x96A779E4, 0x8FBC48A5, 0xA4911B66, 0xBD8A2A27,
            0xF2CBBCE0, 0xEBD08DA1, 0xC0FDDE62, 0xD9E6EF23,
            0x14BCE1BD, 0x0DA7D0FC, 0x268A833F, 0x3F91B27E,
            0x70D024B9, 0x69CB15F8, 0x42E6463B, 0x5BFD777A,
            0xDC656BB5, 0xC57E5AF4, 0xEE530937, 0xF7483876,
            0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72,
    /* T8_2 */
            0x00000000, 0x01C26A37, 0x0384D46E, 0x0246BE59,
            0x0709A8DC, 0x06CBC2EB, 0x048D7CB2, 0x054F1685,
            0x0E1351B8, 0x0FD13B8F, 0x0D9785D6, 0x0C55EFE1,
            0x091AF964, 0x08D89353, 0x0A9E2D0A, 0x0B5C473D,
            0x1C26A370, 0x1DE4C947, 0x1FA2771E, 0x1E601D29,
            0x1B2F0BAC, 0x1AED619B, 0x18ABDFC2, 0x1969B5F5,
            0x1235F2C8, 0x13F798FF, 0x11B126A6, 0x10734C91,
            0x153C5A14, 0x14FE3023, 0x16B88E7A, 0x177AE44D,
            0x384D46E0, 0x398F2CD7, 0x3BC9928E, 0x3A0BF8B9,
            0x3F44EE3C, 0x3E86840B, 0x3CC03A52, 0x3D025065,
            0x365E1758, 0x379C7D6F, 0x35DAC336, 0x3418A901,
            0x3157BF84, 0x3095D5B3, 0x32D36BEA, 0x331101DD,
            0x246BE590, 0x25A98FA7, 0x27EF31FE, 0x262D5BC9,
            0x23624D4C, 0x22A0277B, 0x20E69922, 0x2124F315,
            0x2A78B428, 0x2BBADE1F, 0x29FC6046, 0x283E0A71,
            0x2D711CF4, 0x2CB376C3, 0x2EF5C89A, 0x2F37A2AD,
            0x709A8DC0, 0x7158E7F7, 0x731E59AE, 0x72DC3399,
            0x7793251C, 0x76514F2B, 0x7417F172, 0x75D59B45,
            0x7E89DC78, 0x7F4BB64F, 0x7D0D0816, 0x7CCF6221,
            0x798074A4, 0x78421E93, 0x7A04A0CA, 0x7BC6CAFD,
            0x6CBC2EB0, 0x6D7E4487, 0x6F38FADE, 0x6EFA90E9,
            0x6BB5866C, 0x6A77EC5B, 0x68315202, 0x69F33835,
            0x62AF7F08, 0x636D153F, 0x612BAB66, 0x60E9C151,
            0x65A6D7D4, 0x6464BDE3, 0x662203BA, 0x67E0698D,
            0x48D7CB20, 0x4915A117, 0x4B531F4E, 0x4A917579,
            0x4FDE63FC, 0x4E1C09CB, 0x4C5AB792, 0x4D98DDA5,
            0x46C49A98, 0x4706F0AF, 0x45404EF6, 0x448224C1,
            0x41CD3244, 0x400F5873, 0x4249E62A, 0x438B8C1D,
            0x54F16850, 0x55330267, 0x5775BC3E, 0x56B7D609,
            0x53F8C08C, 0x523AAABB, 0x507C14E2, 0x51BE7ED5,
            0x5AE239E8, 0x5B2053DF, 0x5966ED86, 0x58A487B1,
            0x5DEB9134, 0x5C29FB03, 0x5E6F455A, 0x5FAD2F6D,
            0xE1351B80, 0xE0F771B7, 0xE2B1CFEE, 0xE373A5D9,
            0xE63CB35C, 0xE7FED96B, 0xE5B86732, 0xE47A0D05,
            0xEF264A38, 0xEEE4200F, 0xECA29E56, 0xED60F461,
            0xE82FE2E4, 0xE9ED88D3, 0xEBAB368A, 0xEA695CBD,
            0xFD13B8F0, 0xFCD1D2C7, 0xFE976C9E, 0xFF5506A9,
            0xFA1A102C, 0xFBD87A1B, 0xF99EC442, 0xF85CAE75,
            0xF300E948, 0xF2C2837F, 0xF0843D26, 0xF1465711,
            0xF4094194, 0xF5CB2BA3, 0xF78D95FA, 0xF64FFFCD,
            0xD9785D60, 0xD8BA3757, 0xDAFC890E, 0xDB3EE339,
            0xDE71F5BC, 0xDFB39F8B, 0xDDF521D2, 0xDC374BE5,
            0xD76B0CD8, 0xD6A966EF, 0xD4EFD8B6, 0xD52DB281,
            0xD062A404, 0xD1A0CE33, 0xD3E6706A, 0xD2241A5D,
            0xC55EFE10, 0xC49C9427, 0xC6DA2A7E, 0xC7184049,
            0xC25756CC, 0xC3953CFB, 0xC1D382A2, 0xC011E895,
            0xCB4DAFA8, 0xCA8FC59F, 0xC8C97BC6, 0xC90B11F1,
            0xCC440774, 0xCD866D43, 0xCFC0D31A, 0xCE02B92D,
            0x91AF9640, 0x906DFC77, 0x922B422E, 0x93E92819,
            0x96A63E9C, 0x976454AB, 0x9522EAF2, 0x94E080C5,
            0x9FBCC7F8, 0x9E7EADCF, 0x9C381396, 0x9DFA79A1,
            0x98B56F24, 0x99770513, 0x9B31BB4A, 0x9AF3D17D,
            0x8D893530, 0x8C4B5F07, 0x8E0DE15E, 0x8FCF8B69,
            0x8A809DEC, 0x8B42F7DB, 0x89044982, 0x88C623B5,
            0x839A6488, 0x82580EBF, 0x801EB0E6, 0x81DCDAD1,
            0x8493CC54, 0x8551A663, 0x8717183A, 0x86D5720D,
            0xA9E2D0A0, 0xA820BA97, 0xAA6604CE, 0xABA46EF9,
            0xAEEB787C, 0xAF29124B, 0xAD6FAC12, 0xACADC625,
            0xA7F18118, 0xA633EB2F, 0xA4755576, 0xA5B73F41,
            0xA0F829C4, 0xA13A43F3, 0xA37CFDAA, 0xA2BE979D,
            0xB5C473D0, 0xB40619E7, 0xB640A7BE, 0xB782CD89,
            0xB2CDDB0C, 0xB30FB13B, 0xB1490F62, 0xB08B6555,
            0xBBD72268, 0xBA15485F, 0xB853F606, 0xB9919C31,
            0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED,
    /* T8_3 */
            0x00000000, 0xB8BC6765, 0xAA09C88B, 0x12B5AFEE,
            0x8F629757, 0x37DEF032, 0x256B5FDC, 0x9DD738B9,
            0xC5B428EF, 0x7D084F8A, 0x6FBDE064, 0xD7018701,
            0x4AD6BFB8, 0xF26AD8DD, 0xE0DF7733, 0x58631056,
            0x5019579F, 0xE8A530FA, 0xFA109F14, 0x42ACF871,
            0xDF7BC0C8, 0x67C7A7AD, 0x75720843, 0xCDCE6F26,
            0x95AD7F70, 0x2D111815, 0x3FA4B7FB, 0x8718D09E,
            0x1ACFE827, 0xA2738F42, 0xB0C620AC, 0x087A47C9,
            0xA032AF3E, 0x188EC85B, 0x0A3B67B5, 0xB28700D0,
            0x2F503869, 0x97EC5F0C, 0x8559F0E2, 0x3DE59787,
            0x658687D1, 0xDD3AE0B4, 0xCF8F4F5A, 0x7733283F,
            0xEAE41086, 0x525877E3, 0x40EDD80D, 0xF851BF68,
            0xF02BF8A1, 0x48979FC4, 0x5A22302A, 0xE29E574F,
            0x7F496FF6, 0xC7F50893, 0xD540A77D, 0x6DFCC018,
            0x359FD04E, 0x8D23B72B, 0x9F9618C5, 0x272A7FA0,
            0xBAFD4719, 0x0241207C, 0x10F48F92, 0xA848E8F7,
            0x9B14583D, 0x23A83F58, 0x311D90B6, 0x89A1F7D3,
            0x1476CF6A, 0xACCAA80F, 0xBE7F07E1, 0x06C36084,
            0x5EA070D2, 0xE61C17B7, 0xF4A9B859, 0x4C15DF3C,
            0xD1C2E785, 0x697E80E0, 0x7BCB2F0E, 0xC377486B,
            0xCB0D0FA2, 0x73B168C7, 0x6104C729, 0xD9B8A04C,
            0x446F98F5, 0xFCD3FF90, 0xEE66507E, 0x56DA371B,
            0x0EB9274D, 0xB6054028, 0xA4B0EFC6, 0x1C0C88A3,
            0x81DBB01A, 0x3967D77F, 0x2BD27891, 0x936E1FF4,
            0x3B26F703, 0x839A9066, 0x912F3F88, 0x299358ED,
            0xB4446054, 0x0CF80731, 0x1E4DA8DF, 0xA6F1CFBA,
            0xFE92DFEC, 0x462EB889, 0x549B1767, 0xEC277002,
            0x71F048BB, 0xC94C2FDE, 0xDBF98030, 0x6345E755,
            0x6B3FA09C, 0xD383C7F9, 0xC1366817, 0x798A0F72,
            0xE45D37CB, 0x5CE150AE, 0x4E54FF40, 0xF6E89825,
            0xAE8B8873, 0x1637EF16, 0x048240F8, 0xBC3E279D,
            0x21E91F24, 0x99557841, 0x8BE0D7AF, 0x335CB0CA,
            0xED59B63B, 0x55E5D15E, 0x47507EB0, 0xFFEC19D5,
            0x623B216C, 0xDA874609, 0xC832E9E7, 0x708E8E82,
            0x28ED9ED4, 0x9051F9B1, 0x82E4565F, 0x3A58313A,
            0xA78F0983, 0x1F336EE6, 0x0D86C108, 0xB53AA66D,
            0xBD40E1A4, 0x05FC86C1, 0x1749292F, 0xAFF54E4A,
            0x322276F3, 0x8A9E1196, 0x982BBE78, 0x2097D91D,
            0x78F4C94B, 0xC048AE2E, 0xD2FD01C0, 0x6A4166A5,
            0xF7965E1C, 0x4F2A3979, 0x5D9F9697, 0xE523F1F2,
            0x4D6B1905, 0xF5D77E60, 0xE762D18E, 0x5FDEB6EB,
            0xC2098E52, 0x7AB5E937, 0x680046D9, 0xD0BC21BC,
            0x88DF31EA, 0x3063568F, 0x22D6F961, 0x9A6A9E04,
            0x07BDA6BD, 0xBF01C1D8, 0xADB46E36, 0x15080953,
            0x1D724E9A, 0xA5CE29FF, 0xB77B8611, 0x0FC7E174,
            0x9210D9CD, 0x2AACBEA8, 0x38191146, 0x80A57623,
            0xD8C66675, 0x607A0110, 0x72CFAEFE, 0xCA73C99B,
            0x57A4F122, 0xEF189647, 0xFDAD39A9, 0x45115ECC,
            0x764DEE06, 0xCEF18963, 0xDC44268D, 0x64F841E8,
            0xF92F7951, 0x41931E34, 0x5326B1DA, 0xEB9AD6BF,
            0xB3F9C6E9, 0x0B45A18C, 0x19F00E62, 0xA14C6907,
            0x3C9B51BE, 0x842736DB, 0x96929935, 0x2E2EFE50,
            0x2654B999, 0x9EE8DEFC, 0x8C5D7112, 0x34E11677,
            0xA9362ECE, 0x118A49AB, 0x033FE645, 0xBB838120,
            0xE3E09176, 0x5B5CF613, 0x49E959FD, 0xF1553E98,
            0x6C820621, 0xD43E6144, 0xC68BCEAA, 0x7E37A9CF,
            0xD67F4138, 0x6EC3265D, 0x7C7689B3, 0xC4CAEED6,
            0x591DD66F, 0xE1A1B10A, 0xF3141EE4, 0x4BA87981,
            0x13CB69D7, 0xAB770EB2, 0xB9C2A15C, 0x017EC639,
            0x9CA9FE80, 0x241599E5, 0x36A0360B, 0x8E1C516E,
            0x866616A7, 0x3EDA71C2, 0x2C6FDE2C, 0x94D3B949,
            0x090481F0, 0xB1B8E695, 0xA30D497B, 0x1BB12E1E,
            0x43D23E48, 0xFB6E592D, 0xE9DBF6C3, 0x516791A6,
            0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1,
    /* T8_4 */
            0x00000000, 0x3D6029B0, 0x7AC05360, 0x47A07AD0,
            0xF580A6C0, 0xC8E08F70, 0x8F40F5A0, 0xB220DC10,
            0x30704BC1, 0x0D106271, 0x4AB018A1, 0x77D03111,
            0xC5F0ED01, 0xF890C4B1, 0xBF30BE61, 0x825097D1,
            0x60E09782, 0x5D80BE32, 0x1A20C4E2, 0x2740ED52,
            0x95603142, 0xA80018F2, 0xEFA06222, 0xD2C04B92,
            0x5090DC43, 0x6DF0F5F3, 0x2A508F23, 0x1730A693,
            0xA5107A83, 0x98705333, 0xDFD029E3, 0xE2B00053,
            0xC1C12F04, 0xFCA106B4, 0xBB017C64, 0x866155D4,
            0x344189C4, 0x0921A074, 0x4E81DAA4, 0x73E1F314,
            0xF1B164C5, 0xCCD14D75, 0x8B7137A5, 0xB6111E15,
            0x0431C205, 0x3951EBB5, 0x7EF19165, 0x4391B8D5,
            0xA121B886, 0x9C419136, 0xDBE1EBE6, 0xE681C256,
            0x54A11E46, 0x69C137F6, 0x2E614D26, 0x13016496,
            0x9151F347, 0xAC31DAF7, 0xEB91A027, 0xD6F18997,
            0x64D15587, 0x59B17C37, 0x1E1106E7, 0x23712F57,
            0x58F35849, 0x659371F9, 0x22330B29, 0x1F532299,
            0xAD73FE89, 0x9013D739, 0xD7B3ADE9, 0xEAD38459,
            0x68831388, 0x55E33A38, 0x124340E8, 0x2F236958,
            0x9D03B548, 0xA0639CF8, 0xE7C3E628, 0xDAA3CF98,
            0x3813CFCB, 0x0573E67B, 0x42D39CAB, 0x7FB3B51B,
            0xCD93690B, 0xF0F340BB, 0xB7533A6B, 0x8A3313DB,
            0x0863840A, 0x3503ADBA, 0x72A3D76A, 0x4FC3FEDA,
            0xFDE322CA, 0xC0830B7A, 0x872371AA, 0xBA43581A,
            0x9932774D, 0xA4525EFD, 0xE3F2242D, 0xDE920D9D,
            0x6CB2D18D, 0x51D2F83D, 0x167282ED, 0x2B12AB5D,
            0xA9423C8C, 0x9422153C, 0xD3826FEC, 0xEEE2465C,
            0x5CC29A4C, 0x61A2B3FC, 0x2602C92C, 0x1B62E09C,
            0xF9D2E0CF, 0xC4B2C97F, 0x8312B3AF, 0xBE729A1F,
            0x0C52460F, 0x31326FBF, 0x7692156F, 0x4BF23CDF,
            0xC9A2AB0E, 0xF4C282BE, 0xB362F86E, 0x8E02D1DE,
            0x3C220DCE, 0x0142247E, 0x46E25EAE, 0x7B82771E,
            0xB1E6B092, 0x8C869922, 0xCB26E3F2, 0xF646CA42,
            0x44661652, 0x79063FE2, 0x3EA64532, 0x03C66C82,
            0x8196FB53, 0xBCF6D2E3, 0xFB56A833, 0xC6368183,
            0x74165D93, 0x49767423, 0x0ED60EF3, 0x33B62743,
            0xD1062710, 0xEC660EA0, 0xABC67470, 0x96A65DC0,
            0x248681D0, 0x19E6A860, 0x5E46D2B0, 0x6326FB00,
            0xE1766CD1, 0xDC164561, 0x9BB63FB1, 0xA6D61601,
            0x14F6CA11, 0x2996E3A1, 0x6E369971, 0x5356B0C1,
            0x70279F96, 0x4D47B626, 0x0AE7CCF6, 0x3787E546,
            0x85A73956, 0xB8C710E6, 0xFF676A36, 0xC2074386,
            0x4057D457, 0x7D37FDE7, 0x3A978737, 0x07F7AE87,
            0xB5D77297, 0x88B75B27, 0xCF1721F7, 0xF2770847,
            0x10C70814, 0x2DA721A4, 0x6A075B74, 0x576772C4,
            0xE547AED4, 0xD8278764, 0x9F87FDB4, 0xA2E7D404,
            0x20B743D5, 0x1DD76A65, 0x5A7710B5, 0x67173905,
            0xD537E515, 0xE857CCA5, 0xAFF7B675, 0x92979FC5,
            0xE915E8DB, 0xD475C16B, 0x93D5BBBB, 0xAEB5920B,
            0x1C954E1B, 0x21F567AB, 0x66551D7B, 0x5B3534CB,
            0xD965A31A, 0xE4058AAA, 0xA3A5F07A, 0x9EC5D9CA,
            0x2CE505DA, 0x11852C6A, 0x562556BA, 0x6B457F0A,
            0x89F57F59, 0xB49556E9, 0xF3352C39, 0xCE550589,
            0x7C75D999, 0x4115F029, 0x06B58AF9, 0x3BD5A349,
            0xB9853498, 0x84E51D28, 0xC34567F8, 0xFE254E48,
            0x4C059258, 0x7165BBE8, 0x36C5C138, 0x0BA5E888,
            0x28D4C7DF, 0x15B4EE6F, 0x521494BF, 0x6F74BD0F,
            0xDD54611F, 0xE03448AF, 0xA794327F, 0x9AF41BCF,
            0x18A48C1E, 0x25C4A5AE, 0x6264DF7E, 0x5F04F6CE,
            0xED242ADE, 0xD044036E, 0x97E479BE, 0xAA84500E,
            0x4834505D, 0x755479ED, 0x32F4033D, 0x0F942A8D,
            0xBDB4F69D, 0x80D4DF2D, 0xC774A5FD, 0xFA148C4D,
            0x78441B9C, 0x4524322C, 0x028448FC, 0x3FE4614C,
            0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C,
    /* T8_5 */
            0x00000000, 0xCB5CD3A5, 0x4DC8A10B, 0x869472AE,
            0x9B914216, 0x50CD91B3, 0xD659E31D, 0x1D0530B8,
            0xEC53826D, 0x270F51C8, 0xA19B2366, 0x6AC7F0C3,
            0x77C2C07B, 0xBC9E13DE, 0x3A0A6170, 0xF156B2D5,
            0x03D6029B, 0xC88AD13E, 0x4E1EA390, 0x85427035,
            0x9847408D, 0x531B9328, 0xD58FE186, 0x1ED33223,
            0xEF8580F6, 0x24D95353, 0xA24D21FD, 0x6911F258,
            0x7414C2E0, 0xBF481145, 0x39DC63EB, 0xF280B04E,
            0x07AC0536, 0xCCF0D693, 0x4A64A43D, 0x81387798,
            0x9C3D4720, 0x57619485, 0xD1F5E62B, 0x1AA9358E,
            0xEBFF875B, 0x20A354FE, 0xA6372650, 0x6D6BF5F5,
            0x706EC54D, 0xBB3216E8, 0x3DA66446, 0xF6FAB7E3,
            0x047A07AD, 0xCF26D408, 0x49B2A6A6, 0x82EE7503,
            0x9FEB45BB, 0x54B7961E, 0xD223E4B0, 0x197F3715,
            0xE82985C0, 0x23755665, 0xA5E124CB, 0x6EBDF76E,
            0x73B8C7D6, 0xB8E41473, 0x3E7066DD, 0xF52CB578,
            0x0F580A6C, 0xC404D9C9, 0x4290AB67, 0x89CC78C2,
            0x94C9487A, 0x5F959BDF, 0xD901E971, 0x125D3AD4,
            0xE30B8801, 0x28575BA4, 0xAEC3290A, 0x659FFAAF,
            0x789ACA17, 0xB3C619B2, 0x35526B1C, 0xFE0EB8B9,
            0x0C8E08F7, 0xC7D2DB52, 0x4146A9FC, 0x8A1A7A59,
            0x971F4AE1, 0x5C439944, 0xDAD7EBEA, 0x118B384F,
            0xE0DD8A9A, 0x2B81593F, 0xAD152B91, 0x6649F834,
            0x7B4CC88C, 0xB0101B29, 0x36846987, 0xFDD8BA22,
            0x08F40F5A, 0xC3A8DCFF, 0x453CAE51, 0x8E607DF4,
            0x93654D4C, 0x58399EE9, 0xDEADEC47, 0x15F13FE2,
            0xE4A78D37, 0x2FFB5E92, 0xA96F2C3C, 0x6233FF99,
            0x7F36CF21, 0xB46A1C84, 0x32FE6E2A, 0xF9A2BD8F,
            0x0B220DC1, 0xC07EDE64, 0x46EAACCA, 0x8DB67F6F,
            0x90B34FD7, 0x5BEF9C72, 0xDD7BEEDC, 0x16273D79,
            0xE7718FAC, 0x2C2D5C09, 0xAAB92EA7, 0x61E5FD02,
            0x7CE0CDBA, 0xB7BC1E1F, 0x31286CB1, 0xFA74BF14,
            0x1EB014D8, 0xD5ECC77D, 0x5378B5D3, 0x98246676,
            0x852156CE, 0x4E7D856B, 0xC8E9F7C5, 0x03B52460,
            0xF2E396B5, 0x39BF4510, 0xBF2B37BE, 0x7477E41B,
            0x6972D4A3, 0xA22E0706, 0x24BA75A8, 0xEFE6A60D,
            0x1D661643, 0xD63AC5E6, 0x50AEB748, 0x9BF264ED,
            0x86F75455, 0x4DAB87F0, 0xCB3FF55E, 0x006326FB,
            0xF135942E, 0x3A69478B, 0xBCFD3525, 0x77A1E680,
            0x6AA4D638, 0xA1F8059D, 0x276C7733, 0xEC30A496,
            0x191C11EE, 0xD240C24B, 0x54D4B0E5, 0x9F886340,
            0x828D53F8, 0x49D1805D, 0xCF45F2F3, 0x04192156,
            0xF54F9383, 0x3E134026, 0xB8873288, 0x73DBE12D,
            0x6EDED195, 0xA5820230, 0x2316709E, 0xE84AA33B,
            0x1ACA1375, 0xD196C0D0, 0x5702B27E, 0x9C5E61DB,
            0x815B5163, 0x4A0782C6, 0xCC93F068, 0x07CF23CD,
            0xF6999118, 0x3DC542BD, 0xBB513013, 0x700DE3B6,
            0x6D08D30E, 0xA65400AB, 0x20C07205, 0xEB9CA1A0,
            0x11E81EB4, 0xDAB4CD11, 0x5C20BFBF, 0x977C6C1A,
            0x8A795CA2, 0x41258F07, 0xC7B1FDA9, 0x0CED2E0C,
            0xFDBB9CD9, 0x36E74F7C, 0xB0733DD2, 0x7B2FEE77,
            0x662ADECF, 0xAD760D6A, 0x2BE27FC4, 0xE0BEAC61,
            0x123E1C2F, 0xD962CF8A, 0x5FF6BD24, 0x94AA6E81,
            0x89AF5E39, 0x42F38D9C, 0xC467FF32, 0x0F3B2C97,
            0xFE6D9E42, 0x35314DE7, 0xB3A53F49, 0x78F9ECEC,
            0x65FCDC54, 0xAEA00FF1, 0x28347D5F, 0xE368AEFA,
            0x16441B82, 0xDD18C827, 0x5B8CBA89, 0x90D0692C,
            0x8DD55994, 0x46898A31, 0xC01DF89F, 0x0B412B3A,
            0xFA1799EF, 0x314B4A4A, 0xB7DF38E4, 0x7C83EB41,
            0x6186DBF9, 0xAADA085C, 0x2C4E7AF2, 0xE712A957,
            0x15921919, 0xDECECABC, 0x585AB812, 0x93066BB7,
            0x8E035B0F, 0x455F88AA, 0xC3CBFA04, 0x089729A1,
            0xF9C19B74, 0x329D48D1, 0xB4093A7F, 0x7F55E9DA,
            0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC,
    /* T8_6 */
            0x00000000, 0xA6770BB4, 0x979F1129, 0x31E81A9D,
            0xF44F2413, 0x52382FA7, 0x63D0353A, 0xC5A73E8E,
            0x33EF4E67, 0x959845D3, 0xA4705F4E, 0x020754FA,
            0xC7A06A74, 0x61D761C0, 0x503F7B5D, 0xF64870E9,
            0x67DE9CCE, 0xC1A9977A, 0xF0418DE7, 0x56368653,
            0x9391B8DD, 0x35E6B369, 0x040EA9F4, 0xA279A240,
            0x5431D2A9, 0xF246D91D, 0xC3AEC380, 0x65D9C834,
            0xA07EF6BA, 0x0609FD0E, 0x37E1E793, 0x9196EC27,
            0xCFBD399C, 0x69CA3228, 0x582228B5, 0xFE552301,
            0x3BF21D8F, 0x9D85163B, 0xAC6D0CA6, 0x0A1A0712,
            0xFC5277FB, 0x5A257C4F, 0x6BCD66D2, 0xCDBA6D66,
            0x081D53E8, 0xAE6A585C, 0x9F8242C1, 0x39F54975,
            0xA863A552, 0x0E14AEE6, 0x3FFCB47B, 0x998BBFCF,
            0x5C2C8141, 0xFA5B8AF5, 0xCBB39068, 0x6DC49BDC,
            0x9B8CEB35, 0x3DFBE081, 0x0C13FA1C, 0xAA64F1A8,
            0x6FC3CF26, 0xC9B4C492, 0xF85CDE0F, 0x5E2BD5BB,
            0x440B7579, 0xE27C7ECD, 0xD3946450, 0x75E36FE4,
            0xB044516A, 0x16335ADE, 0x27DB4043, 0x81AC4BF7,
            0x77E43B1E, 0xD19330AA, 0xE07B2A37, 0x460C2183,
            0x83AB1F0D, 0x25DC14B9, 0x14340E24, 0xB2430590,
            0x23D5E9B7, 0x85A2E203, 0xB44AF89E, 0x123DF32A,
            0xD79ACDA4, 0x71EDC610, 0x4005DC8D, 0xE672D739,
            0x103AA7D0, 0xB64DAC64, 0x87A5B6F9, 0x21D2BD4D,
            0xE47583C3, 0x42028877, 0x73EA92EA, 0xD59D995E,
            0x8BB64CE5, 0x2DC14751, 0x1C295DCC, 0xBA5E5678,
            0x7FF968F6, 0xD98E6342, 0xE86679DF, 0x4E11726B,
            0xB8590282, 0x1E2E0936, 0x2FC613AB, 0x89B1181F,
            0x4C162691, 0xEA612D25, 0xDB8937B8, 0x7DFE3C0C,
            0xEC68D02B, 0x4A1FDB9F, 0x7BF7C102, 0xDD80CAB6,
            0x1827F438, 0xBE50FF8C, 0x8FB8E511, 0x29CFEEA5,
            0xDF879E4C, 0x79F095F8, 0x48188F65, 0xEE6F84D1,
            0x2BC8BA5F, 0x8DBFB1EB, 0xBC57AB76, 0x1A20A0C2,
            0x8816EAF2, 0x2E61E146, 0x1F89FBDB, 0xB9FEF06F,
            0x7C59CEE1, 0xDA2EC555, 0xEBC6DFC8, 0x4DB1D47C,
            0xBBF9A495, 0x1D8EAF21, 0x2C66B5BC, 0x8A11BE08,
            0x4FB68086, 0xE9C18B32, 0xD82991AF, 0x7E5E9A1B,
            0xEFC8763C, 0x49BF7D88, 0x78576715, 0xDE206CA1,
            0x1B87522F, 0xBDF0599B, 0x8C184306, 0x2A6F48B2,
            0xDC27385B, 0x7A5033EF, 0x4BB82972, 0xEDCF22C6,
            0x28681C48, 0x8E1F17FC, 0xBFF70D61, 0x198006D5,
            0x47ABD36E, 0xE1DCD8DA, 0xD034C247, 0x7643C9F3,
            0xB3E4F77D, 0x1593FCC9, 0x247BE654, 0x820CEDE0,
            0x74449D09, 0xD23396BD, 0xE3DB8C20, 0x45AC8794,
            0x800BB91A, 0x267CB2AE, 0x1794A833, 0xB1E3A387,
            0x20754FA0, 0x86024414, 0xB7EA5E89, 0x119D553D,
            0xD43A6BB3, 0x724D6007, 0x43A57A9A, 0xE5D2712E,
            0x139A01C7, 0xB5ED0A73, 0x840510EE, 0x22721B5A,
            0xE7D525D4, 0x41A22E60, 0x704A34FD, 0xD63D3F49,
            0xCC1D9F8B, 0x6A6A943F, 0x5B828EA2, 0xFDF58516,
            0x3852BB98, 0x9E25B02C, 0xAFCDAAB1, 0x09BAA105,
            0xFFF2D1EC, 0x5985DA58, 0x686DC0C5, 0xCE1ACB71,
            0x0BBDF5FF, 0xADCAFE4B, 0x9C22E4D6, 0x3A55EF62,
            0xABC30345, 0x0DB408F1, 0x3C5C126C, 0x9A2B19D8,
            0x5F8C2756, 0xF9FB2CE2, 0xC813367F, 0x6E643DCB,
            0x982C4D22, 0x3E5B4696, 0x0FB35C0B, 0xA9C457BF,
            0x6C636931, 0xCA146285, 0xFBFC7818, 0x5D8B73AC,
            0x03A0A617, 0xA5D7ADA3, 0x943FB73E, 0x3248BC8A,
            0xF7EF8204, 0x519889B0, 0x6070932D, 0xC6079899,
            0x304FE870, 0x9638E3C4, 0xA7D0F959, 0x01A7F2ED,
            0xC400CC63, 0x6277C7D7, 0x539FDD4A, 0xF5E8D6FE,
            0x647E3AD9, 0xC209316D, 0xF3E12BF0, 0x55962044,
            0x90311ECA, 0x3646157E, 0x07AE0FE3, 0xA1D90457,
            0x579174BE, 0xF1E67F0A, 0xC00E6597, 0x66796E23,
            0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30,
    /* T8_7 */
            0x00000000, 0xCCAA009E, 0x4225077D, 0x8E8F07E3,
            0x844A0EFA, 0x48E00E64, 0xC66F0987, 0x0AC50919,
            0xD3E51BB5, 0x1F4F1B2B, 0x91C01CC8, 0x5D6A1C56,
            0x57AF154F, 0x9B0515D1, 0x158A1232, 0xD92012AC,
            0x7CBB312B, 0xB01131B5, 0x3E9E3656, 0xF23436C8,
            0xF8F13FD1, 0x345B3F4F, 0xBAD438AC, 0x767E3832,
            0xAF5E2A9E, 0x63F42A00, 0xED7B2DE3, 0x21D12D7D,
            0x2B142464, 0xE7BE24FA, 0x69312319, 0xA59B2387,
            0xF9766256, 0x35DC62C8, 0xBB53652B, 0x77F965B5,
            0x7D3C6CAC, 0xB1966C32, 0x3F196BD1, 0xF3B36B4F,
            0x2A9379E3, 0xE639797D, 0x68B67E9E, 0xA41C7E00,
            0xAED97719, 0x62737787, 0xECFC7064, 0x205670FA,
            0x85CD537D, 0x496753E3, 0xC7E85400, 0x0B42549E,
            0x01875D87, 0xCD2D5D19, 0x43A25AFA, 0x8F085A64,
            0x562848C8, 0x9A824856, 0x140D4FB5, 0xD8A74F2B,
            0xD2624632, 0x1EC846AC, 0x9047414F, 0x5CED41D1,
            0x299DC2ED, 0xE537C273, 0x6BB8C590, 0xA712C50E,
            0xADD7CC17, 0x617DCC89, 0xEFF2CB6A, 0x2358CBF4,
            0xFA78D958, 0x36D2D9C6, 0xB85DDE25, 0x74F7DEBB,
            0x7E32D7A2, 0xB298D73C, 0x3C17D0DF, 0xF0BDD041,
            0x5526F3C6, 0x998CF358, 0x1703F4BB, 0xDBA9F425,
            0xD16CFD3C, 0x1DC6FDA2, 0x9349FA41, 0x5FE3FADF,
            0x86C3E873, 0x4A69E8ED, 0xC4E6EF0E, 0x084CEF90,
            0x0289E689, 0xCE23E617, 0x40ACE1F4, 0x8C06E16A,
            0xD0EBA0BB, 0x1C41A025, 0x92CEA7C6, 0x5E64A758,
            0x54A1AE41, 0x980BAEDF, 0x1684A93C, 0xDA2EA9A2,
            0x030EBB0E, 0xCFA4BB90, 0x412BBC73, 0x8D81BCED,
            0x8744B5F4, 0x4BEEB56A, 0xC561B289, 0x09CBB217,
            0xAC509190, 0x60FA910E, 0xEE7596ED, 0x22DF9673,
            0x281A9F6A, 0xE4B09FF4, 0x6A3F9817, 0xA6959889,
            0x7FB58A25, 0xB31F8ABB, 0x3D908D58, 0xF13A8DC6,
            0xFBFF84DF, 0x37558441, 0xB9DA83A2, 0x7570833C,
            0x533B85DA, 0x9F918544, 0x111E82A7, 0xDDB48239,
            0xD7718B20, 0x1BDB8BBE, 0x95548C5D, 0x59FE8CC3,
            0x80DE9E6F, 0x4C749EF1, 0xC2FB9912, 0x0E51998C,
            0x04949095, 0xC83E900B, 0x46B197E8, 0x8A1B9776,
            0x2F80B4F1, 0xE32AB46F, 0x6DA5B38C, 0xA10FB312,
            0xABCABA0B, 0x6760BA95, 0xE9EFBD76, 0x2545BDE8,
            0xFC65AF44, 0x30CFAFDA, 0xBE40A839, 0x72EAA8A7,
            0x782FA1BE, 0xB485A120, 0x3A0AA6C3, 0xF6A0A65D,
            0xAA4DE78C, 0x66E7E712, 0xE868E0F1, 0x24C2E06F,
            0x2E07E976, 0xE2ADE9E8, 0x6C22EE0B, 0xA088EE95,
            0x79A8FC39, 0xB502FCA7, 0x3B8DFB44, 0xF727FBDA,
            0xFDE2F2C3, 0x3148F25D, 0xBFC7F5BE, 0x736DF520,
            0xD6F6D6A7, 0x1A5CD639, 0x94D3D1DA, 0x5879D144,
            0x52BCD85D, 0x9E16D8C3, 0x1099DF20, 0xDC33DFBE,
            0x0513CD12, 0xC9B9CD8C, 0x4736CA6F, 0x8B9CCAF1,
            0x8159C3E8, 0x4DF3C376, 0xC37CC495, 0x0FD6C40B,
            0x7AA64737, 0xB60C47A9, 0x3883404A, 0xF42940D4,
            0xFEEC49CD, 0x32464953, 0xBCC94EB0, 0x70634E2E,
            0xA9435C82, 0x65E95C1C, 0xEB665BFF, 0x27CC5B61,
            0x2D095278, 0xE1A352E6, 0x6F2C5505, 0xA386559B,
            0x061D761C, 0xCAB77682, 0x44387161, 0x889271FF,
            0x825778E6, 0x4EFD7878, 0xC0727F9B, 0x0CD87F05,
            0xD5F86DA9, 0x19526D37, 0x97DD6AD4, 0x5B776A4A,
            0x51B26353, 0x9D1863CD, 0x1397642E, 0xDF3D64B0,
            0x83D02561, 0x4F7A25FF, 0xC1F5221C, 0x0D5F2282,
            0x079A2B9B, 0xCB302B05, 0x45BF2CE6, 0x89152C78,
            0x50353ED4, 0x9C9F3E4A, 0x121039A9, 0xDEBA3937,
            0xD47F302E, 0x18D530B0, 0x965A3753, 0x5AF037CD,
            0xFF6B144A, 0x33C114D4, 0xBD4E1337, 0x71E413A9,
            0x7B211AB0, 0xB78B1A2E, 0x39041DCD, 0xF5AE1D53,
            0x2C8E0FFF, 0xE0240F61, 0x6EAB0882, 0xA201081C,
            0xA8C40105, 0x646E019B, 0xEAE10678, 0x264B06E6
    };

    public static void main(String[] args) {
        Set xx=new HashSet();
        long start=System.currentTimeMillis();
        int i1 = 100000000;
        for (int i = 0; i< i1; i++)
        {
            PureJavaCrc32 crc32 = new PureJavaCrc32();
            byte[] bytes = String.valueOf(i).getBytes();
            crc32.update(bytes,0,bytes.length);
            long x = crc32.getValue();
            x=x%65535;
           if(xx.contains(x))
            {
                System.out.println(x);
            }

           // xx.add(x);

        }
        long used=System.currentTimeMillis()-start;

        System.out.println("tps "+i1*1000.0/used);
        System.out.println(".............................."+xx.size());

    }
}


================================================
FILE: src/main/java/io/mycat/route/function/ReloadFunction.java
================================================
package io.mycat.route.function;

/**
 * Created by magicdoom on 2016/9/17.
 */
public interface ReloadFunction {
   void reload();
}


================================================
FILE: src/main/java/io/mycat/route/function/SlotFunction.java
================================================
package io.mycat.route.function;

/**
 * Created by magicdoom on 2016/9/17.
 */
public interface SlotFunction {
    int slotValue();
}


================================================
FILE: src/main/java/io/mycat/route/function/TableRuleAware.java
================================================
package io.mycat.route.function;

import io.mycat.config.model.TableConfig;

/**
 * Created by magicdoom on 2016/9/5.
 * 考虑一类新分片算法     属于有状态算法
 * 比如PartitionByCRC32PreSlot 如果迁移过数据的话，slot映射规则会进行改变
 * 所以必须对应一张表单独一个实例      实现此接口后会根据不同表自动创建新实例
 */
public interface TableRuleAware{
    void setTableConfig(TableConfig tableConfig);

    void setRuleName(String ruleName);

    TableConfig getTableConfig();

    String getRuleName();

    /**
     * 如果是实例则返回true,不是实例则是false
     * cjw qq:294712221
     * @return
     */
    boolean isIstance();

}


================================================
FILE: src/main/java/io/mycat/route/handler/HintCatletHandler.java
================================================
package io.mycat.route.handler;

import java.sql.SQLNonTransientException;
import java.util.Map;

import org.slf4j.Logger; import org.slf4j.LoggerFactory;

import io.mycat.MycatServer;
import io.mycat.cache.LayerCachePool;
import io.mycat.catlets.Catlet;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.SystemConfig;
import io.mycat.route.RouteResultset;
import io.mycat.server.ServerConnection;
import io.mycat.sqlengine.EngineCtx;

/**
 * 处理注释中类型为catlet 的情况,每个catlet为一个用户自定义Java代码类，用于进行复杂查询SQL（只能是查询SQL）的自定义执行过程，
 * 目前主要用于跨分片Join的人工智能编码
 */
public class HintCatletHandler implements HintHandler {

	private static final Logger LOGGER = LoggerFactory.getLogger(HintCatletHandler.class);

	/**
	 * 从全局的schema列表中查询指定的schema是否存在， 如果存在则替换connection属性中原有的schema，
	 * 如果不存在，则throws SQLNonTransientException，表示指定的schema 不存在
	 * 
	 * @param sysConfig
	 * @param schema
	 * @param sqlType
	 * @param realSQL
	 * @param charset
	 * @param info
	 * @param cachePool
	 * @param hintSQLValue
	 * @return
	 * @throws SQLNonTransientException
	 */
	@Override
	public RouteResultset route(SystemConfig sysConfig, SchemaConfig schema,
			int sqlType, String realSQL, String charset, ServerConnection sc,
			LayerCachePool cachePool, String hintSQLValue,int hintSqlType, Map hintMap)
			throws SQLNonTransientException {
		// sc.setEngineCtx ctx
		String cateletClass = hintSQLValue;
		if (LOGGER.isDebugEnabled()) {
			LOGGER.debug("load catelet class:" + hintSQLValue + " to run sql "
					+ realSQL);
		}
		try {
			Catlet catlet = (Catlet) MycatServer.getInstance()
					.getCatletClassLoader().getInstanceofClass(cateletClass);
			catlet.route(sysConfig, schema, sqlType, realSQL,charset, sc, cachePool);
			catlet.processSQL(realSQL, new EngineCtx(sc.getSession2()));
		} catch (Exception e) {
			LOGGER.warn("catlet error "+e);
			throw new SQLNonTransientException(e);
		}
		return null;
	}
}


================================================
FILE: src/main/java/io/mycat/route/handler/HintDataNodeHandler.java
================================================
package io.mycat.route.handler;

import java.sql.SQLNonTransientException;
import java.util.Map;

import com.alibaba.druid.sql.ast.expr.SQLTextLiteralExpr;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlLoadDataInFileStatement;
import com.alibaba.druid.sql.parser.SQLStatementParser;
import io.mycat.route.parser.druid.MycatStatementParser;
import io.mycat.server.parser.ServerParse;
import io.mycat.sqlengine.mpp.LoadData;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.LoggerFactory;

import io.mycat.MycatServer;
import io.mycat.backend.datasource.PhysicalDBNode;
import io.mycat.cache.LayerCachePool;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.SystemConfig;
import io.mycat.route.RouteResultset;
import io.mycat.route.util.RouterUtil;
import io.mycat.server.ServerConnection;

/**
 * 处理注释中类型为datanode 的情况
 * 
 * @author zhuam
 */
public class HintDataNodeHandler implements HintHandler {
	
	private static final Logger LOGGER = LoggerFactory.getLogger(HintSchemaHandler.class);

	@Override
	public RouteResultset route(SystemConfig sysConfig, SchemaConfig schema, int sqlType, String realSQL,
			String charset, ServerConnection sc, LayerCachePool cachePool, String hintSQLValue,int hintSqlType, Map hintMap)
					throws SQLNonTransientException {
		
		String stmt = realSQL;
		
		if (LOGGER.isDebugEnabled()) {
			LOGGER.debug("route datanode sql hint from " + stmt);
		}
		
		RouteResultset rrs = new RouteResultset(stmt, sqlType);		
		PhysicalDBNode dataNode = MycatServer.getInstance().getConfig().getDataNodes().get(hintSQLValue);
		if (dataNode != null) {			
			rrs = RouterUtil.routeToSingleNode(rrs, dataNode.getName(), stmt);
		} else {
			String msg = "can't find hint datanode:" + hintSQLValue;
			LOGGER.warn(msg);
			throw new SQLNonTransientException(msg);
		}

		// 处理导入参数初始化
		if(rrs.getSqlType() == ServerParse.LOAD_DATA_INFILE_SQL){
			LOGGER.info("load data use annotation datanode");
			rrs.getNodes()[0].setLoadData(parseLoadDataPram(stmt , charset));
		}
		
		return rrs;
	}

	// 初始化导入参数
	private LoadData parseLoadDataPram(String sql , String connectionCharset)
	{
		SQLStatementParser parser = new MycatStatementParser(sql);
		MySqlLoadDataInFileStatement statement = (MySqlLoadDataInFileStatement) parser.parseStatement();

		LoadData loadData = new LoadData();
		SQLTextLiteralExpr rawLineEnd = (SQLTextLiteralExpr) statement.getLinesTerminatedBy();
		String lineTerminatedBy = rawLineEnd == null ? "\n" : rawLineEnd.getText();
		loadData.setLineTerminatedBy(lineTerminatedBy);

		SQLTextLiteralExpr rawFieldEnd = (SQLTextLiteralExpr) statement.getColumnsTerminatedBy();
		String fieldTerminatedBy = rawFieldEnd == null ? "\t" : rawFieldEnd.getText();
		loadData.setFieldTerminatedBy(fieldTerminatedBy);

		SQLTextLiteralExpr rawEnclosed = (SQLTextLiteralExpr) statement.getColumnsEnclosedBy();
		String enclose = rawEnclosed == null ? null : rawEnclosed.getText();
		loadData.setEnclose(enclose);

		SQLTextLiteralExpr escapseExpr =  (SQLTextLiteralExpr)statement.getColumnsEscaped() ;
		String escapse=escapseExpr==null?"\\":escapseExpr.getText();
		loadData.setEscape(escapse);
		String charset = statement.getCharset() != null ? statement.getCharset() : connectionCharset;
		loadData.setCharset(charset);

		String fileName = parseFileName(sql);
		if(StringUtils.isBlank(fileName)){
			throw new RuntimeException(" file name is null !");
		}

		loadData.setFileName(fileName);

		return loadData ;
	}

	// 处理文件名
	private String parseFileName(String sql)
	{
		if (sql.contains("'"))
		{
			int beginIndex = sql.indexOf("'");
			return sql.substring(beginIndex + 1, sql.indexOf("'", beginIndex + 1));
		} else if (sql.contains("\""))
		{
			int beginIndex = sql.indexOf("\"");
			return sql.substring(beginIndex + 1, sql.indexOf("\"", beginIndex + 1));
		}
		return null;
	}

}


================================================
FILE: src/main/java/io/mycat/route/handler/HintHandler.java
================================================
package io.mycat.route.handler;

import java.sql.SQLNonTransientException;
import java.util.Map;

import io.mycat.cache.LayerCachePool;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.SystemConfig;
import io.mycat.route.RouteResultset;
import io.mycat.server.ServerConnection;

/**
 * 按照注释中包含指定类型的内容做路由解析
 * 
 */
public interface HintHandler {

	public RouteResultset route(SystemConfig sysConfig, SchemaConfig schema,
                                int sqlType, String realSQL, String charset, ServerConnection sc,
                                LayerCachePool cachePool, String hintSQLValue, int hintSqlType, Map hintMap)
			throws SQLNonTransientException;
}


================================================
FILE: src/main/java/io/mycat/route/handler/HintHandlerFactory.java
================================================
package io.mycat.route.handler;

import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.FutureTask;

public class HintHandlerFactory {
	
	private static volatile boolean isInit = false;
	
	 //sql注释的类型处理handler 集合，现在支持两种类型的处理：sql,schema
    private static Map hintHandlerMap = new HashMap();

    private HintHandlerFactory() {
    }
    
    private static void init() {
        hintHandlerMap.put("sql",new HintSQLHandler());
        hintHandlerMap.put("schema",new HintSchemaHandler());
        hintHandlerMap.put("datanode",new HintDataNodeHandler());
        hintHandlerMap.put("catlet",new HintCatletHandler());
        
        // 新增sql hint（注解）/*#mycat:db_type=master*/ 和 /*#mycat:db_type=slave*/  和 /*mycat:db_type=slave*/
        // 该hint可以和 /*balance*/ 一起使用
        // 实现强制走 master 和 强制走 slave
        hintHandlerMap.put("db_type", new HintMasterDBHandler());
        isInit = true;	// 修复多次初始化的bug
    }
    
    // 双重校验锁 fix 线程安全问题
    public static HintHandler getHintHandler(String hintType) {
    	if(!isInit) {
    		synchronized(HintHandlerFactory.class){
    			if(!isInit) {
                    init();
                }
    		}
    	}
    	return hintHandlerMap.get(hintType);
    }
    
}


================================================
FILE: src/main/java/io/mycat/route/handler/HintMasterDBHandler.java
================================================
package io.mycat.route.handler;


import java.sql.SQLNonTransientException;
import java.util.Map;

import org.slf4j.Logger; import org.slf4j.LoggerFactory;
import io.mycat.cache.LayerCachePool;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.SystemConfig;
import io.mycat.route.RouteResultset;
import io.mycat.route.factory.RouteStrategyFactory;
import io.mycat.server.ServerConnection;
import io.mycat.server.parser.ServerParse;

/**
 * 处理情况 sql hint: mycat:db_type=master/slave

 * 后期可能会考虑增加 mycat:db_type=slave_newest，实现走延迟最小的slave
 * @author digdeep@126.com
 */
// /*#mycat:db_type=master*/
// /*#mycat:db_type=slave*/
// /*#mycat:db_type=slave_newest*/
// 强制走 master 和 强制走 slave
public class HintMasterDBHandler implements HintHandler {
	
	private static final Logger LOGGER = LoggerFactory.getLogger(HintMasterDBHandler.class);

	@Override
	public RouteResultset route(SystemConfig sysConfig, SchemaConfig schema, int sqlType, 
			String realSQL, String charset,
			ServerConnection sc, LayerCachePool cachePool, String hintSQLValue, int hintSqlType, Map hintMap)
			throws SQLNonTransientException {
		
//		LOGGER.debug("realSQL: " + realSQL); // select * from travelrecord limit 1
//		LOGGER.debug("sqlType: " + sqlType); // 7
//		LOGGER.debug("schema.getName(): " + schema.getName()); // TESTDB
//		LOGGER.debug("schema.getName(): " + schema.getDataNode()); // null
//		LOGGER.debug("hintSQLValue: " + hintSQLValue); // master/slave
		
		RouteResultset rrs = RouteStrategyFactory.getRouteStrategy()
									.route(sysConfig, schema, sqlType, 
										realSQL, charset, sc, cachePool);
		
		LOGGER.debug("schema.rrs(): " + rrs); // master
		Boolean isRouteToMaster = null;	// 默认不施加任何影响
		
		LOGGER.debug("hintSQLValue:::::::::" + hintSQLValue); // slave
		
		if(hintSQLValue != null && !hintSQLValue.trim().equals("")){
			if(hintSQLValue.trim().equalsIgnoreCase("master")) {
				isRouteToMaster = true;
			}
			if(hintSQLValue.trim().equalsIgnoreCase("slave")){
//				if(rrs.getCanRunInReadDB() != null && !rrs.getCanRunInReadDB()){
//					isRouteToMaster = null;
//					LOGGER.warn(realSQL + " can not run in slave.");
//				}else{
//					isRouteToMaster = false;
//				}
				if(sqlType == ServerParse.DELETE || sqlType == ServerParse.INSERT
						||sqlType == ServerParse.REPLACE || sqlType == ServerParse.UPDATE
						|| sqlType == ServerParse.DDL){
					LOGGER.error("should not use hint 'db_type' to route 'delete', 'insert', 'replace', 'update', 'ddl' to a slave db.");
					isRouteToMaster = null;	// 不施加任何影响
				}else{
					isRouteToMaster = false;
				}
			}
		}
		
		if(isRouteToMaster == null){	// 默认不施加任何影响
			LOGGER.warn(" sql hint 'db_type' error, ignore this hint.");
			return rrs;
		}
		
		if(isRouteToMaster)	 {// 强制走 master
			rrs.setRunOnSlave(false);
		}
		
		if(!isRouteToMaster) {// 强制走slave
			rrs.setRunOnSlave(true);
		}
		
		LOGGER.debug("rrs.getRunOnSlave():" + rrs.getRunOnSlaveDebugInfo());
		return rrs;
	}

}

================================================
FILE: src/main/java/io/mycat/route/handler/HintSQLHandler.java
================================================
package io.mycat.route.handler;

import java.sql.SQLNonTransientException;
import java.sql.Types;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr;
import com.alibaba.druid.sql.ast.expr.SQLNumberExpr;
import com.alibaba.druid.sql.ast.expr.SQLTextLiteralExpr;
import com.alibaba.druid.sql.ast.expr.SQLValuableExpr;
import com.alibaba.druid.sql.ast.statement.*;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.druid.sql.parser.SQLStatementParser;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import io.mycat.cache.LayerCachePool;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.SystemConfig;
import io.mycat.route.*;
import io.mycat.route.factory.RouteStrategyFactory;
import io.mycat.server.ServerConnection;
import io.mycat.server.parser.ServerParse;

/**
 * 处理注释中 类型为sql的情况 （按照 注释中的sql做路由解析，而不是实际的sql）
 */
public class HintSQLHandler implements HintHandler {
	
	private RouteStrategy routeStrategy;
	
	public HintSQLHandler() {
		this.routeStrategy = RouteStrategyFactory.getRouteStrategy();
	}

	@Override
	public RouteResultset route(SystemConfig sysConfig, SchemaConfig schema,
			int sqlType, String realSQL, String charset, ServerConnection sc,
			LayerCachePool cachePool, String hintSQLValue,int hintSqlType, Map hintMap)
            throws SQLNonTransientException {
		
		RouteResultset rrs = routeStrategy.route(sysConfig, schema, hintSqlType,
				hintSQLValue, charset, sc, cachePool);
		
		// 替换RRS中的SQL执行
		RouteResultsetNode[] oldRsNodes = rrs.getNodes();
		RouteResultsetNode[] newRrsNodes = new RouteResultsetNode[oldRsNodes.length];
		for (int i = 0; i < newRrsNodes.length; i++) {
			newRrsNodes[i] = new RouteResultsetNode(oldRsNodes[i].getName(),
					oldRsNodes[i].getSqlType(), realSQL);
            newRrsNodes[i].setSlot(oldRsNodes[i].getSlot());
		}
		rrs.setNodes(newRrsNodes);

		// 判断是否为调用存储过程的SQL语句，这里不能用SQL解析器来解析判断是否为CALL语句
		if (ServerParse.CALL == sqlType) {
			rrs.setCallStatement(true);

             Procedure procedure=parseProcedure(realSQL,hintMap);
            rrs.setProcedure(procedure);
        //    String sql=procedure.toChangeCallSql(null);
            String sql=realSQL;
            for (RouteResultsetNode node : rrs.getNodes())
            {
                node.setProcedure(procedure);
                node.setHintMap(hintMap);
                node.setStatement(sql);
            }

		}

		return rrs;
	}



    private   Procedure parseProcedure(String sql,Map hintMap)
    {
        boolean fields = hintMap.containsKey("list_fields");
        boolean isResultList= hintMap != null && ("list".equals(hintMap.get("result_type"))|| fields);
        Procedure procedure=new Procedure();
        procedure.setOriginSql(sql);
        procedure.setResultList(isResultList);
        List sqls= Splitter.on(";").trimResults().splitToList(sql)    ;
        Set outSet=new HashSet<>();
        for (int i = sqls.size() - 1; i >= 0; i--)
        {
            String s = sqls.get(i);
            if(Strings.isNullOrEmpty(s)) {
                continue;
            }
            SQLStatementParser parser = new MySqlStatementParser(s);
            SQLStatement statement = parser.parseStatement();
            if(statement instanceof SQLSelectStatement)
            {
                MySqlSelectQueryBlock selectQuery= (MySqlSelectQueryBlock) ((SQLSelectStatement) statement).getSelect().getQuery();
                if(selectQuery!=null)
                {
                    List selectItems=   selectQuery.getSelectList();
                    for (SQLSelectItem selectItem : selectItems)
                    {
                        String select = selectItem.toString();
                        outSet.add(select) ;
                        procedure.getSelectColumns().add(select);
                    }
                }
               procedure.setSelectSql(s);
            }  else  if(statement instanceof SQLCallStatement)
            {
                SQLCallStatement sqlCallStatement = (SQLCallStatement) statement;
                procedure.setName(sqlCallStatement.getProcedureName().getSimpleName());
                List paramterList= sqlCallStatement.getParameters();
                for (int i1 = 0; i1 < paramterList.size(); i1++)
                {
                    SQLExpr sqlExpr = paramterList.get(i1);
                    String pName = sqlExpr.toString();
                    String pType=outSet.contains(pName)? ProcedureParameter.OUT:ProcedureParameter.IN;
                    ProcedureParameter parameter=new ProcedureParameter();
                    parameter.setIndex(i1+1);
                    parameter.setName(pName);
                    parameter.setParameterType(pType);
                    if(pName.startsWith("@"))
                    {
                        procedure.getParamterMap().put(pName, parameter);
                    }   else
                    {
                        procedure.getParamterMap().put(String.valueOf(i1+1), parameter);
                    }


                }
                procedure.setCallSql(s);
            }   else  if(statement instanceof SQLSetStatement)
            {
                procedure.setSetSql(s);
                SQLSetStatement setStatement= (SQLSetStatement) statement;
                List sets= setStatement.getItems();
                for (SQLAssignItem set : sets)
                {
                    String name=set.getTarget().toString();
                     SQLExpr value=set.getValue();
                    ProcedureParameter parameter = procedure.getParamterMap().get(name);
                    if(parameter!=null)
                    {
                        if (value instanceof SQLIntegerExpr)
                        {
                           parameter.setValue(((SQLIntegerExpr) value).getNumber());
                            parameter.setJdbcType(Types.INTEGER);
                        }  else   if(value instanceof SQLNumberExpr)
                        {
                            parameter.setValue(((SQLNumberExpr) value).getNumber());
                            parameter.setJdbcType(Types.NUMERIC);
                        }
                        else if(value instanceof SQLTextLiteralExpr)
                        {
                            parameter.setValue(((SQLTextLiteralExpr) value).getText());
                            parameter.setJdbcType(Types.VARCHAR);
                        }
                        else
                        if (value instanceof SQLValuableExpr)
                        {
                            parameter.setValue(((SQLValuableExpr) value).getValue());
                            parameter.setJdbcType(Types.VARCHAR);
                        }
                    }
                }
            }

        }
        if(fields)
        {
            String list_fields =(String) hintMap.get("list_fields");
            List listFields = Splitter.on(",").trimResults().splitToList( list_fields);
            for (String field : listFields)
            {
                if(!procedure.getParamterMap().containsKey(field))
                {
                    ProcedureParameter parameter=new ProcedureParameter();
                    parameter.setParameterType(ProcedureParameter.OUT);
                    parameter.setName(field);
                    parameter.setJdbcType(-10);
                    parameter.setIndex(procedure.getParamterMap().size()+1);
                    procedure.getParamterMap().put(field,parameter);
                }
            }
            procedure.getListFields().addAll(listFields);
        }
        return procedure;
    }
}


================================================
FILE: src/main/java/io/mycat/route/handler/HintSchemaHandler.java
================================================
package io.mycat.route.handler;

import java.sql.SQLNonTransientException;
import java.util.Map;

import org.slf4j.Logger; import org.slf4j.LoggerFactory;

import io.mycat.MycatServer;
import io.mycat.cache.LayerCachePool;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.SystemConfig;
import io.mycat.route.RouteResultset;
import io.mycat.route.RouteStrategy;
import io.mycat.route.factory.RouteStrategyFactory;
import io.mycat.server.ServerConnection;

/**
 * 处理注释中类型为schema 的情况（按照指定schema做路由解析）
 */
public class HintSchemaHandler implements HintHandler {

	private static final Logger LOGGER = LoggerFactory.getLogger(HintSchemaHandler.class);

	private RouteStrategy routeStrategy;
    
    public HintSchemaHandler() {
		this.routeStrategy = RouteStrategyFactory.getRouteStrategy();
	}
	/**
	 * 从全局的schema列表中查询指定的schema是否存在， 如果存在则替换connection属性中原有的schema，
	 * 如果不存在，则throws SQLNonTransientException，表示指定的schema 不存在
	 * 
	 * @param sysConfig
	 * @param schema
	 * @param sqlType
	 * @param realSQL
	 * @param charset
	 * @param info
	 * @param cachePool
	 * @param hintSQLValue
	 * @return
	 * @throws SQLNonTransientException
	 */
	@Override
	public RouteResultset route(SystemConfig sysConfig, SchemaConfig schema,
			int sqlType, String realSQL, String charset, ServerConnection sc,
			LayerCachePool cachePool, String hintSQLValue,int hintSqlType, Map hintMap)
			throws SQLNonTransientException {
	    SchemaConfig tempSchema = MycatServer.getInstance().getConfig().getSchemas().get(hintSQLValue);
		if (tempSchema != null) {
			return routeStrategy.route(sysConfig, tempSchema, sqlType, realSQL, charset, sc, cachePool);
		} else {
			String msg = "can't find hint schema:" + hintSQLValue;
			LOGGER.warn(msg);
			throw new SQLNonTransientException(msg);
		}
	}
}


================================================
FILE: src/main/java/io/mycat/route/impl/AbstractRouteStrategy.java
================================================
package io.mycat.route.impl;

import java.sql.SQLNonTransientException;
import java.sql.SQLSyntaxErrorException;
import java.util.List;

import io.mycat.config.model.TableConfig;
import org.slf4j.Logger; import org.slf4j.LoggerFactory;

import io.mycat.MycatServer;
import io.mycat.cache.LayerCachePool;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.SystemConfig;
import io.mycat.route.RouteResultset;
import io.mycat.route.RouteStrategy;
import io.mycat.route.util.RouterUtil;
import io.mycat.server.ServerConnection;
import io.mycat.server.parser.ServerParse;
import io.mycat.sqlengine.mpp.LoadData;

public abstract class AbstractRouteStrategy implements RouteStrategy {
	
	private static final Logger LOGGER = LoggerFactory.getLogger(AbstractRouteStrategy.class);

	@Override
	public RouteResultset route(SystemConfig sysConfig, SchemaConfig schema, int sqlType, String origSQL,
			String charset, ServerConnection sc, LayerCachePool cachePool) throws SQLNonTransientException {

		//对应schema标签checkSQLschema属性，把表示schema的字符去掉
		if (schema.isCheckSQLSchema()) {
			origSQL = RouterUtil.removeSchema(origSQL, schema.getName());
		}

		/**
     * 处理一些路由之前的逻辑
     * 全局序列号，父子表插入
     */
		if ( beforeRouteProcess(schema, sqlType, origSQL, sc) ) {
			return null;
		}

		/**
		 * SQL 语句拦截
		 */
		String stmt = MycatServer.getInstance().getSqlInterceptor().interceptSQL(origSQL, sqlType);
		if (!origSQL.equals(stmt) && LOGGER.isDebugEnabled()) {
			LOGGER.debug("sql intercepted to " + stmt + " from " + origSQL);
		}


		RouteResultset rrs = new RouteResultset(stmt, sqlType);

		/**
		 * 优化debug loaddata输出cache的日志会极大降低性能
		 */
		if (LOGGER.isDebugEnabled() && origSQL.startsWith(LoadData.loadDataHint)) {
			rrs.setCacheAble(false);
		}

        /**
         * rrs携带ServerConnection的autocommit状态用于在sql解析的时候遇到
         * select ... for update的时候动态设定RouteResultsetNode的canRunInReadDB属性
         */
		if (sc != null ) {
			rrs.setAutocommit(sc.isAutocommit());
		}

		/**
		 * DDL 语句的路由
		 */
		if (ServerParse.DDL == sqlType) {
			return RouterUtil.routeToDDLNode(rrs, sqlType, stmt, schema);
		}

		/**
		 * 检查是否有分片
		 */
		if (schema.isNoSharding() && ServerParse.SHOW != sqlType) {
			rrs = RouterUtil.routeToSingleNode(rrs, schema.getDataNode(), stmt);
		} else {
			RouteResultset returnedSet = routeSystemInfo(schema, sqlType, stmt, rrs);
			if (returnedSet == null) {
				rrs = routeNormalSqlWithAST(schema, stmt, rrs, charset, cachePool,sqlType,sc);
			}
		}

		if (rrs.getSqlType()==ServerParse.INSERT && rrs.getTables()!=null && rrs.getTables().size()!=0) {
			List tables = rrs.getTables();
			boolean isAutoIncrement = false;
			for (String tableName: tables) {
				if (schema.getTables()!=null && schema.getTables().get(tableName)!=null) {
					TableConfig tableConfig = schema.getTables().get(tableName);
					if (tableConfig.isAutoIncrement()) {
						isAutoIncrement = true;
					}
				}
			}
			rrs.setAutoIncrement(isAutoIncrement);
		}

		return rrs;
	}

	/**
	 * 路由之前必要的处理
	 * 主要是全局序列号插入，还有子表插入
	 */
	private boolean beforeRouteProcess(SchemaConfig schema, int sqlType, String origSQL, ServerConnection sc)
			throws SQLNonTransientException {
		
		return RouterUtil.processWithMycatSeq(schema, sqlType, origSQL, sc)
				|| (sqlType == ServerParse.INSERT && RouterUtil.processERChildTable(schema, origSQL, sc))
				|| (sqlType == ServerParse.INSERT && RouterUtil.processInsert(schema, sqlType, origSQL, sc));
	}

	/**
	 * 通过解析AST语法树类来寻找路由
	 */
	public abstract RouteResultset routeNormalSqlWithAST(SchemaConfig schema, String stmt, RouteResultset rrs,
			String charset, LayerCachePool cachePool,int sqlType,ServerConnection sc) throws SQLNonTransientException;

	/**
	 * 路由信息指令, 如 SHOW、SELECT@@、DESCRIBE
	 */
	public abstract RouteResultset routeSystemInfo(SchemaConfig schema, int sqlType, String stmt, RouteResultset rrs)
			throws SQLSyntaxErrorException;

	/**
	 * 解析 Show 之类的语句
	 */
	public abstract RouteResultset analyseShowSQL(SchemaConfig schema, RouteResultset rrs, String stmt)
			throws SQLNonTransientException;

}


================================================
FILE: src/main/java/io/mycat/route/impl/DruidMycatRouteStrategy.java
================================================
package io.mycat.route.impl;

import java.sql.SQLNonTransientException;
import java.sql.SQLSyntaxErrorException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;

import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLObject;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLAllExpr;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr;
import com.alibaba.druid.sql.ast.expr.SQLExistsExpr;
import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
import com.alibaba.druid.sql.ast.expr.SQLInSubQueryExpr;
import com.alibaba.druid.sql.ast.expr.SQLQueryExpr;
import com.alibaba.druid.sql.ast.statement.SQLDeleteStatement;
import com.alibaba.druid.sql.ast.statement.SQLExprTableSource;
import com.alibaba.druid.sql.ast.statement.SQLInsertStatement;
import com.alibaba.druid.sql.ast.statement.SQLReplaceStatement;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.ast.statement.SQLSelectQuery;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.ast.statement.SQLTableSource;
import com.alibaba.druid.sql.ast.statement.SQLUpdateStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlInsertStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.druid.sql.parser.SQLStatementParser;
import com.alibaba.druid.stat.TableStat.Relationship;
import com.google.common.base.Strings;

import io.mycat.MycatServer;
import io.mycat.backend.datasource.PhysicalDBNode;
import io.mycat.backend.mysql.nio.handler.MiddlerQueryResultHandler;
import io.mycat.backend.mysql.nio.handler.MiddlerResultHandler;
import io.mycat.backend.mysql.nio.handler.SecondHandler;
import io.mycat.cache.LayerCachePool;
import io.mycat.config.ErrorCode;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.TableConfig;
import io.mycat.config.model.rule.RuleConfig;
import io.mycat.route.RouteResultset;
import io.mycat.route.RouteResultsetNode;
import io.mycat.route.function.SlotFunction;
import io.mycat.route.impl.middlerResultStrategy.BinaryOpResultHandler;
import io.mycat.route.impl.middlerResultStrategy.InSubQueryResultHandler;
import io.mycat.route.impl.middlerResultStrategy.RouteMiddlerReaultHandler;
import io.mycat.route.impl.middlerResultStrategy.SQLAllResultHandler;
import io.mycat.route.impl.middlerResultStrategy.SQLExistsResultHandler;
import io.mycat.route.impl.middlerResultStrategy.SQLQueryResultHandler;
import io.mycat.route.parser.druid.DruidParser;
import io.mycat.route.parser.druid.DruidParserFactory;
import io.mycat.route.parser.druid.DruidShardingParseInfo;
import io.mycat.route.parser.druid.MycatSchemaStatVisitor;
import io.mycat.route.parser.druid.MycatStatementParser;
import io.mycat.route.parser.druid.RouteCalculateUnit;
import io.mycat.route.parser.util.ParseUtil;
import io.mycat.route.util.RouterUtil;
import io.mycat.server.NonBlockingSession;
import io.mycat.server.ServerConnection;
import io.mycat.server.parser.ServerParse;
import io.mycat.server.util.SchemaUtil;

public class DruidMycatRouteStrategy extends AbstractRouteStrategy {

	public static final Logger LOGGER = LoggerFactory.getLogger(DruidMycatRouteStrategy.class);

	private static Map,RouteMiddlerReaultHandler> middlerResultHandler = new HashMap<>();

	static{
		middlerResultHandler.put(SQLQueryExpr.class, new SQLQueryResultHandler());
		middlerResultHandler.put(SQLBinaryOpExpr.class, new BinaryOpResultHandler());
		middlerResultHandler.put(SQLInSubQueryExpr.class, new InSubQueryResultHandler());
		middlerResultHandler.put(SQLExistsExpr.class, new SQLExistsResultHandler());
		middlerResultHandler.put(SQLAllExpr.class, new SQLAllResultHandler());
	}

	@Override
	public RouteResultset routeNormalSqlWithAST(SchemaConfig schema, String stmt, RouteResultset rrs, String charset,
			LayerCachePool cachePool, int sqlType, ServerConnection sc) throws SQLNonTransientException {
		// 如果是批量的update,delete走特别的流程
		if (sc != null && sc.isAllowMultiStatements()
				&& (sqlType == ServerParse.DELETE || sqlType == ServerParse.UPDATE)) {
			return routeMultiSqlWithAST(schema, stmt, rrs, charset, cachePool, sqlType, sc);

		} else {
			return routeNormalSqlWithAST0(schema, stmt, rrs, charset, cachePool, sqlType, sc);
		}
	}

	private RouteResultset routeNormalSqlWithAST0(SchemaConfig schema, String stmt, RouteResultset rrs, String charset,
			LayerCachePool cachePool, int sqlType, ServerConnection sc) throws SQLNonTransientException {

		/**
		 *  只有mysql时只支持mysql语法
		 */
		SQLStatementParser parser = null;
		if (schema.isNeedSupportMultiDBType()) {
			parser = new MycatStatementParser(stmt);
		} else {
			parser = new MySqlStatementParser(stmt);
		}

		MycatSchemaStatVisitor visitor = null;
		SQLStatement statement;

		/**
		 * 解析出现问题统一抛SQL语法错误
		 */
		try {
            if (parser instanceof MycatStatementParser || sqlType == ServerParse.LOCK) {
                /**
                 * 说明： 1)非mysql数据库因为是jdbc驱动支持多语句，所以无需判断是否为多语句；
                 * 2)lock类型语句，因为druid自身问题且升级后mycat代码大量编译报错，只能沿用当前逻辑，不判断是否为多语句
                 */
                statement = parser.parseStatement();
            } else {
                // 因不支持多语句，添加判断是否为多语句
                List statementList = new ArrayList();

                /** 最多就解析2条，用于判断是否为批量 **/
                parser.parseStatementList(statementList, 2);
                if (statementList.size() > 1) {
                    throw new SQLSyntaxErrorException(
                            "Multi statements is not supported,use single statement instead ");
                } else {
                    statement = statementList.get(0);
                }
            }

			visitor = new MycatSchemaStatVisitor();
		} catch (Exception t) {
			LOGGER.error("DruidMycatRouteStrategyError", t);
			throw new SQLSyntaxErrorException(t);
		}

		/**
		 * 检验unsupported statement
		 */
		checkUnSupportedStatement(statement);

		DruidParser druidParser = DruidParserFactory.create(schema, statement, visitor);
		druidParser.parser(schema, rrs, statement, stmt,cachePool,visitor);
		DruidShardingParseInfo ctx=  druidParser.getCtx() ;
		rrs.setTables(ctx.getTables());

		if(visitor.isSubqueryRelationOr()){
			String err = "In subQuery,the or condition is not supported.";
			LOGGER.error(err);
			throw new SQLSyntaxErrorException(err);
		}

		/* 按照以下情况路由
			1.2.1 可以直接路由.
       		1.2.2 两个表夸库join的sql.调用calat
       		1.2.3 需要先执行subquery 的sql.把subquery拆分出来.获取结果后,与outerquery
		 */

		//add huangyiming 分片规则不一样的且表中带查询条件的则走Catlet
		List tables = ctx.getTables();
		SchemaConfig schemaConf = MycatServer.getInstance().getConfig().getSchemas().get(schema.getName());
		int index = 0;
		RuleConfig firstRule = null;
		boolean directRoute = true;
		Set firstDataNodes = new HashSet();
		Map tconfigs = schemaConf==null?null:schemaConf.getTables();

		Map rulemap = new HashMap<>();
		if(tconfigs!=null){
			for(String tableName : tables){
				TableConfig tc =  tconfigs.get(tableName);
				if(tc == null){
					//add 别名中取
					Map tableAliasMap = ctx.getTableAliasMap();
					if(tableAliasMap !=null && tableAliasMap.get(tableName) !=null){
						tc = schemaConf.getTables().get(tableAliasMap.get(tableName));
					}
				}

				if(index == 0){
					if(tc !=null){
						firstRule=  tc.getRule();
						//没有指定分片规则时,不做处理
						if(firstRule==null){
							continue;
						}
						firstDataNodes.addAll(tc.getDataNodes());
						rulemap.put(tc.getName(), firstRule);
					}
				}else{
					if(tc !=null){
						//ER关系表的时候是可能存在字表中没有tablerule的情况,所以加上判断
						RuleConfig ruleCfg = tc.getRule();
						if(ruleCfg==null){  //没有指定分片规则时,不做处理
							continue;
						}
						Set dataNodes = new HashSet<>(tc.getDataNodes());
						rulemap.put(tc.getName(), ruleCfg);
						//如果匹配规则不相同或者分片的datanode不相同则需要走子查询处理
						if(firstRule!=null&&((ruleCfg !=null && !ruleCfg.getRuleAlgorithm().equals(firstRule.getRuleAlgorithm()) )||( !dataNodes.equals(firstDataNodes)))){
							directRoute = false;
							break;
						}
					}
				}
				index++;
			}
		}

		RouteResultset rrsResult = rrs;
		if(directRoute){ //直接路由
			if(!RouterUtil.isAllGlobalTable(ctx, schemaConf)){
				if(rulemap.size()>1&&!checkRuleField(rulemap,visitor)){
					String err = "In case of slice table,there is no rule field in the relationship condition!";
					LOGGER.error(err);
					throw new SQLSyntaxErrorException(err);
				}
			}
			rrsResult = directRoute(rrs,ctx,schema,druidParser,statement,cachePool);
		}else{
			int subQuerySize = visitor.getSubQuerys().size();
			if(subQuerySize==0&&ctx.getTables().size()==2){ //两表关联,考虑使用catlet
				if(!visitor.getRelationships().isEmpty()){
					rrs.setCacheAble(false);
					rrs.setFinishedRoute(true);
					rrsResult = catletRoute(schema,ctx.getSql(),charset,sc);
				}else{
					rrsResult = directRoute(rrs,ctx,schema,druidParser,statement,cachePool);
				}
			}else if(subQuerySize==1){     //只涉及一张表的子查询,使用  MiddlerResultHandler 获取中间结果后,改写原有 sql 继续执行 TODO 后期可能会考虑多个子查询的情况.
				SQLSelect sqlselect = visitor.getSubQuerys().iterator().next();
				if(!visitor.getRelationships().isEmpty()){     // 当 inner query  和 outer  query  有关联条件时,暂不支持
					String err = "In case of slice table,sql have different rules,the relationship condition is not supported.";
					LOGGER.error(err);
					throw new SQLSyntaxErrorException(err);
				}else{
					SQLSelectQuery sqlSelectQuery = sqlselect.getQuery();
					if(((MySqlSelectQueryBlock)sqlSelectQuery).getFrom() instanceof SQLExprTableSource) {
						rrs.setCacheAble(false);
						rrs.setFinishedRoute(true);
						rrsResult = middlerResultRoute(schema,charset,sqlselect,sqlType,statement,sc);
					}
				}
			}else if(subQuerySize >=2){
				String err = "In case of slice table,sql has different rules,currently only one subQuery is supported.";
				LOGGER.error(err);
				throw new SQLSyntaxErrorException(err);
			}
		}
		return rrsResult;
	}

	// 批量update,delete路由方法
	private RouteResultset routeMultiSqlWithAST(SchemaConfig schema, String stmt, RouteResultset rrs, String charset,
			LayerCachePool cachePool, int sqlType, ServerConnection sc) throws SQLNonTransientException {
		List allNodes = new ArrayList<>(64);
		// 拆分出一个个SQL解析路由
		String remingSql = stmt;
		String eachSqlItem = null;
		do {
			int index = ParseUtil.findNextBreak(remingSql);
			if (index + 1 < remingSql.length() && !ParseUtil.isEOF(remingSql, index)) {
				eachSqlItem = remingSql.substring(0, index);
				remingSql = remingSql.substring(index + 1, remingSql.length());
				RouteResultset rrsTemp = new RouteResultset(eachSqlItem, sqlType);
				RouteResultset tempRrs = routeNormalSqlWithAST0(schema, eachSqlItem, rrsTemp, charset, cachePool,
						sqlType, sc);
				allNodes.addAll(Arrays.asList(tempRrs.getNodes()));
			} else {
				// the last one
				RouteResultset rrsTemp = new RouteResultset(remingSql, sqlType);
				RouteResultset tempRrs = routeNormalSqlWithAST0(schema, remingSql, rrsTemp, charset, cachePool, sqlType,
						sc);
				allNodes.addAll(Arrays.asList(tempRrs.getNodes()));
				rrs = rrsTemp;
				break;
			}
		} while (true);

		if (allNodes.size() >= 1) {
			// merge
			rrs.setStatement(stmt);
			rrs.setNodes(allNodes.toArray(new RouteResultsetNode[0]));
			rrs.mergeSameNode();
			return rrs;
		} else {
			throw new SQLNonTransientException("mycat parse error on sql:" + stmt);
		}
	}
	/**
	 * 子查询中存在关联查询的情况下,检查关联字段是否是分片字段
	 * @param rulemap
	 * @param ships
	 * @return
	 */
	private boolean checkRuleField(Map rulemap,MycatSchemaStatVisitor visitor){

		if(!MycatServer.getInstance().getConfig().getSystem().isSubqueryRelationshipCheck()){
			return true;
		}

		Set ships = visitor.getRelationships();
		Iterator iter = ships.iterator();
		while(iter.hasNext()){
			Relationship ship = iter.next();
			String lefttable = ship.getLeft().getTable().toUpperCase();
			String righttable = ship.getRight().getTable().toUpperCase();
			// 如果是同一个表中的关联条件,不做处理
			if(lefttable.equals(righttable)){
				return true;
			}
			RuleConfig leftconfig = rulemap.get(lefttable);
			RuleConfig rightconfig = rulemap.get(righttable);

			if(null!=leftconfig&&null!=rightconfig
					&&leftconfig.equals(rightconfig)
					&&leftconfig.getColumn().equals(ship.getLeft().getName().toUpperCase())
					&&rightconfig.getColumn().equals(ship.getRight().getName().toUpperCase())){
				return true;
			}
		}
		return false;
	}

	private RouteResultset middlerResultRoute(final SchemaConfig schema,final String charset,final SQLSelect sqlselect,
			final int sqlType,final SQLStatement statement,final ServerConnection sc){

		final String middlesql = SQLUtils.toMySqlString(sqlselect);

		MiddlerResultHandler middlerResultHandler =  new MiddlerQueryResultHandler<>(new SecondHandler() {
			@Override
			public void doExecute(List param) {
				sc.getSession2().setMiddlerResultHandler(null);
				String sqls = null;
				// 路由计算
				RouteResultset rrs = null;
				try {

					sqls = buildSql(statement,sqlselect,param);
					rrs = MycatServer
							.getInstance()
							.getRouterservice()
							.route(MycatServer.getInstance().getConfig().getSystem(),
									schema, sqlType,sqls.toLowerCase(), charset,sc );

				} catch (Exception e) {
					StringBuilder s = new StringBuilder();
					LOGGER.warn(s.append(this).append(sqls).toString() + " err:" + e.toString(),e);
					String msg = e.getMessage();
					sc.writeErrMessage(ErrorCode.ER_PARSE_ERROR, msg == null ? e.getClass().getSimpleName() : msg);
					return;
				}
				NonBlockingSession noBlockSession =  new NonBlockingSession(sc.getSession2().getSource());
				noBlockSession.setMiddlerResultHandler(null);
				//session的预编译标示传递
				noBlockSession.setPrepared(sc.getSession2().isPrepared());
				if (rrs != null) {
					noBlockSession.setCanClose(false);
					noBlockSession.execute(rrs, ServerParse.SELECT);
				}
			}
		} );
		sc.getSession2().setMiddlerResultHandler(middlerResultHandler);
		sc.getSession2().setCanClose(false);

		// 路由计算
		RouteResultset rrs = null;
		try {
			rrs = MycatServer
					.getInstance()
					.getRouterservice()
					.route(MycatServer.getInstance().getConfig().getSystem(),
							schema, ServerParse.SELECT, middlesql, charset, sc);

		} catch (Exception e) {
			StringBuilder s = new StringBuilder();
			LOGGER.warn(s.append(this).append(middlesql).toString() + " err:" + e.toString(),e);
			String msg = e.getMessage();
			sc.writeErrMessage(ErrorCode.ER_PARSE_ERROR, msg == null ? e.getClass().getSimpleName() : msg);
			return null;
		}

		if(rrs!=null){
			rrs.setCacheAble(false);
		}
		return rrs;
	}

	/**
	 * 获取子查询执行结果后,改写原始sql 继续执行.
	 * @param statement
	 * @param sqlselect
	 * @param param
	 * @return
	 */
	private String buildSql(SQLStatement statement,SQLSelect sqlselect,List param){

		SQLObject parent = sqlselect.getParent();
		RouteMiddlerReaultHandler handler = middlerResultHandler.get(parent.getClass());
		if(handler==null){
			throw new UnsupportedOperationException(parent.getClass()+" current is not supported ");
		}
		return handler.dohandler(statement, sqlselect, parent, param);
	}

	/**
	 * 两个表的情况，catlet
	 * @param schema
	 * @param stmt
	 * @param charset
	 * @param sc
	 * @return
	 */
	private RouteResultset catletRoute(SchemaConfig schema,String stmt,String charset,ServerConnection sc){
		RouteResultset rrs = null;
		try {
			rrs = MycatServer
					.getInstance()
					.getRouterservice()
					.route(MycatServer.getInstance().getConfig().getSystem(),
							schema, ServerParse.SELECT, "/*!mycat:catlet=io.mycat.catlets.ShareJoin */ "+stmt, charset, sc);

		}catch(Exception e){

		}
		return rrs;
	}

	/**
	 *  直接结果路由
	 * @param rrs
	 * @param ctx
	 * @param schema
	 * @param druidParser
	 * @param statement
	 * @param cachePool
	 * @return
	 * @throws SQLNonTransientException
	 */
	private RouteResultset directRoute(RouteResultset rrs,DruidShardingParseInfo ctx,SchemaConfig schema,
			DruidParser druidParser,SQLStatement statement,LayerCachePool cachePool) throws SQLNonTransientException{

		//改写sql：如insert语句主键自增长, 在直接结果路由的情况下,进行sql 改写处理
		druidParser.changeSql(schema, rrs, statement,cachePool);

		/**
		 * DruidParser 解析过程中已完成了路由的直接返回
		 */
		if ( rrs.isFinishedRoute() ) {
			return rrs;
		}

		/**
		 * 没有from的select语句或其他
		 */
		if((ctx.getTables() == null || ctx.getTables().size() == 0)&&(ctx.getTableAliasMap()==null||ctx.getTableAliasMap().isEmpty()))
		{
			return RouterUtil.routeToSingleNode(rrs, schema.getRandomDataNode(), druidParser.getCtx().getSql());
		}

		if(druidParser.getCtx().getRouteCalculateUnits().size() == 0) {
			RouteCalculateUnit routeCalculateUnit = new RouteCalculateUnit();
			druidParser.getCtx().addRouteCalculateUnit(routeCalculateUnit);
		}

		SortedSet nodeSet = new TreeSet();
		boolean isAllGlobalTable = RouterUtil.isAllGlobalTable(ctx, schema);
		for(RouteCalculateUnit unit: druidParser.getCtx().getRouteCalculateUnits()) {
			RouteResultset rrsTmp = RouterUtil.tryRouteForTables(schema, druidParser.getCtx(), unit, rrs, isSelect(statement), cachePool);
			if(rrsTmp != null&&rrsTmp.getNodes()!=null) {
				for(RouteResultsetNode node :rrsTmp.getNodes()) {
					nodeSet.add(node);
				}
			}
			if(isAllGlobalTable) {//都是全局表时只计算一遍路由
				break;
			}
		}

		RouteResultsetNode[] nodes = new RouteResultsetNode[nodeSet.size()];
		int i = 0;
		for (RouteResultsetNode aNodeSet : nodeSet) {
			nodes[i] = aNodeSet;
			if(statement instanceof MySqlInsertStatement &&ctx.getTables().size()==1&&schema.getTables().containsKey(ctx.getTables().get(0))) {
				RuleConfig rule = schema.getTables().get(ctx.getTables().get(0)).getRule();
				if(rule!=null&&  rule.getRuleAlgorithm() instanceof SlotFunction){
					aNodeSet.setStatement(ParseUtil.changeInsertAddSlot(aNodeSet.getStatement(),aNodeSet.getSlot()));
				}
			}
			i++;
		}
		rrs.setNodes(nodes);

		//分表
		/**
		 *  subTables="t_order$1-2,t_order3"
		 *目前分表 1.6 开始支持 幵丏 dataNode 在分表条件下只能配置一个，分表条件下不支持join。
		 */
		if(rrs.isDistTable()){
			return this.routeDisTable(statement,rrs);
		}
		return rrs;
	}

	private SQLExprTableSource getDisTable(SQLTableSource tableSource,RouteResultsetNode node) throws SQLSyntaxErrorException{
		if(node.getSubTableName()==null){
			String msg = " sub table not exists for " + node.getName() + " on " + tableSource;
			LOGGER.error("DruidMycatRouteStrategyError " + msg);
			throw new SQLSyntaxErrorException(msg);
		}

		SQLIdentifierExpr sqlIdentifierExpr = new SQLIdentifierExpr();
		sqlIdentifierExpr.setParent(tableSource.getParent());
		sqlIdentifierExpr.setName(node.getSubTableName());
		SQLExprTableSource from2 = new SQLExprTableSource(sqlIdentifierExpr);
		return from2;
	}

	private RouteResultset routeDisTable(SQLStatement statement, RouteResultset rrs) throws SQLSyntaxErrorException{
		SQLTableSource tableSource = null;
		if(statement instanceof SQLInsertStatement) {
			SQLInsertStatement insertStatement = (SQLInsertStatement) statement;
			tableSource = insertStatement.getTableSource();
			for (RouteResultsetNode node : rrs.getNodes()) {
				SQLExprTableSource from2 = getDisTable(tableSource, node);
				insertStatement.setTableSource(from2);
				node.setStatement(insertStatement.toString());
			}
		}
		if(statement instanceof SQLDeleteStatement) {
			SQLDeleteStatement deleteStatement = (SQLDeleteStatement) statement;
			tableSource = deleteStatement.getTableSource();
			SQLTableSource from = deleteStatement.getFrom();
			for (RouteResultsetNode node : rrs.getNodes()) {
				SQLExprTableSource from2 = getDisTable(tableSource, node);

				if (from == null) {
					from2.setAlias(tableSource.toString());
					deleteStatement.setFrom(from2);
				} else {
					String alias = from.getAlias();
					from2.setAlias(alias);
					deleteStatement.setFrom(from2);
				}

				node.setStatement(deleteStatement.toString());
			}
		}
		if(statement instanceof SQLUpdateStatement) {
			SQLUpdateStatement updateStatement = (SQLUpdateStatement) statement;
			tableSource = updateStatement.getTableSource();

			String alias = tableSource.getAlias();
			SQLExprTableSource exprSource = (SQLExprTableSource) tableSource;
			SQLIdentifierExpr expr = (SQLIdentifierExpr) exprSource.getExpr();
			alias = alias == null ? expr.getName() : alias;

			for (RouteResultsetNode node : rrs.getNodes()) {
				SQLExprTableSource from2 = getDisTable(tableSource, node);

				//修复
				// EXPLAIN UPDATE travelrecord SET user_id = 'Fred' WHERE id IN 1; => UPDATE travelrecord travelrecord SET user_id = 'Fred' WHERE id IN (1)
				if (!from2.toString().equals(alias)){
					from2.setAlias(alias);
				}
				updateStatement.setTableSource(from2);
				node.setStatement(updateStatement.toString());
			}
		}

		return rrs;
	}

	/**
	 * SELECT 语句
	 */
	private boolean isSelect(SQLStatement statement) {
		if(statement instanceof SQLSelectStatement) {
			return true;
		}
		return false;
	}

	/**
	 * 检验不支持的SQLStatement类型 ：不支持的类型直接抛SQLSyntaxErrorException异常
	 * @param statement
	 * @throws SQLSyntaxErrorException
	 */
	private void checkUnSupportedStatement(SQLStatement statement) throws SQLSyntaxErrorException {
		//不支持replace语句
		if (statement instanceof SQLReplaceStatement) {
			throw new SQLSyntaxErrorException(" ReplaceStatement can't be supported,use insert into ...on duplicate key update... instead ");
		}
	}

	/**
	 * 分析 SHOW SQL
	 */
	@Override
	public RouteResultset analyseShowSQL(SchemaConfig schema,
			RouteResultset rrs, String stmt) throws SQLSyntaxErrorException {

		String[] fields = SchemaUtil.parseShowTable(stmt);
        if ("1".equals(fields[0])) {// show tables
			String relSchema = fields[3];
			String tableName = fields[8];
            //
            if (relSchema != null && !relSchema.equalsIgnoreCase(schema.getName())) {
                schema = MycatServer.getInstance().getConfig().getSchemas().get(relSchema);
                if (schema == null) {
                    throw new SQLSyntaxErrorException("not found schema : " + relSchema);
                }
            }
			// 2020/03/19 Ken.Li
			// if (StringUtils.isNotBlank(tableName) && tableName.indexOf("%") < 0) {
			if (StringUtils.isNotBlank(tableName) && tableName.indexOf("%") < 0 && !schema.getTables().isEmpty()) {
                TableConfig tableConfig = schema.getTables().get(tableName.toUpperCase());
                if (tableConfig != null) {
                    String dataNode = RouterUtil.getAliveRandomDataNode(tableConfig);
                    PhysicalDBNode dataNodeObj = MycatServer.getInstance().getConfig().getDataNodes().get(dataNode);
                    if (StringUtils.isNotBlank(dataNodeObj.getDatabase())) {
                        stmt = stmt.replaceAll(relSchema, dataNodeObj.getDatabase());
                    }
                    return RouterUtil.routeToSingleNode(rrs, dataNode, stmt);
                }
            }
            // remove db
            if (StringUtils.isNotBlank(relSchema)) {
				if(schema.getDataNode() !=null) {
					// 2020/03/19 Ken.Li
					// stmt = stmt.replaceAll(relSchema, schema.getDataNode());
					PhysicalDBNode dataNode = MycatServer.getInstance().getConfig().getDataNodes().get(schema.getDataNode());
					stmt = stmt.replaceAll(relSchema, dataNode.getDatabase());
				} else {
					if(fields[2] !=null ) {
						stmt = stmt.replaceAll( fields[2] + "\\s*" + relSchema, "");
					}
				}
            }
            //
            String defaultNode = schema.getDataNode();
            if (!Strings.isNullOrEmpty(defaultNode)) {
                return RouterUtil.routeToSingleNode(rrs, defaultNode, stmt);
            }

            return RouterUtil.routeToMultiNode(false, rrs, schema.getMetaDataNodes(), stmt);
		}
		
		String upStmt = stmt.toUpperCase();
		/**
		 *  show index or column
		 */
		int[] indx = RouterUtil.getSpecPos(upStmt, 0);
		if (indx[0] > 0) {
			/**
			 *  has table
			 */
			int[] repPos = { indx[0] + indx[1], 0 };
			String tableName = RouterUtil.getShowTableName(schema,stmt, repPos);
			/**
			 *  IN DB pattern
			 */
			int[] indx2 = RouterUtil.getSpecPos(upStmt, indx[0] + indx[1] + 1);
			if (indx2[0] > 0) {// find LIKE OR WHERE
				repPos[1] = RouterUtil.getSpecEndPos(upStmt, indx2[0] + indx2[1]);

			}
			stmt = stmt.substring(0, indx[0]) + " FROM " + tableName + stmt.substring(repPos[1]);
			RouterUtil.routeForTableMeta(rrs, schema, tableName, stmt);
			return rrs;

		}

		/**
		 *  show create table tableName
		 */
		int[] createTabInd = RouterUtil.getCreateTablePos(upStmt, 0);
		if (createTabInd[0] > 0) {
			int tableNameIndex = createTabInd[0] + createTabInd[1];
			if (upStmt.length() > tableNameIndex) {
				String tableName = stmt.substring(tableNameIndex).trim();
				int ind2 = tableName.indexOf('.');
				if (ind2 > 0) {
					tableName = tableName.substring(ind2 + 1);
				}
				RouterUtil.routeForTableMeta(rrs, schema, tableName, stmt);
				return rrs;
			}
		}

		return RouterUtil.routeToSingleNode(rrs, schema.getRandomDataNode(), stmt);
	}


//	/**
//	 * 为一个表进行条件路由
//	 * @param schema
//	 * @param tablesAndConditions
//	 * @param tablesRouteMap
//	 * @throws SQLNonTransientException
//	 */
//	private static RouteResultset findRouteWithcConditionsForOneTable(SchemaConfig schema, RouteResultset rrs,
//		Map> conditions, String tableName, String sql) throws SQLNonTransientException {
//		boolean cache = rrs.isCacheAble();
//	    //为分库表找路由
//		tableName = tableName.toUpperCase();
//		TableConfig tableConfig = schema.getTables().get(tableName);
//		//全局表或者不分库的表略过（全局表后面再计算）
//		if(tableConfig.isGlobalTable()) {
//			return null;
//		} else {//非全局表
//			Set routeSet = new HashSet();
//			String joinKey = tableConfig.getJoinKey();
//			for(Map.Entry> condition : conditions.entrySet()) {
//				String colName = condition.getKey();
//				//条件字段是拆分字段
//				if(colName.equals(tableConfig.getPartitionColumn())) {
//					Set columnPairs = condition.getValue();
//
//					for(ColumnRoutePair pair : columnPairs) {
//						if(pair.colValue != null) {
//							Integer nodeIndex = tableConfig.getRule().getRuleAlgorithm().calculate(pair.colValue);
//							if(nodeIndex == null) {
//								String msg = "can't find any valid datanode :" + tableConfig.getName()
//										+ " -> " + tableConfig.getPartitionColumn() + " -> " + pair.colValue;
//								LOGGER.warn(msg);
//								throw new SQLNonTransientException(msg);
//							}
//							String node = tableConfig.getDataNodes().get(nodeIndex);
//							if(node != null) {//找到一个路由节点
//								routeSet.add(node);
//							}
//						}
//						if(pair.rangeValue != null) {
//							Integer[] nodeIndexs = tableConfig.getRule().getRuleAlgorithm()
//									.calculateRange(pair.rangeValue.beginValue.toString(), pair.rangeValue.endValue.toString());
//							for(Integer idx : nodeIndexs) {
//								String node = tableConfig.getDataNodes().get(idx);
//								if(node != null) {//找到一个路由节点
//									routeSet.add(node);
//								}
//							}
//						}
//					}
//				} else if(joinKey != null && joinKey.equals(colName)) {
//					Set dataNodeSet = RouterUtil.ruleCalculate(
//							tableConfig.getParentTC(), condition.getValue());
//					if (dataNodeSet.isEmpty()) {
//						throw new SQLNonTransientException(
//								"parent key can't find any valid datanode ");
//					}
//					if (LOGGER.isDebugEnabled()) {
//						LOGGER.debug("found partion nodes (using parent partion rule directly) for child table to update  "
//								+ Arrays.toString(dataNodeSet.toArray()) + " sql :" + sql);
//					}
//					if (dataNodeSet.size() > 1) {
//						return RouterUtil.routeToMultiNode(rrs.isCacheAble(), rrs, schema.getAllDataNodes(), sql);
//					} else {
//						rrs.setCacheAble(true);
//						return RouterUtil.routeToSingleNode(rrs, dataNodeSet.iterator().next(), sql);
//					}
//				} else {//条件字段不是拆分字段也不是join字段,略过
//					continue;
//
//				}
//			}
//			return RouterUtil.routeToMultiNode(cache, rrs, routeSet, sql);
//
//		}
//
//	}

	public RouteResultset routeSystemInfo(SchemaConfig schema, int sqlType,
			String stmt, RouteResultset rrs) throws SQLSyntaxErrorException {
		switch(sqlType){
			case ServerParse.SHOW:// if origSQL is like show tables
				return analyseShowSQL(schema, rrs, stmt);
			case ServerParse.SELECT://if origSQL is like select @@
				int index = stmt.indexOf("@@");
				if(index > 0 && "SELECT".equals(stmt.substring(0, index).trim().toUpperCase())){
					return analyseDoubleAtSgin(schema, rrs, stmt);
				}
				break;
			case ServerParse.DESCRIBE:// if origSQL is meta SQL, such as describe table
				int ind = stmt.indexOf(' ');
				stmt = stmt.trim();
				return analyseDescrSQL(schema, rrs, stmt, ind + 1);
		}
		return null;
	}

	/**
	 * 对Desc语句进行分析 返回数据路由集合
	 * 	 *
	 * @param schema   				数据库名
	 * @param rrs    				数据路由集合
	 * @param stmt   				执行语句
	 * @param ind    				第一个' '的位置
	 * @return RouteResultset		(数据路由集合)
	 * @author mycat
	 */
	private static RouteResultset analyseDescrSQL(SchemaConfig schema,
			RouteResultset rrs, String stmt, int ind) {

		final String MATCHED_FEATURE = "DESCRIBE ";
		final String MATCHED2_FEATURE = "DESC ";
		int pos = 0;
		while (pos < stmt.length()) {
			char ch = stmt.charAt(pos);
			// 忽略处理注释 /* */ BEN
			if(ch == '/' &&  pos+4 < stmt.length() && stmt.charAt(pos+1) == '*') {
				if(stmt.substring(pos+2).indexOf("*/") != -1) {
					pos += stmt.substring(pos+2).indexOf("*/")+4;
					continue;
				} else {
					// 不应该发生这类情况。
					throw new IllegalArgumentException("sql 注释 语法错误");
				}
			} else if(ch == 'D'||ch == 'd') {
				// 匹配 [describe ]
				if(pos+MATCHED_FEATURE.length() < stmt.length() && (stmt.substring(pos).toUpperCase().indexOf(MATCHED_FEATURE) != -1)) {
					pos = pos + MATCHED_FEATURE.length();
					break;
				} else if(pos+MATCHED2_FEATURE.length() < stmt.length() && (stmt.substring(pos).toUpperCase().indexOf(MATCHED2_FEATURE) != -1)) {
					pos = pos + MATCHED2_FEATURE.length();
					break;
				} else {
					pos++;
				}
			} else {
				break;
			}
		}

		// 重置ind坐标。BEN GONG
		ind = pos;
		int[] repPos = { ind, 0 };
		String tableName = RouterUtil.getTableName(stmt, repPos);

		stmt = stmt.substring(0, ind) + tableName + stmt.substring(repPos[1]);
		RouterUtil.routeForTableMeta(rrs, schema, tableName, stmt);
		return rrs;
	}

	/**
	 * 根据执行语句判断数据路由
	 *
	 * @param schema     			数据库名
	 * @param rrs		  		 	数据路由集合
	 * @param stmt		  	 		执行sql
	 * @return RouteResultset		数据路由集合
	 * @throws SQLSyntaxErrorException
	 * @author mycat
	 */
	private RouteResultset analyseDoubleAtSgin(SchemaConfig schema,
			RouteResultset rrs, String stmt) throws SQLSyntaxErrorException {
		String upStmt = stmt.toUpperCase();
		int atSginInd = upStmt.indexOf(" @@");
		if (atSginInd > 0) {
			return RouterUtil.routeToMultiNode(false, rrs, schema.getMetaDataNodes(), stmt);
		}
		return RouterUtil.routeToSingleNode(rrs, schema.getRandomDataNode(), stmt);
	}
}

================================================
FILE: src/main/java/io/mycat/route/impl/middlerResultStrategy/BinaryOpResultHandler.java
================================================
package io.mycat.route.impl.middlerResultStrategy;

import java.util.List;

import com.alibaba.druid.sql.ast.SQLExprImpl;
import com.alibaba.druid.sql.ast.SQLObject;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr;
import com.alibaba.druid.sql.ast.expr.SQLInListExpr;
import com.alibaba.druid.sql.ast.expr.SQLInSubQueryExpr;
import com.alibaba.druid.sql.ast.expr.SQLListExpr;
import com.alibaba.druid.sql.ast.expr.SQLNullExpr;
import com.alibaba.druid.sql.ast.expr.SQLQueryExpr;
import com.alibaba.druid.sql.ast.statement.SQLSelect;

public class BinaryOpResultHandler implements RouteMiddlerReaultHandler {

	@Override
	public String dohandler(SQLStatement statement, SQLSelect sqlselect, SQLObject parent, List param) {
		
		SQLBinaryOpExpr pp = (SQLBinaryOpExpr)parent;
		if(pp.getLeft() instanceof SQLQueryExpr){
			SQLQueryExpr left = (SQLQueryExpr)pp.getLeft();
			if(left.getSubQuery().equals(sqlselect)){
				SQLExprImpl listExpr = null;
				if(null==param||param.isEmpty()){
					listExpr = new SQLNullExpr();
				}else{
					listExpr = new SQLListExpr();
					listExpr.setParent(left.getParent());
					((SQLListExpr)listExpr).getItems().addAll(param);
				}
				pp.setLeft(listExpr);
			}
		}else if(pp.getRight() instanceof SQLQueryExpr){
			SQLQueryExpr right = (SQLQueryExpr)pp.getRight();
			if(right.getSubQuery().equals(sqlselect)){
				SQLExprImpl listExpr = null;
				if(null==param||param.isEmpty()){
					listExpr = new SQLNullExpr();
				}else{
					listExpr = new SQLListExpr();
					listExpr.setParent(right.getParent());
					((SQLListExpr)listExpr).getItems().addAll(param);
				}
				pp.setRight(listExpr);
				
			}
		}else if(pp.getLeft() instanceof SQLInSubQueryExpr){
			SQLInSubQueryExpr left = (SQLInSubQueryExpr)pp.getLeft();
			if(left.getSubQuery().equals(sqlselect)){
				SQLExprImpl inlistExpr = null;
				if(null==param||param.isEmpty()){
					inlistExpr = new SQLNullExpr();
				}else{
					inlistExpr = new SQLInListExpr();
					((SQLInListExpr)inlistExpr).setTargetList(param);
					((SQLInListExpr)inlistExpr).setExpr(pp.getRight());
					((SQLInListExpr)inlistExpr).setNot(left.isNot());
					((SQLInListExpr)inlistExpr).setParent(left.getParent());
				}
				pp.setLeft(inlistExpr);
			}
		}else if(pp.getRight() instanceof SQLInSubQueryExpr){
			SQLInSubQueryExpr right = (SQLInSubQueryExpr)pp.getRight();
			if(right.getSubQuery().equals(sqlselect)){
				SQLExprImpl listExpr = null;
				if(null==param||param.isEmpty()){
					listExpr = new SQLNullExpr();
				}else{
					listExpr = new SQLListExpr();
					((SQLListExpr)listExpr).getItems().addAll(param);
				}
				pp.setRight(listExpr);
				
			}
		}
		return statement.toString();
	}

}


================================================
FILE: src/main/java/io/mycat/route/impl/middlerResultStrategy/InSubQueryResultHandler.java
================================================
package io.mycat.route.impl.middlerResultStrategy;

import java.util.List;

import com.alibaba.druid.sql.ast.SQLExprImpl;
import com.alibaba.druid.sql.ast.SQLObject;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr;
import com.alibaba.druid.sql.ast.expr.SQLInListExpr;
import com.alibaba.druid.sql.ast.expr.SQLInSubQueryExpr;
import com.alibaba.druid.sql.ast.expr.SQLNullExpr;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;


public class InSubQueryResultHandler implements RouteMiddlerReaultHandler {
	
	@Override
	public String dohandler(SQLStatement statement,SQLSelect sqlselect,SQLObject parent,List param) {
		SQLExprImpl inlistExpr = null;
		if(null==param||param.isEmpty()){
			inlistExpr = new SQLNullExpr();
		}else{
			inlistExpr = new SQLInListExpr();
			((SQLInListExpr)inlistExpr).setTargetList(param);
			((SQLInListExpr)inlistExpr).setExpr(((SQLInSubQueryExpr)parent).getExpr());
			((SQLInListExpr)inlistExpr).setNot(((SQLInSubQueryExpr)parent).isNot());
			((SQLInListExpr)inlistExpr).setParent(sqlselect.getParent());
		}
		if(parent.getParent() instanceof MySqlSelectQueryBlock){
			((MySqlSelectQueryBlock)parent.getParent()).setWhere(inlistExpr);
		}else if(parent.getParent() instanceof SQLBinaryOpExpr){
			SQLBinaryOpExpr pp = ((SQLBinaryOpExpr)parent.getParent());
			if(pp.getLeft().equals(parent)){
				pp.setLeft(inlistExpr);
			}else if(pp.getRight().equals(parent)){
				pp.setRight(inlistExpr);
			}
		}
		return statement.toString();
	}

}


================================================
FILE: src/main/java/io/mycat/route/impl/middlerResultStrategy/RouteMiddlerReaultHandler.java
================================================
package io.mycat.route.impl.middlerResultStrategy;

import java.util.List;

import com.alibaba.druid.sql.ast.SQLObject;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.statement.SQLSelect;

public interface RouteMiddlerReaultHandler {
	
	/**
	 * 处理中间结果
	 * @param statement
	 * @param sqlselect
	 * @param param
	 * @return
	 */
	String dohandler(SQLStatement statement,SQLSelect sqlselect,SQLObject parent,List param);

}


================================================
FILE: src/main/java/io/mycat/route/impl/middlerResultStrategy/SQLAllResultHandler.java
================================================
package io.mycat.route.impl.middlerResultStrategy;

import java.util.List;

import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLExprImpl;
import com.alibaba.druid.sql.ast.SQLObject;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator;
import com.alibaba.druid.sql.ast.expr.SQLNullExpr;
import com.alibaba.druid.sql.ast.expr.SQLValuableExpr;
import com.alibaba.druid.sql.ast.statement.SQLSelect;

/**
 * 对于 = 
	select * from test where id = all (select id from mytab where xxx)   --->
	改写后 sql为   all: select * from test where id = val1 and id = val2 …
 * @author lyj
 *
 */
public class SQLAllResultHandler implements RouteMiddlerReaultHandler{

	@Override
	public String dohandler(SQLStatement statement, SQLSelect sqlselect, SQLObject parent, List param) {
		if(parent.getParent() instanceof SQLBinaryOpExpr){
			
			SQLExprImpl inlistExpr = null;
			if(null==param||param.isEmpty()){
				inlistExpr = new SQLNullExpr();
				SQLBinaryOpExpr xp = (SQLBinaryOpExpr)parent.getParent();
				xp.setOperator(SQLBinaryOperator.Is);
				if(xp.getRight().equals(parent)){
					xp.setRight(inlistExpr);
				}else if(xp.getLeft().equals(parent)){
					xp.setLeft(inlistExpr);
				}
			}else{
				int len = param.size();
				
				SQLBinaryOpExpr xp = (SQLBinaryOpExpr)parent.getParent();
				SQLExpr left = null;
				if(xp.getRight().equals(parent)){
					left = xp.getLeft();
				}else if(xp.getLeft().equals(parent)){
					left = xp.getRight();
				}
								
				SQLBinaryOpExpr p = xp;
				for(int i=0;i items = pp.getItems();
			for(int i=0;i offset + "LEAR ".length()) {
			char c1 = stmt.charAt(++offset);
			char c2 = stmt.charAt(++offset);
			char c3 = stmt.charAt(++offset);
			char c4 = stmt.charAt(++offset);
			char c5 = stmt.charAt(++offset);
			if ((c1 == 'L' || c1 == 'l') && (c2 == 'E' || c2 == 'e')
					&& (c3 == 'A' || c3 == 'a') && (c4 == 'R' || c4 == 'r')
					&& (c5 == ' ' || c5 == '\t' || c5 == '\r' || c5 == '\n')) {
				return (offset << 8) | CLEAR;
			}
		}
		return OTHER;
	}

	private static int oCheck(String stmt, int offset) {
		if (stmt.length() > ++offset) {
			switch (stmt.charAt(offset)) {
			case 'F':
			case 'f':
				return ofCheck(stmt, offset);
			case 'N':
			case 'n':
				return onCheck(stmt, offset);
			default:
				return OTHER;
			}
		}
		return OTHER;
	}

	private static int onCheck(String stmt, int offset) {
		if (stmt.length() > offset + "line".length()) {
			char c1 = stmt.charAt(++offset);
			char c2 = stmt.charAt(++offset);
			char c3 = stmt.charAt(++offset);
			char c4 = stmt.charAt(++offset);
			if ((c1 == 'l' || c1 == 'L')
					&& (c2 == 'i' || c2 == 'I')
					&& (c3 == 'n' || c3 == 'N')
					&& (c4 == 'e' || c4 == 'E')
					&& (stmt.length() == ++offset || ParseUtil.isEOF(stmt
							.charAt(offset)))) {
				return ONLINE;
			}
		}
		return OTHER;
	}

	private static int ofCheck(String stmt, int offset) {
		if (stmt.length() > offset + "fline".length()) {
			char c1 = stmt.charAt(++offset);
			char c2 = stmt.charAt(++offset);
			char c3 = stmt.charAt(++offset);
			char c4 = stmt.charAt(++offset);
			char c5 = stmt.charAt(++offset);
			if ((c1 == 'f' || c1 == 'F')
					&& (c2 == 'l' || c2 == 'L')
					&& (c3 == 'i' || c3 == 'I')
					&& (c4 == 'n' || c4 == 'N')
					&& (c5 == 'e' || c5 == 'E')
					&& (stmt.length() == ++offset || ParseUtil.isEOF(stmt
							.charAt(offset)))) {
				return OFFLINE;
			}
		}
		return OTHER;
	}

	private static int sCheck(String stmt, int offset) {
		if (stmt.length() > ++offset) {
			switch (stmt.charAt(offset)) {
			case 'E':
			case 'e':
				return seCheck(stmt, offset);
			case 'H':
			case 'h':
				return show(stmt, offset);
			case 'W':
			case 'w':
				return swh(stmt, offset);
			case 'T':
			case 't':
				return stop(stmt, offset);
			default:
				return OTHER;
			}
		}
		return OTHER;
	}

	private static int seCheck(String stmt, int offset) {
		if (stmt.length() > ++offset) {
			switch (stmt.charAt(offset)) {
			case 'L':
			case 'l':
				return select(stmt, offset);
			case 'T':
			case 't':
				if (stmt.length() > ++offset) {
					char c = stmt.charAt(offset);
					if (c == ' ' || c == '\r' || c == '\n' || c == '\t'
							|| c == '/' || c == '#') {
						return SET;
					}
				}
				return OTHER;
			default:
				return OTHER;
			}
		}
		return OTHER;
	}

	private static int rCheck(String stmt, int offset) {
		if (stmt.length() > ++offset) {
			switch (stmt.charAt(offset)) {
			case 'E':
			case 'e':
				return reload(stmt, offset);
			case 'O':
			case 'o':
				return rollback(stmt, offset);
			default:
				return OTHER;
			}
		}
		return OTHER;
	}

	// RELOAD' '
	private static int reload(String stmt, int offset) {
		if (stmt.length() > offset + 5) {
			char c1 = stmt.charAt(++offset);
			char c2 = stmt.charAt(++offset);
			char c3 = stmt.charAt(++offset);
			char c4 = stmt.charAt(++offset);
			char c5 = stmt.charAt(++offset);
			if ((c1 == 'L' || c1 == 'l') && (c2 == 'O' || c2 == 'o')
					&& (c3 == 'A' || c3 == 'a') && (c4 == 'D' || c4 == 'd')
					&& (c5 == ' ' || c5 == '\t' || c5 == '\r' || c5 == '\n')) {
				return (offset << 8) | RELOAD;
			}
		}
		return OTHER;
	}

	// ROLLBACK' '
	private static int rollback(String stmt, int offset) {
		if (stmt.length() > offset + 7) {
			char c1 = stmt.charAt(++offset);
			char c2 = stmt.charAt(++offset);
			char c3 = stmt.charAt(++offset);
			char c4 = stmt.charAt(++offset);
			char c5 = stmt.charAt(++offset);
			char c6 = stmt.charAt(++offset);
			char c7 = stmt.charAt(++offset);
			if ((c1 == 'L' || c1 == 'l') && (c2 == 'L' || c2 == 'l')
					&& (c3 == 'B' || c3 == 'b') && (c4 == 'A' || c4 == 'a')
					&& (c5 == 'C' || c5 == 'c') && (c6 == 'K' || c6 == 'k')
					&& (c7 == ' ' || c7 == '\t' || c7 == '\r' || c7 == '\n')) {
				return (offset << 8) | ROLLBACK;
			}
		}
		return OTHER;
	}

	// SELECT' '
	private static int select(String stmt, int offset) {
		if (stmt.length() > offset + 4) {
			char c1 = stmt.charAt(++offset);
			char c2 = stmt.charAt(++offset);
			char c3 = stmt.charAt(++offset);
			char c4 = stmt.charAt(++offset);
			if ((c1 == 'E' || c1 == 'e') && (c2 == 'C' || c2 == 'c')
					&& (c3 == 'T' || c3 == 't')
					&& (c4 == ' ' || c4 == '\t' || c4 == '\r' || c4 == '\n')) {
				return (offset << 8) | SELECT;
			}
		}
		return OTHER;
	}

	// SHOW' '
	private static int show(String stmt, int offset) {
		if (stmt.length() > offset + 3) {
			char c1 = stmt.charAt(++offset);
			char c2 = stmt.charAt(++offset);
			char c3 = stmt.charAt(++offset);
			if ((c1 == 'O' || c1 == 'o') && (c2 == 'W' || c2 == 'w')
					&& (c3 == ' ' || c3 == '\t' || c3 == '\r' || c3 == '\n')) {
				return (offset << 8) | SHOW;
			}
		}
		return OTHER;
	}

	// SWITCH' '
	private static int swh(String stmt, int offset) {
		if (stmt.length() > offset + 5) {
			char c1 = stmt.charAt(++offset);
			char c2 = stmt.charAt(++offset);
			char c3 = stmt.charAt(++offset);
			char c4 = stmt.charAt(++offset);
			char c5 = stmt.charAt(++offset);
			if ((c1 == 'I' || c1 == 'i') && (c2 == 'T' || c2 == 't')
					&& (c3 == 'C' || c3 == 'c') && (c4 == 'H' || c4 == 'h')
					&& (c5 == ' ' || c5 == '\t' || c5 == '\r' || c5 == '\n')) {
				return (offset << 8) | SWITCH;
			}
		}
		return OTHER;
	}

	// STOP' '
	private static int stop(String stmt, int offset) {
		if (stmt.length() > offset + 3) {
			char c1 = stmt.charAt(++offset);
			char c2 = stmt.charAt(++offset);
			char c3 = stmt.charAt(++offset);
			if ((c1 == 'O' || c1 == 'o') && (c2 == 'P' || c2 == 'p')
					&& (c3 == ' ' || c3 == '\t' || c3 == '\r' || c3 == '\n')) {
				return (offset << 8) | STOP;
			}
		}
		return OTHER;
	}

	// KILL @
	private static int kill(String stmt, int offset) {
		if (stmt.length() > offset + 3) {
			char c1 = stmt.charAt(++offset);
			char c2 = stmt.charAt(++offset);
			char c3 = stmt.charAt(++offset);
			char c4 = stmt.charAt(++offset);
			if ((c1 == 'I' || c1 == 'i') && (c2 == 'L' || c2 == 'l')
					&& (c3 == 'L' || c3 == 'l')
					&& (c4 == ' ' || c4 == '\t' || c4 == '\r' || c4 == '\n')) {
				while (stmt.length() > ++offset) {
					switch (stmt.charAt(offset)) {
					case ' ':
					case '\t':
					case '\r':
					case '\n':
						continue;
					case '@':
						return killConnection(stmt, offset);
					default:
						return OTHER;
					}
				}
				return OTHER;
			}
		}
		return OTHER;
	}

	// KILL @@CONNECTION' ' XXXXXX
	private static int killConnection(String stmt, int offset) {
		if (stmt.length() > offset + "@CONNECTION ".length()) {
			char c1 = stmt.charAt(++offset);
			char c2 = stmt.charAt(++offset);
			char c3 = stmt.charAt(++offset);
			char c4 = stmt.charAt(++offset);
			char c5 = stmt.charAt(++offset);
			char c6 = stmt.charAt(++offset);
			char c7 = stmt.charAt(++offset);
			char c8 = stmt.charAt(++offset);
			char c9 = stmt.charAt(++offset);
			char c10 = stmt.charAt(++offset);
			char c11 = stmt.charAt(++offset);
			char c12 = stmt.charAt(++offset);
			if ((c1 == '@')
					&& (c2 == 'C' || c2 == 'c')
					&& (c3 == 'O' || c3 == 'o')
					&& (c4 == 'N' || c4 == 'n')
					&& (c5 == 'N' || c5 == 'n')
					&& (c6 == 'E' || c6 == 'e')
					&& (c7 == 'C' || c7 == 'c')
					&& (c8 == 'T' || c8 == 't')
					&& (c9 == 'I' || c9 == 'i')
					&& (c10 == 'O' || c10 == 'o')
					&& (c11 == 'N' || c11 == 'n')
					&& (c12 == ' ' || c12 == '\t' || c12 == '\r' || c12 == '\n')) {
				while (stmt.length() > ++offset) {
					switch (stmt.charAt(offset)) {
					case ' ':
					case '\t':
					case '\r':
					case '\n':
						continue;
					default:
						return (offset << 8) | KILL_CONN;
					}
				}
			}
		}
		return OTHER;
	}

}

================================================
FILE: src/main/java/io/mycat/route/parser/ManagerParseClear.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.parser;

import io.mycat.route.parser.util.ParseUtil;

/**
 * @author mycat
 */
public class ManagerParseClear {

    public static final int OTHER = -1;
    public static final int SLOW_SCHEMA = 1;
    public static final int SLOW_DATANODE = 2;

    public static int parse(String stmt, int offset) {
        int i = offset;
        for (; i < stmt.length(); i++) {
            switch (stmt.charAt(i)) {
            case ' ':
                continue;
            case '/':
            case '#':
                i = ParseUtil.comment(stmt, i);
                continue;
            case '@':
                return clear2Check(stmt, i);
            default:
                return OTHER;
            }
        }
        return OTHER;
    }

    // CLEAR @@SLOW
    static int clear2Check(String stmt, int offset) {
        if (stmt.length() > ++offset && stmt.charAt(offset) == '@'
                &&stmt.length() > offset + "SLOW ".length()) {
                char c1 = stmt.charAt(++offset);
                char c2 = stmt.charAt(++offset);
                char c3 = stmt.charAt(++offset);
                char c4 = stmt.charAt(++offset);
                char c5 = stmt.charAt(++offset);
                if ((c1 == 'S' || c1 == 's') && (c2 == 'L' || c2 == 'l') && (c3 == 'O' || c3 == 'o')
                        && (c4 == 'W' || c4 == 'w') && (c5 == ' ')) {
                    while (stmt.length() > ++offset) {
                        switch (stmt.charAt(offset)) {
                        case ' ':
                            continue;
                        case 'W':
                        case 'w':
                            return clear2WhereCheck(stmt, offset);
                        default:
                            return OTHER;
                        }
                    }
                }
        }
        return OTHER;
    }

    // CLEAR @@SLOW WHERE
    static int clear2WhereCheck(String stmt, int offset) {
        if (stmt.length() > offset + "HERE ".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            if ((c1 == 'H' || c1 == 'h') && (c2 == 'E' || c2 == 'e') && (c3 == 'R' || c3 == 'r')
                    && (c4 == 'E' || c4 == 'e') && (c5 == ' ')) {
                while (stmt.length() > ++offset) {
                    switch (stmt.charAt(offset)) {
                    case ' ':
                        continue;
                    case 'D':
                    case 'd':
                        return clear2DCheck(stmt, offset);
                    case 'S':
                    case 's':
                        return clear2SCheck(stmt, offset);
                    default:
                        return OTHER;
                    }
                }
            }
        }
        return OTHER;
    }

    // CLEAR @@SLOW WHERE DATANODE = XXXXXX
    static int clear2DCheck(String stmt, int offset) {
        if (stmt.length() > offset + "ATANODE".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);
            char c7 = stmt.charAt(++offset);
            if ((c1 == 'A' || c1 == 'a') && (c2 == 'T' || c2 == 't') && (c3 == 'A' || c3 == 'a')
                    && (c4 == 'N' || c4 == 'n') && (c5 == 'O' || c5 == 'o') && (c6 == 'D' || c6 == 'd')
                    && (c7 == 'E' || c7 == 'e')) {
                while (stmt.length() > ++offset) {
                    switch (stmt.charAt(offset)) {
                    case ' ':
                        continue;
                    case '=':
                        while (stmt.length() > ++offset) {
                            switch (stmt.charAt(offset)) {
                            case ' ':
                                continue;
                            default:
                                return (offset << 8) | SLOW_DATANODE;
                            }
                        }
                        return OTHER;
                    default:
                        return OTHER;
                    }
                }
            }
        }
        return OTHER;
    }

    // CLEAR @@SLOW WHERE SCHEMA = XXXXXX
    static int clear2SCheck(String stmt, int offset) {
        if (stmt.length() > offset + "CHEMA".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            if ((c1 == 'C' || c1 == 'c') && (c2 == 'H' || c2 == 'h') && (c3 == 'E' || c3 == 'e')
                    && (c4 == 'M' || c4 == 'm') && (c5 == 'A' || c5 == 'a')) {
                while (stmt.length() > ++offset) {
                    switch (stmt.charAt(offset)) {
                    case ' ':
                        continue;
                    case '=':
                        while (stmt.length() > ++offset) {
                            switch (stmt.charAt(offset)) {
                            case ' ':
                                continue;
                            default:
                                return (offset << 8) | SLOW_SCHEMA;
                            }
                        }
                        return OTHER;
                    default:
                        return OTHER;
                    }
                }
            }
        }
        return OTHER;
    }

}

================================================
FILE: src/main/java/io/mycat/route/parser/ManagerParseHeartbeat.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.parser;

import io.mycat.route.parser.util.Pair;

/**
 * @author songwie
 */
public final class ManagerParseHeartbeat {

    public static final int OTHER = -1;
    public static final int DATASOURCE = 1;
    
   // SHOW @@HEARTBEAT
    static int show2HeaCheck(String stmt, int offset) {
        if (stmt.length() > offset + "RTBEAT".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);
            if ((c1 == 'R' || c1 == 'r') && (c2 == 'T' || c2 == 't') & (c3 == 'B' || c3 == 'b')
                    && (c4 == 'E' || c4 == 'e') & (c5 == 'A' || c5 == 'a') && (c6 == 'T' || c6 == 't')) {
            	if (stmt.length() > offset + ".DETAIL".length()) {
            		char c7 = stmt.charAt(++offset);
                	if(c7 == '.'){
                		return show2HeaDetailCheck(stmt,offset);
                	}
            	}
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return ManagerParseShow.HEARTBEAT;
            }
        }
        return OTHER;
    }
    // SHOW @@HEARTBEAT.DETAIL
    static int show2HeaDetailCheck(String stmt, int offset) {
        if (stmt.length() > offset + "DETAIL".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);
            if ((c1 == 'D' || c1 == 'd') && (c2 == 'E' || c2 == 'e') & (c3 == 'T' || c3 == 't')
                    && (c4 == 'A' || c4 == 'a') & (c5 == 'I' || c5 == 'i') && (c6 == 'L' || c6 == 'l')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return ManagerParseShow.HEARTBEAT_DETAIL;
            }
        }
        return OTHER;
    }

    public static Pair getPair(String stmt) {
        int offset = stmt.indexOf("@@");
        String s = stmt.substring(++offset + " heartbeat.detail".length());
        char c = s.charAt(0);
        offset = 0;
        if(c == ' '){
        	char c1 = s.charAt(++offset);
    		char c2 = s.charAt(++offset);
    		char c3 = s.charAt(++offset);
    		char c4 = s.charAt(++offset);
    		char c5 = s.charAt(++offset);
    		char c6 = s.charAt(++offset);
    		char c7 = s.charAt(++offset);
    		char c8 = s.charAt(++offset);
    		char c9 = s.charAt(++offset);
    		char c10 = s.charAt(++offset);
    		char c11 = s.charAt(++offset);
    		if ((c1 == 'W' || c1 == 'w') && (c2 == 'H' || c2 == 'h') && (c3 == 'E' || c3 == 'e')
                    && (c4 == 'R' || c4 == 'r') && (c5 == 'E' || c5 == 'e')
                    && c6 == ' ' && (c7 == 'N' || c7 == 'n') && (c8 == 'A' || c8 == 'a') && (c9 == 'M' || c9 == 'm')
                    && (c10 == 'E' || c10 == 'e') && (c11 == '=')) {
    	        String name = s.substring(++offset).trim();
                return new Pair("name", name);
    		}
        }
        return new Pair("name", "");
    }
 
}

================================================
FILE: src/main/java/io/mycat/route/parser/ManagerParseKill.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.parser;

import io.mycat.route.parser.util.ParseUtil;

/**
 * @author mycat
 */
public final class ManagerParseKill {

    public static final int OTHER = -1;
    public static final int CONNECTION = 1;

    public static int parse(String stmt, int offset) {
        int i = offset;
        for (; i < stmt.length(); i++) {
            switch (stmt.charAt(i)) {
            case ' ':
                continue;
            case '/':
            case '#':
                i = ParseUtil.comment(stmt, i);
                continue;
            case '@':
                return kill2Check(stmt, i);
            default:
                return OTHER;
            }
        }
        return OTHER;
    }

    // KILL @@CONNECTION
    static int kill2Check(String stmt, int offset) {
        if (stmt.length() > ++offset && stmt.charAt(offset) == '@'
                && stmt.length() > offset + 10) {
                char c1 = stmt.charAt(++offset);
                char c2 = stmt.charAt(++offset);
                char c3 = stmt.charAt(++offset);
                char c4 = stmt.charAt(++offset);
                char c5 = stmt.charAt(++offset);
                char c6 = stmt.charAt(++offset);
                char c7 = stmt.charAt(++offset);
                char c8 = stmt.charAt(++offset);
                char c9 = stmt.charAt(++offset);
                char c10 = stmt.charAt(++offset);
                if ((c1 == 'C' || c1 == 'c') && (c2 == 'O' || c2 == 'o') && (c3 == 'N' || c3 == 'n')
                        && (c4 == 'N' || c4 == 'n') && (c5 == 'E' || c5 == 'e') && (c6 == 'C' || c6 == 'c')
                        && (c7 == 'T' || c7 == 't') && (c8 == 'I' || c8 == 'i') && (c9 == 'O' || c9 == 'o')
                        && (c10 == 'N' || c10 == 'n')) {
                    if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                        return OTHER;
                    }
                    return CONNECTION;
                }
        }
        return OTHER;
    }

}

================================================
FILE: src/main/java/io/mycat/route/parser/ManagerParseReload.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.parser;

import io.mycat.route.parser.util.ParseUtil;

/**
 * @author mycat
 */
public final class ManagerParseReload {

    public static final int OTHER = -1;
    public static final int CONFIG = 1;
    public static final int ROUTE = 2;
    public static final int USER = 3;
    public static final int USER_STAT = 4;
    public static final int CONFIG_ALL = 5;
    public static final int SQL_SLOW = 6;
    public static final int QUERY_CF = 8;
    public static final int SQL_STAT = 9;
       
    public static int parse(String stmt, int offset) {
        int i = offset;
        for (; i < stmt.length(); i++) {
            switch (stmt.charAt(i)) {
            case ' ':
                continue;
            case '/':
            case '#':
                i = ParseUtil.comment(stmt, i);
                continue;
            case '@':
                return reload2Check(stmt, i);
            default:
                return OTHER;
            }
        }
        return OTHER;
    }

    static int reload2Check(String stmt, int offset) {
        if (stmt.length() > ++offset && stmt.charAt(offset) == '@'
                && stmt.length() > ++offset) {
                switch (stmt.charAt(offset)) {
                case 'C':
                case 'c':
                    return reload2CCheck(stmt, offset);
                case 'R':
                case 'r':
                    return reload2RCheck(stmt, offset);
                case 'U':
                case 'u':
                    return reload2UCheck(stmt, offset);
                case 'S':
                case 's':
                    return reload2SCheck(stmt, offset);       
                case 'Q':
                case 'q':
                    return reload2QCheck(stmt, offset);     
                default:
                    return OTHER;
                }
        }
        return OTHER;
    }

    // RELOAD @@CONFIG
    static int reload2CCheck(String stmt, int offset) {
        if (stmt.length() > offset + 5) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            if ((c1 == 'O' || c1 == 'o') && (c2 == 'N' || c2 == 'n') && (c3 == 'F' || c3 == 'f')
                    && (c4 == 'I' || c4 == 'i') && (c5 == 'G' || c5 == 'g')) {
                if (stmt.length() > offset + 4)
                {
                    char c6 = stmt.charAt(++offset);
                    char c7 = stmt.charAt(++offset);
                    char c8 = stmt.charAt(++offset);
                    char c9 = stmt.charAt(++offset);
                    if ((c6 == '_' || c6 == '-') && (c7 == 'A' || c7 == 'a') && (c8 == 'L' || c8 == 'l')
                            && (c9 == 'L' || c9 == 'l') ) {
                          return CONFIG_ALL;
                    }
                }
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }

                return CONFIG;
            }
        }
        return OTHER;
    }

    // RELOAD @@ROUTE
    static int reload2RCheck(String stmt, int offset) {
        if (stmt.length() > offset + 4) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            if ((c1 == 'O' || c1 == 'o') && (c2 == 'U' || c2 == 'u') && (c3 == 'T' || c3 == 't')
                    && (c4 == 'E' || c4 == 'e')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return ROUTE;
            }
        }
        return OTHER;
    }

    // RELOAD @@USER
    static int reload2UCheck(String stmt, int offset) {
        if (stmt.length() > offset + 3) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            if ((c1 == 'S' || c1 == 's') && (c2 == 'E' || c2 == 'e') && (c3 == 'R' || c3 == 'r')) {
            	
            	
            	if (stmt.length() > offset + 5)
                {
                    char c6 = stmt.charAt(++offset);
                    char c7 = stmt.charAt(++offset);
                    char c8 = stmt.charAt(++offset);
                    char c9 = stmt.charAt(++offset);
                    char c10 = stmt.charAt(++offset);
                    
                    if ((c6 == '_' || c6 == '-') && (c7 == 'S' || c7 == 's') && (c8 == 'T' || c8 == 't')
                            && (c9 == 'A' || c9 == 'a') && (c10 == 'T' || c10 == 't') ) {
                          return USER_STAT;
                    }
                }
            	
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return USER;
            }
        }
        return OTHER;
    }
    
    // RELOAD @@SQL
    static int reload2SCheck(String stmt, int offset) {
        if (stmt.length() > offset + 4) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);  
            
            // reload @@sqlslow
            if ((c1 == 'Q' || c1 == 'q') && (c2 == 'L' || c2 == 'l') && (c3 == 's' || c3 == 'S')
                    && (c4 == 'L' || c4 == 'l') && (c5 == 'O' || c5 == 'o') && (c6 == 'W' || c6 == 'w')
                    && stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return SQL_SLOW ;
            }

            // reload @@sqlstat
            if ((c1 == 'Q' || c1 == 'q') && (c2 == 'L' || c2 == 'l') && (c3 == 's' || c3 == 'S')
                && (c4 == 'T' || c4 == 't') && (c5 == 'A' || c5 == 'a') && (c6 == 'T' || c6 == 't')
                && stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                return SQL_STAT ;
            }

            return OTHER;
        }
        return OTHER;
    }
    
    // RELOAD @@QUERY
    static int reload2QCheck(String stmt, int offset) {
        if (stmt.length() > offset + 4) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);  
            char c7 = stmt.charAt(++offset);

            if ((c1 == 'U' || c1 == 'u') && (c2 == 'E' || c2 == 'e') && (c3 == 'R' || c3 == 'r')
                    && (c4 == 'Y' || c4 == 'y') && (c5 == '_' ) && (c6 == 'C' || c6 == 'c') && (c7 == 'F' || c7 == 'f') ) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return QUERY_CF ;
                }
                return OTHER;
            }
        }
        return OTHER;
    }
}

================================================
FILE: src/main/java/io/mycat/route/parser/ManagerParseRollback.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.parser;

import io.mycat.route.parser.util.ParseUtil;

/**
 * @author mycat
 */
public final class ManagerParseRollback {

    public static final int OTHER = -1;
    public static final int CONFIG = 1;
    public static final int ROUTE = 2;
    public static final int USER = 3;

    public static int parse(String stmt, int offset) {
        int i = offset;
        for (; i < stmt.length(); i++) {
            switch (stmt.charAt(i)) {
            case ' ':
                continue;
            case '/':
            case '#':
                i = ParseUtil.comment(stmt, i);
                continue;
            case '@':
                return rollback2Check(stmt, i);
            default:
                return OTHER;
            }
        }
        return OTHER;
    }

    static int rollback2Check(String stmt, int offset) {
        if (stmt.length() > ++offset && stmt.charAt(offset) == '@'
                && stmt.length() > ++offset) {
                switch (stmt.charAt(offset)) {
                case 'C':
                case 'c':
                    return rollback2CCheck(stmt, offset);
                case 'R':
                case 'r':
                    return rollback2RCheck(stmt, offset);
                case 'U':
                case 'u':
                    return rollback2UCheck(stmt, offset);
                default:
                    return OTHER;
                }
        }
        return OTHER;
    }

    // ROLLBACK @@CONFIG
    static int rollback2CCheck(String stmt, int offset) {
        if (stmt.length() > offset + 5) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            if ((c1 == 'O' || c1 == 'o') && (c2 == 'N' || c2 == 'n') && (c3 == 'F' || c3 == 'f')
                    && (c4 == 'I' || c4 == 'i') && (c5 == 'G' || c5 == 'g')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return CONFIG;
            }
        }
        return OTHER;
    }

    // ROLLBACK @@ROUTE
    static int rollback2RCheck(String stmt, int offset) {
        if (stmt.length() > offset + 4) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            if ((c1 == 'O' || c1 == 'o') && (c2 == 'U' || c2 == 'u') && (c3 == 'T' || c3 == 't')
                    && (c4 == 'E' || c4 == 'e')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return ROUTE;
            }
        }
        return OTHER;
    }

    // ROLLBACK @@USER
    static int rollback2UCheck(String stmt, int offset) {
        if (stmt.length() > offset + 3) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            if ((c1 == 'S' || c1 == 's') && (c2 == 'E' || c2 == 'e') && (c3 == 'R' || c3 == 'r')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return USER;
            }
        }
        return OTHER;
    }

}

================================================
FILE: src/main/java/io/mycat/route/parser/ManagerParseSelect.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.parser;

import io.mycat.route.parser.util.ParseUtil;

/**
 * @author mycat
 */
public final class ManagerParseSelect {

    public static final int OTHER = -1;
    public static final int VERSION_COMMENT = 1;
    public static final int SESSION_AUTO_INCREMENT = 2;
    public static final int SESSION_TX_READ_ONLY = 3;

    private static final char[] _VERSION_COMMENT = "VERSION_COMMENT".toCharArray();
    private static final char[] _SESSION_AUTO_INCREMENT = "SESSION.AUTO_INCREMENT_INCREMENT".toCharArray();
    private static final char[] _SESSION_TX_READ_ONLY = "SESSION.TX_READ_ONLY".toCharArray();

    public static int parse(String stmt, int offset) {
        int i = offset;
        for (; i < stmt.length(); i++) {
            switch (stmt.charAt(i)) {
            case ' ':
                continue;
            case '/':
            case '#':
                i = ParseUtil.comment(stmt, i);
                continue;
            case '@':
                return select2Check(stmt, i);
            default:
                return OTHER;
            }
        }
        return OTHER;
    }

    static int select2Check(String stmt, int offset) {
        if (stmt.length() > ++offset && stmt.charAt(offset) == '@'
                && stmt.length() > ++offset) {
                switch (stmt.charAt(offset)) {
                case 'S':
                case 's':
                    return select2SCheck(stmt, offset);
                case 'V':
                case 'v':
                    return select2VCheck(stmt, offset);
                default:
                    return OTHER;
                }
        }
        return OTHER;
    }

    // VERSION_COMMENT
    static int select2VCheck(String stmt, int offset) {
        int length = offset + _VERSION_COMMENT.length;
        if (stmt.length() >= length
                && ParseUtil.compare(stmt, offset, _VERSION_COMMENT)) {
                if (stmt.length() > length && stmt.charAt(length) != ' ') {
                    return OTHER;
                }
                return VERSION_COMMENT;
        }
        return OTHER;
    }

    // SESSION.AUTO_INCREMENT_INCREMENT or SESSION.TX_READ_ONLY
    static int select2SCheck(String stmt, int offset) {
        int length = offset + _SESSION_AUTO_INCREMENT.length;
        if (stmt.length() >= length
                && ParseUtil.compare(stmt, offset, _SESSION_AUTO_INCREMENT)) {
                if (stmt.length() > length && stmt.charAt(length) != ' ') {
                    return OTHER;
                }
                return SESSION_AUTO_INCREMENT;
        } else if (stmt.length() >= (offset + _SESSION_TX_READ_ONLY.length) 
        		&& ParseUtil.compare(stmt, offset, _SESSION_TX_READ_ONLY)) {
        	if (stmt.length() > length && stmt.charAt(length) != ' ') {
                return OTHER;
            }
        	return SESSION_TX_READ_ONLY;
        }
        
        return OTHER;
    }

}

================================================
FILE: src/main/java/io/mycat/route/parser/ManagerParseShow.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.parser;

import io.mycat.route.parser.util.ParseUtil;

/**
 * @author mycat
 */
public final class ManagerParseShow {

    public static final int OTHER = -1;
    public static final int COMMAND = 1;
    public static final int CONNECTION = 2;
    public static final int DATABASE = 3;
    public static final int DATANODE = 4;
    public static final int DATASOURCE = 5;
    public static final int HELP = 6;
    public static final int PARSER = 7;
    public static final int PROCESSOR = 8;
    public static final int ROUTER = 9;
    public static final int SERVER = 10;
    public static final int SQL = 11;
    public static final int SQL_DETAIL = 12;
    public static final int SQL_EXECUTE = 13;
    public static final int SQL_SLOW = 14;
    public static final int SQL_SUM_USER = 15;
    public static final int SQL_SUM_TABLE = 16;
    public static final int SQL_HIGH = 17;
    public static final int SQL_CONDITION = 18;
    public static final int SQL_LARGE = 19;
    public static final int SQL_RESULTSET = 20;
    
    public static final int THREADPOOL = 21;
    public static final int TIME_CURRENT = 22;
    public static final int TIME_STARTUP = 23;
    public static final int VERSION = 24;
    public static final int VARIABLES = 25;
    public static final int COLLATION = 26;
    public static final int CONNECTION_SQL = 27;
    public static final int DATANODE_WHERE = 28;
    public static final int DATASOURCE_WHERE = 29;
    public static final int HEARTBEAT = 30;
    public static final int SLOW_DATANODE = 31;
    public static final int SLOW_SCHEMA = 32;
    public static final int BACKEND = 33;
    public static final int BACKEND_OLD = 34;
    
    public static final int CACHE = 35;
    public static final int SESSION = 36;
    public static final int SYSPARAM = 37;
    public static final int SYSLOG = 38;
    public static final int HEARTBEAT_DETAIL = 39;
    public static final int DATASOURCE_SYNC = 40;
    public static final int DATASOURCE_SYNC_DETAIL = 41;
    public static final int DATASOURCE_CLUSTER = 42;

	public static final int WHITE_HOST = 43;
	public static final int WHITE_HOST_SET = 44;
    public static final int DIRECTMEMORY_TOTAL = 45;
    public static final int DIRECTMEMORY_DETAILl = 46;

    public static final int CHECK_GLOBAL = 47;

    public static int parse(String stmt, int offset) {
        int i = offset;
        for (; i < stmt.length(); i++) {
            switch (stmt.charAt(i)) {
            case ' ':
                continue;
            case '/':
            case '#':
                i = ParseUtil.comment(stmt, i);
                continue;
            case '@':
                return show2Check(stmt, i);
            case 'C':
            case 'c':
                return showCCheck(stmt, i);
			case 'd':
			case 'D':
				return show2DCheck(stmt, i);
			case 'V':
            case 'v':
                return showVCheck(stmt, i);
            default:
                return OTHER;
            }
        }
        return OTHER;
    }

    // SHOW @
    static int show2Check(String stmt, int offset) {
        if (stmt.length() > ++offset && stmt.charAt(offset) == '@'
                && stmt.length() > ++offset) {
                switch (stmt.charAt(offset)) {
                case 'B':
                case 'b':
                    return show2BCheck(stmt, offset);
                case 'C':
                case 'c':
                    return show2CCheck(stmt, offset);
                case 'D':
                case 'd':
                    return show2DCheck(stmt, offset);
                case 'H':
                case 'h':
                    return show2HCheck(stmt, offset);
                case 'P':
                case 'p':
                    return show2PCheck(stmt, offset);
                case 'R':
                case 'r':
                    return show2RCheck(stmt, offset);
                case 'S':
                case 's':
                    return show2SCheck(stmt, offset);
                case 'T':
                case 't':
                    return show2TCheck(stmt, offset);
                case 'V':
                case 'v':
                    return show2VCheck(stmt, offset);
                case 'W':
                case 'w':
                    return show2WCheck(stmt, offset);                    
                default:
                    return OTHER;
                }
        }
        return OTHER;
    }

    // SHOW COLLATION
    static int showCCheck(String stmt, int offset) {
        if (stmt.length() > offset + "OLLATION".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);
            char c7 = stmt.charAt(++offset);
            char c8 = stmt.charAt(++offset);
            if ((c1 == 'O' || c1 == 'o') && (c2 == 'L' || c2 == 'l') && (c3 == 'L' || c3 == 'l')
                    && (c4 == 'A' || c4 == 'a') && (c5 == 'T' || c5 == 't') && (c6 == 'I' || c6 == 'i')
                    && (c7 == 'O' || c7 == 'o') && (c8 == 'N' || c8 == 'n')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return COLLATION;
            }
        }
        return OTHER;
    }

    // SHOW VARIABLES
    static int showVCheck(String stmt, int offset) {
        if (stmt.length() > offset + "ARIABLES".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);
            char c7 = stmt.charAt(++offset);
            char c8 = stmt.charAt(++offset);
            if ((c1 == 'A' || c1 == 'a') && (c2 == 'R' || c2 == 'r') && (c3 == 'I' || c3 == 'i')
                    && (c4 == 'A' || c4 == 'a') && (c5 == 'B' || c5 == 'b') && (c6 == 'L' || c6 == 'l')
                    && (c7 == 'E' || c7 == 'e') && (c8 == 'S' || c8 == 's')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return VARIABLES;
            }
        }
        return OTHER;
    }

    // SHOW @@BACKEND
    static int show2BCheck(String stmt, int offset) {
        if (stmt.length() > offset + "ACKEND".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);
            if ((c1 == 'A' || c1 == 'a') && (c2 == 'C' || c2 == 'c') && (c3 == 'K' || c3 == 'k')
                    && (c4 == 'E' || c4 == 'e') && (c5 == 'N' || c5 == 'n') && (c6 == 'D' || c6 == 'd')) {
                
                if (stmt.length() > ++offset) {
                    switch (stmt.charAt(offset)) {
                    case ';':
                    case ' ':
                        return BACKEND;
                    case '.':
                        return show2BackendOld(stmt, offset);
                    default:
                        return OTHER;
                    }
                }
                return BACKEND;
                
            }
        }
        return OTHER;
    }
    
    static int show2BackendOld(String stmt, int offset) {
    	  if (stmt.length() > offset + "OLD".length()) {
              char c1 = stmt.charAt(++offset);
              char c2 = stmt.charAt(++offset);
              char c3 = stmt.charAt(++offset);
              if ((c1 == 'O' || c1 == 'o') && (c2 == 'L' || c2 == 'l') && (c3 == 'D' || c3 == 'd')) {
                  if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                      return OTHER;
                  }
                  return BACKEND_OLD;
              }
          }
          return OTHER;
    }

    // SHOW @@C
    static int show2CCheck(String stmt, int offset) {
        if (stmt.length() > ++offset) {
            switch (stmt.charAt(offset)) {
            case 'O':
            case 'o':
                return show2CoCheck(stmt, offset);
            case 'A':
            case 'a':
            	 return show2CACheck(stmt, offset);
            case 'h':
            case 'H':
            	 return show2CHCheck(stmt, offset);
            default:
                return OTHER;
            }
        }
        return OTHER;
    }
    // SHOW @@CHECK_GLOBAL
    private static int show2CHCheck(String stmt, int offset) {
    	if (stmt.length() > offset + "ECK_GLOBAL".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);
            char c7 = stmt.charAt(++offset);
            char c8 = stmt.charAt(++offset);
            char c9 = stmt.charAt(++offset);
            char c10 = stmt.charAt(++offset);
            
            if ((c1 == 'E' || c1 == 'e') && (c2 == 'C' || c2 == 'c') && (c3 == 'K' || c3 == 'k')
                    && (c4 == '_') && (c5 == 'G' || c5 == 'g') && (c6 == 'L' || c6 == 'l')
                    && (c7 == 'O' || c7 == 'o') && (c8 == 'B' || c8 == 'b')  && (c9 == 'A' || c9 == 'a')
                    && (c10 == 'L' || c10 == 'l')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return CHECK_GLOBAL;
            }
        }
        return OTHER;
	}

	// SHOW @@CACHE
    private static int show2CACheck(String stmt, int offset) {
    	String remain=stmt.substring(offset);
    	 if(remain.equalsIgnoreCase("ACHE"))
    	 {
    		 return CACHE;
    	 }
    	return OTHER;
	}

	// SHOW @@DATA
    static int show2DCheck(String stmt, int offset) {
        if (stmt.length() > offset + "ATA".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            if ((c1 == 'A' || c1 == 'a') && (c2 == 'T' || c2 == 't') && (c3 == 'A' || c3 == 'a')
                    && stmt.length() > ++offset) {
                    switch (stmt.charAt(offset)) {
                    case 'B':
                    case 'b':
                        return show2DataBCheck(stmt, offset);
                    case 'N':
                    case 'n':
                        return show2DataNCheck(stmt, offset);
                    case 'S':
                    case 's':
                        return show2DataSCheck(stmt, offset);
                    default:
                        return OTHER;
                    }
            }else if( (c1 == 'I'|| c1 == 'i')
                    &&(c2 == 'R' || c2 == 'r')
                    && (c3 == 'E' || c3 == 'e')
                    && stmt.length() > ++offset){   /**DIRECTMEMORY**/
                    switch (stmt.charAt(offset)) {
                        case 'C':
                        case 'c':
                            return show2DirectMemoryCheck(stmt,offset);
                        default:
                            return OTHER;
                    }
            }
        }
        return OTHER;
    }
 // SHOW @@DIRECT_MEMORY=1 or 0
    static int show2DirectMemoryCheck(String stmt, int offset) {
        if (stmt.length() > offset + "TMEMORY".length()) {

            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);
            char c7 = stmt.charAt(++offset);
            char c8 = stmt.charAt(++offset);

            if ((c1 == 'T' || c1 == 't')
                    && (c2 == 'M' || c2 == 'm')
                    && (c3 == 'E' || c3 == 'e')
                    && (c4 == 'M' || c4 == 'm')
                    && (c5 == 'O' || c5 == 'o')
                    && (c6 == 'R' || c6 == 'r')
                    && (c7 == 'Y' || c7 == 'y')
                    && (c8 == '=' || c8 == '=')
                    && stmt.length() > ++offset) {

                switch (stmt.charAt(offset)) {
                    case '1':
                        return DIRECTMEMORY_TOTAL;
                    case '2':
                        return DIRECTMEMORY_DETAILl;
                    default:
                        return OTHER;
                }

            }
        }

        return OTHER;
    }
    // SHOW @@DataSyn
    static int show2DataSynCheck(String stmt, int offset) {
        if (stmt.length() > ++offset) {
            switch (stmt.charAt(offset)) {
            case 'S':
            case 's':
            	if (stmt.length() > offset + "yn".length()) {
            		char c1 = stmt.charAt(++offset);
                    char c2 = stmt.charAt(++offset);
                    if ((c1 == 'Y' || c1 == 'y') && (c2 == 'N' || c2 == 'n')){
                        switch (stmt.charAt(++offset)) {
                        case 'S':
                        case 's':
                        	return show2SynStatuslCheck(stmt,offset);
                        case 'D':
                        case 'd':
                        	return show2SynDetailCheck(stmt,offset);
                        default:
                            return OTHER;
                        }

                    }else{
                    	return OTHER;
                    }
            	}
            case 'C':
            case 'c':
            	if (stmt.length() > offset + "luster".length()) {
            		char c1 = stmt.charAt(++offset);
                    char c2 = stmt.charAt(++offset);
                    char c3 = stmt.charAt(++offset);
                    char c4 = stmt.charAt(++offset);
                    char c5 = stmt.charAt(++offset);
                    char c6 = stmt.charAt(++offset);
                    if ((c1 == 'L' || c1 == 'l') && (c2 == 'U' || c2 == 'u') 
                    		&& (c3 == 'S' || c3 == 's') && (c4 == 'T' || c4 == 't')
                    		&& (c5 == 'E' || c5 == 'e')&& (c6 == 'R' || c6 == 'r') ){
                    	return DATASOURCE_CLUSTER;
                    }else{
                    	return OTHER;
                    }
            	}
            default:
                return OTHER;
            }
        }
        return OTHER;
    }
    //show @@datasource.syndetail
    static int show2SynDetailCheck(String stmt, int offset) {
        if (stmt.length() > offset + "etail".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);

            if ((c1 == 'E' || c1 == 'e') && (c2 == 'T' || c2 == 't') && (c3 == 'A' || c3 == 'a')
                    && (c4 == 'I' || c4 == 'i') && (c5 == 'L' || c5 == 'l')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return DATASOURCE_SYNC_DETAIL;
            }
        }
        return OTHER;
    }
    //show @@datasource.synstatus  
    static int show2SynStatuslCheck(String stmt, int offset) {
        if (stmt.length() > offset + "tatus".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);

            if ((c1 == 'T' || c1 == 't') && (c2 == 'A' || c2 == 'a') && (c3 == 'T' || c3 == 't')
                    && (c4 == 'U' || c4 == 'u') && (c5 == 'S' || c5 == 's')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return DATASOURCE_SYNC;
            }
        }
        return OTHER;
    }

    // SHOW @@HELP
    static int show2HCheck(String stmt, int offset) {
        if (stmt.length() > ++offset) {
            switch (stmt.charAt(offset)) {
            case 'E':
            case 'e':
                return show2HeCheck(stmt, offset);
            default:
                return OTHER;
            }
        }
        return OTHER;
    }

    // SHOW @@HE
    static int show2HeCheck(String stmt, int offset) {
        if (stmt.length() > ++offset) {
            switch (stmt.charAt(offset)) {
            case 'L':
            case 'l':
                return show2HelCheck(stmt, offset);
            case 'A':
            case 'a':
                return ManagerParseHeartbeat.show2HeaCheck(stmt, offset);
            default:
                return OTHER;
            }
        }
        return OTHER;
    }

    // SHOW @@HELP
    static int show2HelCheck(String stmt, int offset) {
        if (stmt.length() > offset + "P".length()) {
            char c1 = stmt.charAt(++offset);
            if ((c1 == 'P' || c1 == 'p')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return HELP;
            }
        }
        return OTHER;
    }

    

    // SHOW @@P
    static int show2PCheck(String stmt, int offset) {
        if (stmt.length() > ++offset) {
            switch (stmt.charAt(offset)) {
            case 'A':
            case 'a':
                return show2PaCheck(stmt, offset);
            case 'R':
            case 'r':
                return show2PrCheck(stmt, offset);
            default:
                return OTHER;
            }
        }
        return OTHER;
    }

    // SHOW @@ROUTER
    static int show2RCheck(String stmt, int offset) {
        if (stmt.length() > offset + "OUTER".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            if ((c1 == 'O' || c1 == 'o') && (c2 == 'U' || c2 == 'u') && (c3 == 'T' || c3 == 't')
                    && (c4 == 'E' || c4 == 'e') && (c5 == 'R' || c5 == 'r')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return ROUTER;
            }
        }
        return OTHER;
    }

    // SHOW @@S
    static int show2SCheck(String stmt, int offset) {
        if (stmt.length() > ++offset) {
            switch (stmt.charAt(offset)) {
            case 'E':
            case 'e':
                return show2SeCheck(stmt, offset);
            case 'Q':
            case 'q':
                return show2SqCheck(stmt, offset);
            case 'L':
            case 'l':
                return show2SlCheck(stmt, offset);
            case 'Y':
            case 'y':
            	return show2SyCheck(stmt, offset);
            default:
                return OTHER;
            }
        }
        return OTHER;
    }
    
	// SHOW @@SLOW
    static int show2SlCheck(String stmt, int offset) {
        if (stmt.length() > offset + "OW ".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            if ((c1 == 'O' || c1 == 'o') && (c2 == 'W' || c2 == 'w') && c3 == ' ') {
                while (stmt.length() > ++offset) {
                    switch (stmt.charAt(offset)) {
                    case ' ':
                        continue;
                    case 'W':
                    case 'w':
                        return show2SlowWhereCheck(stmt, offset);
                    default:
                        return OTHER;
                    }
                }
            }
        }
        return OTHER;
    }
    
    // SHOW @@SYSPARAM
    static int show2sysparam(String stmt, int offset) {
        if (stmt.length() > offset + "ARAM".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);

            if ((c1 == 'A' || c1 == 'a')  && (c2 == 'R' || c2 == 'r') 
            		&& (c3 == 'A' || c3 == 'a') && (c4 == 'M' || c4 == 'm')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return SYSPARAM;
            }
        }
        return OTHER;
    }
    
    static int show2syslog(String stmt, int offset) {
    	
    	if (stmt.length() > offset + "SLOG".length()) {    		
    		 
    		 char c1 = stmt.charAt(++offset);
             char c2 = stmt.charAt(++offset);
             char c3 = stmt.charAt(++offset);
             
             if ( (c1 == 'O' || c1 == 'o') && (c2 == 'G' || c2 == 'g') && c3 == ' ' ) {
            	 
            	 char c4 = stmt.charAt(++offset);
                 char c5 = stmt.charAt(++offset);
                 char c6 = stmt.charAt(++offset);
                 char c7 = stmt.charAt(++offset);
                 char c8 = stmt.charAt(++offset);
                 
                 if ((c4 == 'L' || c4 == 'l') && (c5 == 'I' || c5 == 'i') && (c6 == 'M' || c6 == 'm')
                         && (c7 == 'I' || c7 == 'i') && (c8 == 'T' || c8 == 't')  ) {
                	 
                     while (stmt.length() > ++offset) {
                         switch (stmt.charAt(offset)) {
                         case ' ':
                             continue;
                         case '=':
                             while (stmt.length() > ++offset) {
                                 switch (stmt.charAt(offset)) {
                                 case ' ':
                                     continue;
                                 default:
                                     return (offset << 8) | SYSLOG;
                                 }
                             }
                             return OTHER;
                         default:
                             return OTHER;
                         }
                     }
                 }

                 return SYSLOG;
             }
    	}
    	
    	return OTHER; 
    }
    
    // SHOW @@SYSPARAM
    // SHOW @@SYSLOG LIMIT=1000
    static int show2SyCheck(String stmt, int offset) {
    	
    	if (stmt.length() > offset + "YS".length()) {    		
    		char c1 = stmt.charAt(++offset);
    		char c2 = stmt.charAt(++offset);
    		if ( c1 == 'S' || c1 == 's' ) {	    		
    			switch (c2) {
	            case 'L':
	            case 'l':
	                return show2syslog(stmt, offset);
	            case 'P':
	            case 'p':
	                return show2sysparam(stmt, offset);
	            default:
	                return OTHER;
	            }
    		}
    	}
        return OTHER;    	
    }
   
    

    // SHOW @@SLOW WHERE
    static int show2SlowWhereCheck(String stmt, int offset) {
        if (stmt.length() > offset + "HERE".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            if ((c1 == 'H' || c1 == 'h') && (c2 == 'E' || c2 == 'e') && (c3 == 'R' || c3 == 'r')
                    && (c4 == 'E' || c4 == 'e')) {
                while (stmt.length() > ++offset) {
                    switch (stmt.charAt(offset)) {
                    case ' ':
                        continue;
                    case 'D':
                    case 'd':
                        return show2SlowWhereDCheck(stmt, offset);
                    case 'S':
                    case 's':
                        return show2SlowWhereSCheck(stmt, offset);
                    default:
                        return OTHER;
                    }
                }
            }
        }
        return OTHER;
    }

    // SHOW @@SLOW WHERE DATANODE= XXXXXX
    static int show2SlowWhereDCheck(String stmt, int offset) {
        if (stmt.length() > offset + "ATANODE".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);
            char c7 = stmt.charAt(++offset);
            if ((c1 == 'A' || c1 == 'a') && (c2 == 'T' || c2 == 't') && (c3 == 'A' || c3 == 'a')
                    && (c4 == 'N' || c4 == 'n') && (c5 == 'O' || c5 == 'o') && (c6 == 'D' || c6 == 'd')
                    && (c7 == 'E' || c7 == 'e')) {
                while (stmt.length() > ++offset) {
                    switch (stmt.charAt(offset)) {
                    case ' ':
                        continue;
                    case '=':
                        while (stmt.length() > ++offset) {
                            switch (stmt.charAt(offset)) {
                            case ' ':
                                continue;
                            default:
                                return (offset << 8) | SLOW_DATANODE;
                            }
                        }
                        return OTHER;
                    default:
                        return OTHER;
                    }
                }
            }
        }
        return OTHER;
    }

    // SHOW @@SLOW WHERE SCHEMA= XXXXXX
    static int show2SlowWhereSCheck(String stmt, int offset) {
        if (stmt.length() > offset + "CHEMA".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            if ((c1 == 'C' || c1 == 'c') && (c2 == 'H' || c2 == 'h') && (c3 == 'E' || c3 == 'e')
                    && (c4 == 'M' || c4 == 'm') && (c5 == 'A' || c5 == 'a')) {
                while (stmt.length() > ++offset) {
                    switch (stmt.charAt(offset)) {
                    case ' ':
                        continue;
                    case '=':
                        while (stmt.length() > ++offset) {
                            switch (stmt.charAt(offset)) {
                            case ' ':
                                continue;
                            default:
                                return (offset << 8) | SLOW_SCHEMA;
                            }
                        }
                        return OTHER;
                    default:
                        return OTHER;
                    }
                }
            }
        }
        return OTHER;
    }

    // SHOW @@T
    static int show2TCheck(String stmt, int offset) {
        if (stmt.length() > ++offset) {
            switch (stmt.charAt(offset)) {
            case 'H':
            case 'h':
                return show2ThCheck(stmt, offset);
            case 'I':
            case 'i':
                return show2TiCheck(stmt, offset);
            default:
                return OTHER;
            }
        }
        return OTHER;
    }

    // SHOW @@VERSION
    static int show2VCheck(String stmt, int offset) {
        if (stmt.length() > offset + "ERSION".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);
            if ((c1 == 'E' || c1 == 'e') && (c2 == 'R' || c2 == 'r') && (c3 == 'S' || c3 == 's')
                    && (c4 == 'I' || c4 == 'i') && (c5 == 'O' || c5 == 'o') && (c6 == 'N' || c6 == 'n')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return VERSION;
            }
        }
        return OTHER;
    }
    // SHOW @@White  ip白名单
    static int show2WCheck(String stmt, int offset) {
        if (stmt.length() > offset + "HITE".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            if ((c1 == 'H' || c1 == 'h') && (c2 == 'I' || c2 == 'i') && (c3 == 'T' || c3 == 't')
                    && (c4 == 'E' || c4 == 'e') ) {
                if (stmt.length() > ++offset && stmt.charAt(offset) == '.') {
                    return show2WhiteCheck(stmt, offset);
                }
                return WHITE_HOST;
            }
        }
        return OTHER;
    }
    static int show2WhiteCheck(String stmt, int offset) {
        if (stmt.length() > offset + "set".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);

            if ((c1 == 'S' || c1 == 's') && (c2 == 'E' || c2 == 'e') && (c3 == 'T' || c3 == 't')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) == '=') {
                    return WHITE_HOST_SET;
                }
                return OTHER;
            }
        }
        return OTHER;
    }
    // SHOW @@CO
    static int show2CoCheck(String stmt, int offset) {
        if (stmt.length() > ++offset) {
            switch (stmt.charAt(offset)) {
            case 'M':
            case 'm':
                return show2ComCheck(stmt, offset);
            case 'N':
            case 'n':
                return show2ConCheck(stmt, offset);
            default:
                return OTHER;
            }
        }
        return OTHER;
    }

    // SHOW @@DATABASE
    static int show2DataBCheck(String stmt, int offset) {
        if (stmt.length() > offset + "ASE".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            if ((c1 == 'A' || c1 == 'a') && (c2 == 'S' || c2 == 's') && (c3 == 'E' || c3 == 'e')) {
//                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
//                    return OTHER;
//                }
                return DATABASE;
            }
        }
        return OTHER;
    }

    // SHOW @@DATANODE
    static int show2DataNCheck(String stmt, int offset) {
        if (stmt.length() > offset + "ODE".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            if ((c1 == 'O' || c1 == 'o') && (c2 == 'D' || c2 == 'd') && (c3 == 'E' || c3 == 'e')) {
                while (stmt.length() > ++offset) {
                    switch (stmt.charAt(offset)) {
                    case ' ':
                        continue;
                    case 'W':
                    case 'w':
                        return show2DataNWhereCheck(stmt, offset);
                    default:
                        return OTHER;
                    }
                }
                return DATANODE;
            }
        }
        return OTHER;
    }

    // SHOW @@DATANODE WHERE
    static int show2DataNWhereCheck(String stmt, int offset) {
        if (stmt.length() > offset + "HERE".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            if ((c1 == 'H' || c1 == 'h') && (c2 == 'E' || c2 == 'e') && (c3 == 'R' || c3 == 'r')
                    && (c4 == 'E' || c4 == 'e')) {
                while (stmt.length() > ++offset) {
                    switch (stmt.charAt(offset)) {
                    case ' ':
                        continue;
                    case 'S':
                    case 's':
                        return show2DataNWhereSchemaCheck(stmt, offset);
                    default:
                        return OTHER;
                    }
                }
            }
        }
        return OTHER;
    }

    // SHOW @@DATANODE WHERE SCHEMA = XXXXXX
    static int show2DataNWhereSchemaCheck(String stmt, int offset) {
        if (stmt.length() > offset + "CHEMA".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            if ((c1 == 'C' || c1 == 'c') && (c2 == 'H' || c2 == 'h') && (c3 == 'E' || c3 == 'e')
                    && (c4 == 'M' || c4 == 'm') && (c5 == 'A' || c5 == 'a')) {
                while (stmt.length() > ++offset) {
                    switch (stmt.charAt(offset)) {
                    case ' ':
                        continue;
                    case '=':
                        while (stmt.length() > ++offset) {
                            switch (stmt.charAt(offset)) {
                            case ' ':
                                continue;
                            default:
                                return (offset << 8) | DATANODE_WHERE;
                            }
                        }
                        return OTHER;
                    default:
                        return OTHER;
                    }
                }
            }
        }
        return OTHER;
    }

    // SHOW @@DATASOURCE
    static int show2DataSCheck(String stmt, int offset) {
        if (stmt.length() > offset + "OURCE".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            if ((c1 == 'O' || c1 == 'o') && (c2 == 'U' || c2 == 'u') && (c3 == 'R' || c3 == 'r')
                    && (c4 == 'C' || c4 == 'c') && (c5 == 'E' || c5 == 'e')) {
                while (stmt.length() > ++offset) {
                    switch (stmt.charAt(offset)) {
                    case ' ':
                        continue;
                    case 'W':
                    case 'w':
                        return show2DataSWhereCheck(stmt, offset);
                    case '.':
                        return show2DataSynCheck(stmt, offset);
                    default:
                        return OTHER;
                    }
                }

                return DATASOURCE;
            }
        }
        return OTHER;
    }

    // SHOW @@DATASOURCE WHERE
    static int show2DataSWhereCheck(String stmt, int offset) {
        if (stmt.length() > offset + "HERE".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            if ((c1 == 'H' || c1 == 'h') && (c2 == 'E' || c2 == 'e') && (c3 == 'R' || c3 == 'r')
                    && (c4 == 'E' || c4 == 'e')) {
                while (stmt.length() > ++offset) {
                    switch (stmt.charAt(offset)) {
                    case ' ':
                        continue;
                    case 'd':
                    case 'D':
                        return show2DataSWhereDatanodeCheck(stmt, offset);
                    default:
                        return OTHER;
                    }
                }
            }
        }
        return OTHER;
    }

    // SHOW @@DATASOURCE WHERE DATANODE = XXXXXX
    static int show2DataSWhereDatanodeCheck(String stmt, int offset) {
        if (stmt.length() > offset + "ATANODE".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);
            char c7 = stmt.charAt(++offset);
            if ((c1 == 'A' || c1 == 'a') && (c2 == 'T' || c2 == 't') && (c3 == 'A' || c3 == 'a')
                    && (c4 == 'N' || c4 == 'n') && (c5 == 'O' || c5 == 'o') && (c6 == 'D' || c6 == 'd')
                    && (c7 == 'E' || c7 == 'e')) {
                while (stmt.length() > ++offset) {
                    switch (stmt.charAt(offset)) {
                    case ' ':
                        continue;
                    case '=':
                        while (stmt.length() > ++offset) {
                            switch (stmt.charAt(offset)) {
                            case ' ':
                                continue;
                            default:
                                return (offset << 8) | DATASOURCE_WHERE;
                            }
                        }
                        return OTHER;
                    default:
                        return OTHER;
                    }
                }
            }
        }
        return OTHER;
    }

    // SHOW @@PARSER
    static int show2PaCheck(String stmt, int offset) {
        if (stmt.length() > offset + "RSER".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            if ((c1 == 'R' || c1 == 'r') && (c2 == 'S' || c2 == 's') && (c3 == 'E' || c3 == 'e')
                    && (c4 == 'R' || c4 == 'r')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return PARSER;
            }
        }
        return OTHER;
    }

    // SHOW @@PROCESSOR
    static int show2PrCheck(String stmt, int offset) {
        if (stmt.length() > offset + "OCESSOR".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);
            char c7 = stmt.charAt(++offset);
            if ((c1 == 'O' || c1 == 'o') && (c2 == 'C' || c2 == 'c') && (c3 == 'E' || c3 == 'e')
                    && (c4 == 'S' || c4 == 's') && (c5 == 'S' || c5 == 's') && (c6 == 'O' || c6 == 'o')
                    && (c7 == 'R' || c7 == 'r')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return PROCESSOR;
            }
        }
        return OTHER;
    }

    // SHOW @@SERVER
    // SHOW @@SESSION
    static int show2SeCheck(String stmt, int offset) {
        if (stmt.length() > offset + "SSION".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            if ((c1 == 'S' || c1 == 's') && (c2 == 'S' || c2 == 's') && (c3 == 'I' || c3 == 'i')
                    && (c4 == 'O' || c4 == 'o') && (c5 == 'N' || c5 == 'n')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return SESSION;
            }
        }
        else if (stmt.length() > offset + "RVER".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            if ((c1 == 'R' || c1 == 'r') && (c2 == 'V' || c2 == 'v') && (c3 == 'E' || c3 == 'e')
                    && (c4 == 'R' || c4 == 'r')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return SERVER;
            }
        }
        return OTHER;
    }

    // SHOW @@THREADPOOL
    static int show2ThCheck(String stmt, int offset) {
        if (stmt.length() > offset + "READPOOL".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);
            char c7 = stmt.charAt(++offset);
            char c8 = stmt.charAt(++offset);
            if ((c1 == 'R' || c1 == 'r') && (c2 == 'E' || c2 == 'e') && (c3 == 'A' || c3 == 'a')
                    && (c4 == 'D' || c4 == 'd') && (c5 == 'P' || c5 == 'p') && (c6 == 'O' || c6 == 'o')
                    && (c7 == 'O' || c7 == 'o') && (c8 == 'L' || c8 == 'l')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return THREADPOOL;
            }
        }
        return OTHER;
    }

    // SHOW @@TIME.
    static int show2TiCheck(String stmt, int offset) {
        if (stmt.length() > offset + "ME.".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            if ((c1 == 'M' || c1 == 'm') && (c2 == 'E' || c2 == 'e') && (c3 == '.')
                    && (stmt.length() > ++offset)) {
                    switch (stmt.charAt(offset)) {
                    case 'C':
                    case 'c':
                        return show2TimeCCheck(stmt, offset);
                    case 'S':
                    case 's':
                        return show2TimeSCheck(stmt, offset);
                    default:
                        return OTHER;
                    }
            }
        }
        return OTHER;
    }

    // SHOW @@COMMAND
    static int show2ComCheck(String stmt, int offset) {
        if (stmt.length() > offset + "MAND".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            if ((c1 == 'M' || c1 == 'm') && (c2 == 'A' || c2 == 'a') && (c3 == 'N' || c3 == 'n')
                    && (c4 == 'D' || c4 == 'd')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return COMMAND;
            }
        }
        return OTHER;
    }

    // SHOW @@CONNECTION
    static int show2ConCheck(String stmt, int offset) {
        if (stmt.length() > offset + "NECTION".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);
            char c7 = stmt.charAt(++offset);
            if ((c1 == 'N' || c1 == 'n') && (c2 == 'E' || c2 == 'e') && (c3 == 'C' || c3 == 'c')
                    && (c4 == 'T' || c4 == 't') && (c5 == 'I' || c5 == 'i') && (c6 == 'O' || c6 == 'o')
                    && (c7 == 'N' || c7 == 'n')) {
                if (stmt.length() > ++offset) {
                    switch (stmt.charAt(offset)) {
                    case ' ':
                        return CONNECTION;
                    case '.':
                        return show2ConnectonSQL(stmt, offset);
                    default:
                        return OTHER;
                    }
                }
                return CONNECTION;
            }
        }
        return OTHER;
    }

    // SHOW @@CONNECTION.SQL
    static int show2ConnectonSQL(String stmt, int offset) {
        if (stmt.length() > offset + "SQL".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            if ((c1 == 'S' || c1 == 's') && (c2 == 'Q' || c2 == 'q') && (c3 == 'L' || c3 == 'l')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return CONNECTION_SQL;
            }
        }
        return OTHER;
    }

    // SHOW @@TIME.CURRENT
    static int show2TimeCCheck(String stmt, int offset) {
        if (stmt.length() > offset + "URRENT".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);
            if ((c1 == 'U' || c1 == 'u') && (c2 == 'R' || c2 == 'r') && (c3 == 'R' || c3 == 'r')
                    && (c4 == 'E' || c4 == 'e') && (c5 == 'N' || c5 == 'n') && (c6 == 'T' || c6 == 't')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return TIME_CURRENT;
            }
        }
        return OTHER;
    }

    // SHOW @@TIME.STARTUP
    static int show2TimeSCheck(String stmt, int offset) {
        if (stmt.length() > offset + "TARTUP".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);
            if ((c1 == 'T' || c1 == 't') && (c2 == 'A' || c2 == 'a') && (c3 == 'R' || c3 == 'r')
                    && (c4 == 'T' || c4 == 't') && (c5 == 'U' || c5 == 'u') && (c6 == 'P' || c6 == 'p')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return TIME_STARTUP;
            }
        }
        return OTHER;
    }

    // SHOW @@SQ
    static int show2SqCheck(String stmt, int offset) {
        if (stmt.length() > ++offset) {
            switch (stmt.charAt(offset)) {
            case 'L':
            case 'l':
                return show2SqlCheck(stmt, offset);
            default:
                return OTHER;
            }
        }
        return OTHER;
    }

    // SHOW @@SQL
    static int show2SqlCheck(String stmt, int offset) {
        if (stmt.length() > ++offset) {
            switch (stmt.charAt(offset)) {
            case '.':
                return show2SqlDotCheck(stmt, offset);
            case ' ':
                return show2SqlBlankCheck(stmt, offset);
            default:            	          	
            	return SQL;
            }
        } else {
        	return SQL;
        }
    }

    // SHOW @@SQL.
    static int show2SqlDotCheck(String stmt, int offset) {
        if (stmt.length() > ++offset) {
            switch (stmt.charAt(offset)) {
            case 'D':
            case 'd':
                return show2SqlDCheck(stmt, offset);
            case 'E':
            case 'e':
                return show2SqlECheck(stmt, offset);
            case 'S':
            case 's':
            	char c1 = stmt.charAt(++offset);
            	switch (c1) {
            	case 'L':
            	case 'l':
                	return show2SqlSLCheck(stmt, offset);
            	case 'U':
            	case 'u':
            		return show2SqlSUCheck(stmt, offset);
            	}
            case 'H':
            case 'h':
            	return show2SqlHCheck(stmt, offset);
            case 'L':
            case 'l':
            	return show2SqlLCheck(stmt, offset);
            case 'C':
            case 'c':
            	return show2SqlCCheck(stmt, offset);
            case 'R':
            case 'r':
            	return show2SqlRCheck(stmt, offset);
            default:
                return OTHER;
            }
        }
        return OTHER;
    }

    // SHOW @@SQL WHERE ID = XXXXXX
    static int show2SqlBlankCheck(String stmt, int offset) {
        for (++offset; stmt.length() > offset;) {
            switch (stmt.charAt(offset)) {
            case ' ':
            	return SQL;
            case 'W':
            case 'w':
                if (isWhere(stmt, offset)) {
                    return SQL;
                } else {
                    return OTHER;
                }
            default:
            	return (offset << 8) | SQL;
            }
        }

        return OTHER;
    }

    // SHOW @@SQL.DETAIL WHERE ID = XXXXXX
    static int show2SqlDCheck(String stmt, int offset) {
        if (stmt.length() > offset + "ETAIL".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            if ((c1 == 'E' || c1 == 'e') && (c2 == 'T' || c2 == 't') && (c3 == 'A' || c3 == 'a')
                    && (c4 == 'I' || c4 == 'i') && (c5 == 'L' || c5 == 'l')) {
                for (++offset; stmt.length() > offset; ++offset) {
                    switch (stmt.charAt(offset)) {
                    case ' ':
                        continue;
                    case 'W':
                    case 'w':
                        if (isWhere(stmt, offset)) {
                            return SQL_DETAIL;
                        } else {
                            return OTHER;
                        }
                    default:
                        return OTHER;
                    }
                }
            }
        }
        return OTHER;
    }

    // SHOW @@SQL.EXECUTE
    static int show2SqlECheck(String stmt, int offset) {
        if (stmt.length() > offset + "XECUTE".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);
            if ((c1 == 'X' || c1 == 'x') && (c2 == 'E' || c2 == 'e') && (c3 == 'C' || c3 == 'c')
                    && (c4 == 'U' || c4 == 'u') && (c5 == 'T' || c5 == 't') && (c6 == 'E' || c6 == 'e')) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return SQL_EXECUTE;
            }
        }
        return OTHER;
    }

    // SHOW @@SQL.SLOW
    static int show2SqlSLCheck(String stmt, int offset) {
        if (stmt.length() > offset + "OW".length()) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            if ((c1 == 'O' || c1 == 'o') && (c2 == 'W' || c2 == 'w')) {
            	
            	while (stmt.length() > ++offset) {
	           		 switch (stmt.charAt(offset)) {
	           		 case ' ':
	                     continue;
	                 default:
	                   	 return (offset << 8) | SQL_SLOW;	 
	           		 }
	           	}
            	
                return SQL_SLOW;
            }
        }
        return OTHER;
    }
    
    // SHOW @@SQL.HIGH
    static int show2SqlHCheck(String stmt, int offset) {
    	
    	if (stmt.length() > offset + "IGH".length()) {
    		char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            if ((c1 == 'I' || c1 == 'i') && (c2 == 'G' || c2 == 'g') && (c3 == 'H' || c3 == 'h') ) {
            	
            	while (stmt.length() > ++offset) {
            		 switch (stmt.charAt(offset)) {
            		 case ' ':
                         continue;
                     default:
                    	 return (offset << 8) | SQL_HIGH;	 
            		 }
            	}
            	
                return SQL_HIGH;
            }
        }
        return OTHER;
    }
    
    // SHOW @@SQL.RESULTSET
    static int show2SqlRCheck(String stmt, int offset) {
    	
    	if (stmt.length() > offset + "ESULTSET".length()) {
    		char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);
            char c7 = stmt.charAt(++offset);
            char c8 = stmt.charAt(++offset);
            if ((c1 == 'E' || c1 == 'e') && (c2 == 'S' || c2 == 's') && (c3 == 'U' || c3 == 'u')&&
            		(c4 == 'l' || c4 == 'i') && (c5 == 'T' || c5 == 't') && (c6 == 'S' || c6 == 's')&&
            		(c7 == 'E' || c7 == 'e') && (c8 == 'T' || c8 == 't') ) {
            	
            	while (stmt.length() > ++offset) {
            		 switch (stmt.charAt(offset)) {
            		 case ' ':
                         continue;
                     default:
                    	 return (offset << 8) | SQL_RESULTSET;	 
            		 }
            	}
            	
                return SQL_RESULTSET;
            }
        }
        return OTHER;
    }
    
    // SHOW @@SQL.LARGE
    static int show2SqlLCheck(String stmt, int offset) {
    	
    	if (stmt.length() > offset + "ARGE".length()) {
    		char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            if ((c1 == 'A' || c1 == 'a') && (c2 == 'R' || c2 == 'r') && (c3 == 'G' || c3 == 'g') && (c4 == 'E' || c4 == 'e') ) {
            	
            	while (stmt.length() > ++offset) {
            		 switch (stmt.charAt(offset)) {
            		 case ' ':
                         continue;
                     default:
                    	 return (offset << 8) | SQL_LARGE;	 
            		 }
            	}
            	
                return SQL_LARGE;
            }
        }
        return OTHER;
    }
    
    // SHOW @@sql.condition
    static int show2SqlCCheck(String stmt, int offset) {
    	
    	if (stmt.length() > offset + "ONDITION".length()) {
    		char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            char c6 = stmt.charAt(++offset);
            char c7 = stmt.charAt(++offset);
            char c8 = stmt.charAt(++offset);
            if ( (c1 == 'O' || c1 == 'o') && (c2 == 'N' || c2 == 'n') && (c3 == 'D' || c3 == 'd') &&
            		(c4 == 'I' || c4 == 'i') && (c5 == 'T' || c5 == 't') && (c6 == 'I' || c6 == 'i') &&
            		(c7 == 'O' || c7 == 'o') && (c8 == 'N' || c8 == 'n') ) {
                if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                    return OTHER;
                }
                return SQL_CONDITION;
            }
        }
        return OTHER;
    }
    
    // SHOW @@SQL.SUM
    static int show2SqlSUCheck(String stmt, int offset) {
    	if (stmt.length() > offset + "M".length()) {
            char c1 = stmt.charAt(++offset);
            if ( c1 == 'M' || c1 == 'm') {
                if (stmt.length() > ++offset && stmt.charAt(offset) == '.') {
                	
                	/**
                	 *  TODO: modify by zhuam
                	 *   
                	 *  兼容之前指令
                	 *  在保留 SHOW @@SQL.SUM 指令的同时， 扩展支持  SHOW @@SQL.SUM.TABLE 、 SHOW @@SQL.SUM.USER   
                	 */       	
                	if ( stmt.length() > (offset+4) ) {                		
	                	char c2 = stmt.charAt(++offset);
	                	char c3 = stmt.charAt(++offset);
	                	char c4 = stmt.charAt(++offset);
	                	char c5 = stmt.charAt(++offset);	                	
	                	
	                	if ( (c2 == 'U' || c2 == 'u') && (c3 == 'S' || c3 == 's')
	                	  && (c4 == 'E' || c4 == 'e') && (c5 == 'R' || c5 == 'r') ) {	
	                		return SQL_SUM_USER;
	                		
	                	} else if ( (c2 == 'T' || c2 == 't') && (c3 == 'A' || c3 == 'a')
				             	 && (c4 == 'B' || c4 == 'b') && (c5 == 'L' || c5 == 'l')
                                && stmt.length() > (offset+1)) {

	                			 char c6 = stmt.charAt(++offset);
	                			 if ( c6 == 'E' || c6 == 'e') {
	                				 
                	            	while (stmt.length() > ++offset) {
                		           		 switch (stmt.charAt(offset)) {
                		           		 case ' ':
                		                     continue;
                		                 default:
                		                   	 return (offset << 8) | SQL_SUM_TABLE;	 
                		           		 }
                		           	}
	 
	                				return SQL_SUM_TABLE;
	                			 }
	                	}
	                	
                	} 
                	
                    return OTHER;
                }
                
            	while (stmt.length() > ++offset) {
              		 switch (stmt.charAt(offset)) {
              		 case ' ':
                           continue;
                       default:
                      	 return (offset << 8) | SQL_SUM_USER;	 
              		 }
                 	}
                return SQL_SUM_USER;
            }
        }
        return OTHER;
    }
    

    static boolean isWhere(String stmt, int offset) {
        if (stmt.length() > offset + 5) {
            char c1 = stmt.charAt(++offset);
            char c2 = stmt.charAt(++offset);
            char c3 = stmt.charAt(++offset);
            char c4 = stmt.charAt(++offset);
            char c5 = stmt.charAt(++offset);
            if ((c1 == 'H' || c1 == 'h') && (c2 == 'E' || c2 == 'e') && (c3 == 'R' || c3 == 'r')
                    && (c4 == 'E' || c4 == 'e') && (c5 == ' ')) {
                boolean jump1 = false;
                for (++offset; stmt.length() > offset && !jump1; ++offset) {
                    switch (stmt.charAt(offset)) {
                    case ' ':
                        continue;
                    case 'I':
                    case 'i':
                        jump1 = true;
                        break;
                    default:
                        return false;
                    }
                }
                if ((stmt.length() > offset) && (stmt.charAt(offset) == 'D' || stmt.charAt(offset) == 'd')) {
                    boolean jump2 = false;
                    for (++offset; stmt.length() > offset && !jump2; ++offset) {
                        switch (stmt.charAt(offset)) {
                        case ' ':
                            continue;
                        case '=':
                            jump2 = true;
                            break;
                        default:
                            return false;
                        }
                    }
                    return isSqlId(stmt, offset);
                }
            }
        }
        return false;
    }

    static boolean isSqlId(String stmt, int offset) {
        String id = stmt.substring(offset).trim();
        try {
            Long.parseLong(id);
        } catch (Exception e) {
            return false;
        }
        return true;
    }

    public static String getWhereParameter(String stmt) {
        int offset = stmt.indexOf('=');
        ++offset;
        return stmt.substring(offset).trim();
    }

}

================================================
FILE: src/main/java/io/mycat/route/parser/ManagerParseStop.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.parser;

import io.mycat.route.parser.util.Pair;
import io.mycat.route.parser.util.ParseUtil;
import io.mycat.util.SplitUtil;

/**
 * @author mycat
 */
public final class ManagerParseStop {

    public static final int OTHER = -1;
    public static final int HEARTBEAT = 1;

    public static int parse(String stmt, int offset) {
        int i = offset;
        for (; i < stmt.length(); i++) {
            switch (stmt.charAt(i)) {
            case ' ':
                continue;
            case '/':
            case '#':
                i = ParseUtil.comment(stmt, i);
                continue;
            case '@':
                return stop2Check(stmt, i);
            default:
                return OTHER;
            }
        }
        return OTHER;
    }

    public static Pair getPair(String stmt) {
        int offset = stmt.indexOf("@@");
        String s = stmt.substring(offset + 11).trim();
        int p1 = s.lastIndexOf(':');
        if (p1 == -1) {
            String[] src = SplitUtil.split(s, ',', '$', '-', '[', ']');
            return new Pair(src, null);
        } else {
            String[] src = SplitUtil.split(s, ':', true);
            String[] src1 = SplitUtil.split(src[0], ',', '$', '-', '[', ']');
            return new Pair(src1, Integer.valueOf(src[1]));
        }
    }

    // HEARTBEAT
    static int stop2Check(String stmt, int offset) {
        if (stmt.length() > ++offset && stmt.charAt(offset) == '@'
                && stmt.length() > offset + 9) {
                char c1 = stmt.charAt(++offset);
                char c2 = stmt.charAt(++offset);
                char c3 = stmt.charAt(++offset);
                char c4 = stmt.charAt(++offset);
                char c5 = stmt.charAt(++offset);
                char c6 = stmt.charAt(++offset);
                char c7 = stmt.charAt(++offset);
                char c8 = stmt.charAt(++offset);
                char c9 = stmt.charAt(++offset);
                if ((c1 == 'H' || c1 == 'h') && (c2 == 'E' || c2 == 'e') && (c3 == 'A' || c3 == 'a')
                        && (c4 == 'R' || c4 == 'r') && (c5 == 'T' || c5 == 't') && (c6 == 'B' || c6 == 'b')
                        && (c7 == 'E' || c7 == 'e') && (c8 == 'A' || c8 == 'a') && (c9 == 'T' || c9 == 't')) {
                    if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                        return OTHER;
                    }
                    return HEARTBEAT;
                }
        }
        return OTHER;
    }

}

================================================
FILE: src/main/java/io/mycat/route/parser/ManagerParseSwitch.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.parser;

import io.mycat.route.parser.util.Pair;
import io.mycat.route.parser.util.ParseUtil;
import io.mycat.util.SplitUtil;

/**
 * @author mycat
 */
public final class ManagerParseSwitch {

    public static final int OTHER = -1;
    public static final int DATASOURCE = 1;

    public static int parse(String stmt, int offset) {
        int i = offset;
        for (; i < stmt.length(); i++) {
            switch (stmt.charAt(i)) {
            case ' ':
                continue;
            case '/':
            case '#':
                i = ParseUtil.comment(stmt, i);
                continue;
            case '@':
                return switch2Check(stmt, i);
            default:
                return OTHER;
            }
        }
        return OTHER;
    }

    public static Pair getPair(String stmt) {
        int offset = stmt.indexOf("@@");
        String s = stmt.substring(offset + 12).trim();
        int p1 = s.lastIndexOf(':');
        if (p1 == -1) {
            String[] src = SplitUtil.split(s, ',', '$', '-', '[', ']');
            return new Pair(src, null);
        } else {
            String[] src = SplitUtil.split(s, ':', true);
            String[] src1 = SplitUtil.split(src[0], ',', '$', '-', '[', ']');
            return new Pair(src1, Integer.valueOf(src[1]));
        }
    }

    // DATASOURCE
    static int switch2Check(String stmt, int offset) {
        if (stmt.length() > ++offset && stmt.charAt(offset) == '@'
                && stmt.length() > offset + 10) {
                char c1 = stmt.charAt(++offset);
                char c2 = stmt.charAt(++offset);
                char c3 = stmt.charAt(++offset);
                char c4 = stmt.charAt(++offset);
                char c5 = stmt.charAt(++offset);
                char c6 = stmt.charAt(++offset);
                char c7 = stmt.charAt(++offset);
                char c8 = stmt.charAt(++offset);
                char c9 = stmt.charAt(++offset);
                char c10 = stmt.charAt(++offset);
                if ((c1 == 'D' || c1 == 'd') && (c2 == 'A' || c2 == 'a') && (c3 == 'T' || c3 == 't')
                        && (c4 == 'A' || c4 == 'a') && (c5 == 'S' || c5 == 's') && (c6 == 'O' || c6 == 'o')
                        && (c7 == 'U' || c7 == 'u') && (c8 == 'R' || c8 == 'r') && (c9 == 'C' || c9 == 'c')
                        && (c10 == 'E' || c10 == 'e')) {
                    if (stmt.length() > ++offset && stmt.charAt(offset) != ' ') {
                        return OTHER;
                    }
                    return DATASOURCE;
                }
        }
        return OTHER;
    }

}

================================================
FILE: src/main/java/io/mycat/route/parser/druid/DruidParser.java
================================================
package io.mycat.route.parser.druid;

import java.sql.SQLNonTransientException;

import com.alibaba.druid.sql.ast.SQLStatement;

import io.mycat.cache.LayerCachePool;
import io.mycat.config.model.SchemaConfig;
import io.mycat.route.RouteResultset;

/**
 * 对SQLStatement解析
 * 主要通过visitor解析和statement解析：有些类型的SQLStatement通过visitor解析足够了，
 *  有些只能通过statement解析才能得到所有信息
 *  有些需要通过两种方式解析才能得到完整信息
 * @author wang.dw
 *
 */
public interface DruidParser {
	/**
	 * 使用MycatSchemaStatVisitor解析,得到tables、tableAliasMap、conditions等
	 * @param schema
	 * @param stmt
	 */
	public void parser(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt, String originSql,LayerCachePool cachePool,MycatSchemaStatVisitor schemaStatVisitor) throws SQLNonTransientException;
	
	/**
	 * statement方式解析
	 * 子类可覆盖（如果visitorParse解析得不到表名、字段等信息的，就通过覆盖该方法来解析）
	 * 子类覆盖该方法一般是将SQLStatement转型后再解析（如转型为MySqlInsertStatement）
	 */
	public void statementParse(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt) throws SQLNonTransientException;

	/**
	 * 子类可覆盖（如果该方法解析得不到表名、字段等信息的，就覆盖该方法，覆盖成空方法，然后通过statementPparse去解析）
	 * 通过visitor解析：有些类型的Statement通过visitor解析得不到表名、
	 * @param stmt
	 */
	public void visitorParse(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt, MycatSchemaStatVisitor visitor)
			throws SQLNonTransientException;
	
	/**
	 * 改写sql：加limit，加group by、加order by如有些没有加limit的可以通过该方法增加
	 * @param schema
	 * @param rrs
	 * @param stmt
	 * @throws SQLNonTransientException
	 */
	public void changeSql(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt,LayerCachePool cachePool) throws SQLNonTransientException;
	/**
	 * 获取解析到的信息
	 * @return
	 */
	public DruidShardingParseInfo getCtx();
	
}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/DruidParserFactory.java
================================================
package io.mycat.route.parser.druid;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.statement.SQLAlterTableStatement;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlCreateTableStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlDeleteStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlInsertStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlLockTableStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlUpdateStatement;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;

import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.TableConfig;
import io.mycat.route.parser.druid.impl.DefaultDruidParser;
import io.mycat.route.parser.druid.impl.DruidAlterTableParser;
import io.mycat.route.parser.druid.impl.DruidCreateTableParser;
import io.mycat.route.parser.druid.impl.DruidDeleteParser;
import io.mycat.route.parser.druid.impl.DruidInsertParser;
import io.mycat.route.parser.druid.impl.DruidLockTableParser;
import io.mycat.route.parser.druid.impl.DruidSelectDb2Parser;
import io.mycat.route.parser.druid.impl.DruidSelectOracleParser;
import io.mycat.route.parser.druid.impl.DruidSelectParser;
import io.mycat.route.parser.druid.impl.DruidSelectPostgresqlParser;
import io.mycat.route.parser.druid.impl.DruidSelectSqlServerParser;
import io.mycat.route.parser.druid.impl.DruidUpdateParser;

/**
 * DruidParser的工厂类
 *
 * @author wdw
 */
public class DruidParserFactory
{

    public static DruidParser create(SchemaConfig schema, SQLStatement statement, SchemaStatVisitor visitor)
    {
        DruidParser parser = null;
        if (statement instanceof SQLSelectStatement)
        {
            if(schema.isNeedSupportMultiDBType())
            {
                parser = getDruidParserForMultiDB(schema, statement, visitor);

            }

            if (parser == null)
            {
                parser = new DruidSelectParser();
            }
        } else if (statement instanceof MySqlInsertStatement)
        {
            parser = new DruidInsertParser();
        } else if (statement instanceof MySqlDeleteStatement)
        {
            parser = new DruidDeleteParser();
        } else if (statement instanceof MySqlCreateTableStatement)
        {
            parser = new DruidCreateTableParser();
        } else if (statement instanceof MySqlUpdateStatement)
        {
            parser = new DruidUpdateParser();
        } else if (statement instanceof SQLAlterTableStatement)
        {
            parser = new DruidAlterTableParser();
        } else if (statement instanceof MySqlLockTableStatement) {
        	parser = new DruidLockTableParser();
        } else
        {
            parser = new DefaultDruidParser();
        }

        return parser;
    }

    private static DruidParser getDruidParserForMultiDB(SchemaConfig schema, SQLStatement statement, SchemaStatVisitor visitor)
    {
        DruidParser parser=null;
        //先解出表，判断表所在db的类型，再根据不同db类型返回不同的解析
        /**
         * 不能直接使用visitor变量，防止污染后续sql解析
         * @author SvenAugustus
         */
		MycatSchemaStatVisitor _visitor = SchemaStatVisitorFactory.create(schema);
        List tables = parseTables(statement, _visitor);
        for (String table : tables)
        {
            Set dbTypes =null;
            TableConfig tableConfig = schema.getTables().get(table);
            if(tableConfig==null)
            {
                dbTypes=new HashSet<>();
                dbTypes.add(schema.getDefaultDataNodeDbType())  ;
            }else
            {
                dbTypes = tableConfig.getDbTypes();
            }
            if (dbTypes.contains("oracle"))
            {
                parser = new DruidSelectOracleParser();
                ((DruidSelectOracleParser)parser).setInvocationHandler(SqlMethodInvocationHandlerFactory.getForOracle());
                break;
            } else if (dbTypes.contains("db2"))
            {
                parser = new DruidSelectDb2Parser();
                break;
            } else if (dbTypes.contains("sqlserver"))
            {
                parser = new DruidSelectSqlServerParser();
                break;
            } else if (dbTypes.contains("postgresql"))
            {
                parser = new DruidSelectPostgresqlParser();
                ((DruidSelectPostgresqlParser)parser).setInvocationHandler(SqlMethodInvocationHandlerFactory.getForPgsql());
                break;
            }
        }
        return parser;
    }


	private static List parseTables(SQLStatement stmt, MycatSchemaStatVisitor schemaStatVisitor)
    {
        List tables = new ArrayList<>();
        stmt.accept(schemaStatVisitor);

        if (schemaStatVisitor.getAliasMap() != null)
        {
            for (Map.Entry entry : schemaStatVisitor.getAliasMap().entrySet())
            {
                String key = entry.getKey();
                String value = entry.getValue();
                if (value != null && value.indexOf("`") >= 0)
                {
                    value = value.replaceAll("`", "");
                }
                //表名前面带database的，去掉
                if (key != null)
                {
                    int pos = key.indexOf("`");
                    if (pos > 0)
                    {
                        key = key.replaceAll("`", "");
                    }
                    pos = key.indexOf(".");
                    if (pos > 0)
                    {
                        key = key.substring(pos + 1);
                    }

					// if (key.equalsIgnoreCase(value))
					// {
					tables.add(value.toUpperCase());
					// }
                }
            }

        }
        return tables;
    }


}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/DruidSequenceHandler.java
================================================
package io.mycat.route.parser.druid;

import java.io.UnsupportedEncodingException;
import java.lang.reflect.Constructor;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import io.mycat.MycatServer;
import io.mycat.config.MycatConfig;
import io.mycat.config.model.SystemConfig;
import io.mycat.route.SessionSQLPair;
import io.mycat.route.sequence.handler.*;
import io.mycat.util.TimeUtil;

/**
 * 使用Druid解析器实现对Sequence处理
 *
 * @author 兵临城下
 * @date 2015/03/13
 */
public class DruidSequenceHandler {
    private final SequenceHandler sequenceHandler;

    /**
     * 分段锁
     */
    private final static Map segmentLock = new ConcurrentHashMap<>();

    /**
     * 获取MYCAT SEQ的匹配语句
     */
    private final static String MATCHED_FEATURE = "NEXT VALUE FOR MYCATSEQ_";

    private final  Pattern pattern;

    public DruidSequenceHandler(int seqHandlerType,String sequnceHandlerPattern) {
      this.pattern =  Pattern.compile(sequnceHandlerPattern, Pattern.CASE_INSENSITIVE);
        switch (seqHandlerType) {
            case SystemConfig.SEQUENCEHANDLER_MYSQLDB:
                sequenceHandler = IncrSequenceMySQLHandler.getInstance();
                break;
            case SystemConfig.SEQUENCEHANDLER_LOCALFILE:
                sequenceHandler = IncrSequencePropHandler.getInstance();
                break;
            case SystemConfig.SEQUENCEHANDLER_LOCAL_TIME:
                sequenceHandler = IncrSequenceTimeHandler.getInstance();
                break;
            case SystemConfig.SEQUENCEHANDLER_ZK_DISTRIBUTED:
                sequenceHandler = DistributedSequenceHandler.getInstance(MycatServer.getInstance().getConfig().getSystem());
                break;
            case SystemConfig.SEQUENCEHANDLER_ZK_GLOBAL_INCREMENT:
                sequenceHandler = IncrSequenceZKHandler.getInstance();
                break;
            case SystemConfig.SEQUENCEHANDLER_DEF_GLOBAL_INCREMENT:
                try {
                    String sequenceHanlderClass = MycatServer.getInstance().getConfig().getSystem().getSequenceHanlderClass();
                    Class aClass = Class.forName(sequenceHanlderClass);
                    Constructor constructor=aClass.getConstructor();
                    sequenceHandler  =(SequenceHandler) constructor.newInstance();
                }catch (Exception e){
                    e.printStackTrace();
                    throw new RuntimeException(e);
                }
                break;
            default:
                throw new java.lang.IllegalArgumentException("Invalid sequnce handler type " + seqHandlerType);
        }
    }

    /**
     * 根据原sql获取可执行的sql
     *
     * @param sql
     * @return
     * @throws UnsupportedEncodingException
     */
    public String getExecuteSql(SessionSQLPair pair, String charset) throws UnsupportedEncodingException,InterruptedException {
    	String executeSql = pair.sql;
        if (null != pair.sql && !"".equals(pair.sql)) {
            Matcher matcher = pattern.matcher(executeSql);
            if(matcher.find()){
            	String tableName = matcher.group(2);
                ReentrantLock lock = getSegLock(tableName);
				lock.lock();
				try {
                	matcher = pattern.matcher(executeSql);
                	while(matcher.find()){
                		long value = sequenceHandler.nextId(tableName.toUpperCase());
                        executeSql = executeSql.replaceFirst(matcher.group(1), Long.toString(value));
                        pair.session.getSource().setLastWriteTime(TimeUtil.currentTimeMillis());
                    }
				} finally {
					lock.unlock();
				}
            }
        }
        return executeSql;
    }

    /*
     * 获取分段锁
     * @param name
     * @return
     */
    private ReentrantLock getSegLock(String name){
    	ReentrantLock lock = segmentLock.get(name);
    	if(lock==null){
    		synchronized (segmentLock) {
    			lock = segmentLock.get(name);
				if(lock==null){
					lock = new ReentrantLock();
					segmentLock.put(name, lock);
				}
			}
    	}
    	return lock;
    }


    //just for test
    public String getTableName(String sql) {
        Matcher matcher = pattern.matcher(sql);
        if (matcher.find()) {
            return matcher.group(2);
        }
        return null;
    }


}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/DruidShardingParseInfo.java
================================================
package io.mycat.route.parser.druid;

import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.stat.TableStat;
import com.alibaba.druid.stat.TableStat.Name;

import io.mycat.route.util.RouterUtil;

/**
 * druid parser result
 * @author wang.dw
 *
 */
public class DruidShardingParseInfo {
	/**
	 * 一个sql中可能有多个WhereUnit（如子查询中的where可能导致多个）
	 */
	private List whereUnits = new ArrayList();
	
	private List routeCalculateUnits = new ArrayList();
	
	/**
	 * （共享属性）
	 */
	private String sql = "";
	
	//tables为路由计算共享属性，多组RouteCalculateUnit使用同样的tables
	private List tables = new ArrayList();
	
//	private RouteCalculateUnit routeCalculateUnit = new RouteCalculateUnit(this); 

	/**
	 * key table alias, value talbe realname;
	 */
	private Map tableAliasMap = new LinkedHashMap();

	private SchemaStatVisitor visitor;

	public Map getTableAliasMap() {
		return tableAliasMap;
	}

	public void setTableAliasMap(Map tableAliasMap) {
		this.tableAliasMap = tableAliasMap;
	}

	public String getSql() {
		return sql;
	}

	public void setSql(String sql) {
		this.sql = sql;
	}

	public List getTables() {
		return tables;
	}

	public void addTable(String tableName) {
		this.tables.add(tableName);
	}

	public RouteCalculateUnit getRouteCalculateUnit() {
		return routeCalculateUnits.get(0);
	}
	
	public List getRouteCalculateUnits() {
		return routeCalculateUnits;
	}
	
	public void setRouteCalculateUnits(List routeCalculateUnits) {
		this.routeCalculateUnits = routeCalculateUnits;
	}
	
	public void addRouteCalculateUnit(RouteCalculateUnit routeCalculateUnit) {
		this.routeCalculateUnits.add(routeCalculateUnit);
	}
	

	public void clear() {
		for(RouteCalculateUnit unit : routeCalculateUnits ) {
			unit.clear();
		}
	}
	
	public void setVisitor(SchemaStatVisitor visitor) {
		
		this.visitor = visitor;
	}
	
	public SchemaStatVisitor getVisitor(){
		
		return this.visitor;
	}

	public void addTables(Map map, String sessionSchema) {
		
		int dotIndex;
		for(Name _name : map.keySet()){
			
			String _tableName = _name.getName().toString().toUpperCase();
			//系统表直接跳过，路由到默认datanode
			if(RouterUtil.isSystemSchema(_tableName)){
				continue;
			}
			if ((dotIndex = _tableName.indexOf('.')) != -1) {
				String schemaInSQL = _tableName.substring(0, dotIndex);
				if (schemaInSQL.equalsIgnoreCase(sessionSchema)) {
					_tableName = _tableName.substring(dotIndex + 1);
				}
			}
			addTable(_tableName);
		}
	}

}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/LoadDataOutputVisitor.java
================================================
package io.mycat.route.parser.druid;

import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlLoadDataInFileStatement;
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlOutputVisitor;

/**
 * Created by nange on 2015/10/20.
 */
public class LoadDataOutputVisitor extends MySqlOutputVisitor
{
    public LoadDataOutputVisitor(Appendable appender)
    {
        super(appender);
    }
    @Override
    public boolean visit(MySqlLoadDataInFileStatement x) {
        print("LOAD DATA ");

        if (x.isLowPriority()) {
            print("LOW_PRIORITY ");
        }

        if (x.isConcurrent()) {
            print("CONCURRENT ");
        }

        if (x.isLocal()) {
            print("LOCAL ");
        }

        print("INFILE ");

        x.getFileName().accept(this);

        if (x.isReplicate()) {
            print(" REPLACE ");
        }

        if (x.isIgnore()) {
            print(" IGNORE ");
        }

        print(" INTO TABLE ");
        x.getTableName().accept(this);
        if(x.getCharset()!=null)
        {
            print(" CHARACTER SET ");
            print("'"+x.getCharset()+"'");
        }

        if (x.getColumnsTerminatedBy() != null || x.getColumnsEnclosedBy() != null || x.getColumnsEscaped() != null) {
            print(" COLUMNS");
            if (x.getColumnsTerminatedBy() != null) {
                print(" TERMINATED BY ");
                x.getColumnsTerminatedBy().accept(this);
            }

            if (x.getColumnsEnclosedBy() != null) {
                if (x.isColumnsEnclosedOptionally()) {
                    print(" OPTIONALLY");
                }
                print(" ENCLOSED BY ");
                x.getColumnsEnclosedBy().accept(this);
            }

            if (x.getColumnsEscaped() != null) {
                print(" ESCAPED BY ");
                x.getColumnsEscaped().accept(this);
            }
        }

        if (x.getLinesStartingBy() != null || x.getLinesTerminatedBy() != null) {
            print(" LINES");
            if (x.getLinesStartingBy() != null) {
                print(" STARTING BY ");
                x.getLinesStartingBy().accept(this);
            }

            if (x.getLinesTerminatedBy() != null) {
                print(" TERMINATED BY ");
                x.getLinesTerminatedBy().accept(this);
            }
        }

        if(x.getIgnoreLinesNumber() != null) {
            print(" IGNORE ");
            x.getIgnoreLinesNumber().accept(this);
            print(" LINES");
        }

        if (x.getColumns().size() != 0) {
            print(" (");
            printAndAccept(x.getColumns(), ", ");
            print(")");
        }

        if (x.getSetList().size() != 0) {
            print(" SET ");
            printAndAccept(x.getSetList(), ", ");
        }

        return false;
    }
}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/LoadDataStatement.java
================================================
package io.mycat.route.parser.druid;

import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlLoadDataInFileStatement;


public class LoadDataStatement extends MySqlLoadDataInFileStatement
{

    public String toString()
    {
        StringBuilder out = new StringBuilder();
        this.accept(new LoadDataOutputVisitor(out));

        return out.toString();
    }
}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/MycatExprParser.java
================================================
package io.mycat.route.parser.druid;

import com.alibaba.druid.sql.ast.statement.SQLSelectItem;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlExprParser;
import com.alibaba.druid.sql.parser.Lexer;
import com.alibaba.druid.sql.parser.Token;

/**
 * Created by nange on 2015/3/13.
 */
public class MycatExprParser extends MySqlExprParser
{
    public static final String[] max_agg_functions = {"AVG", "COUNT", "GROUP_CONCAT", "MAX", "MIN", "STDDEV", "SUM", "ROW_NUMBER"};

    public MycatExprParser(Lexer lexer)
    {
        super(lexer);
        super.aggregateFunctions = max_agg_functions;
    }

    public MycatExprParser(String sql)
    {
        super(new MycatLexer(sql));
        lexer.nextToken();
    }
    @Override
    public SQLSelectItem parseSelectItem()
    {
        parseTop();
        return super.parseSelectItem();
    }
    public void parseTop()
    {
        if (lexer.token() == Token.TOP)
        {
            lexer.nextToken();

            boolean paren = false;
            if (lexer.token() == Token.LPAREN)
            {
                paren = true;
                lexer.nextToken();
            }

            if (paren)
            {
                accept(Token.RPAREN);
            }

            if (lexer.token() == Token.LITERAL_INT)
            {
                lexer.mark();
                lexer.nextToken();
            }
            if (lexer.token() == Token.IDENTIFIER)
            {
                lexer.nextToken();

            }
            if (lexer.token() == Token.EQ||lexer.token() == Token.DOT)
            {
                lexer.nextToken();
            } else  if(lexer.token() != Token.STAR)
            {
                lexer.reset();
            }
            if (lexer.token() == Token.PERCENT)
            {
                lexer.nextToken();
            }


        }


    }
}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/MycatLexer.java
================================================
package io.mycat.route.parser.druid;

import java.util.HashMap;
import java.util.Map;

import com.alibaba.druid.sql.dialect.mysql.parser.MySqlLexer;
import com.alibaba.druid.sql.parser.Keywords;
import com.alibaba.druid.sql.parser.Token;

/**
 * Created by magicdoom on 2015/3/13.
 */
public class MycatLexer extends MySqlLexer
{
    public final static Keywords DEFAULT_MYCAT_KEYWORDS;
    static {
        Map map = new HashMap();

        map.putAll(Keywords.DEFAULT_KEYWORDS.getKeywords());

        map.put("DUAL", Token.DUAL);
        map.put("FALSE", Token.FALSE);
        map.put("IDENTIFIED", Token.IDENTIFIED);
        map.put("IF", Token.IF);
        map.put("KILL", Token.KILL);

        map.put("LIMIT", Token.LIMIT);
        map.put("TRUE", Token.TRUE);
        map.put("BINARY", Token.BINARY);
        map.put("SHOW", Token.SHOW);
        map.put("CACHE", Token.CACHE);
        map.put("ANALYZE", Token.ANALYZE);
        map.put("OPTIMIZE", Token.OPTIMIZE);
        map.put("ROW", Token.ROW);
        map.put("BEGIN", Token.BEGIN);
        map.put("END", Token.END);

        map.put("TOP", Token.TOP);


        DEFAULT_MYCAT_KEYWORDS = new Keywords(map);
    }


    public MycatLexer(char[] input, int inputLength, boolean skipComment)
    {
        super(input, inputLength, skipComment);
		super.keywords = DEFAULT_MYCAT_KEYWORDS;
    }

    public MycatLexer(String input)
    {
        super(input);
		super.keywords = DEFAULT_MYCAT_KEYWORDS;
    }
}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/MycatSchemaStatVisitor.java
================================================
package io.mycat.route.parser.druid;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLCommentHint;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLExprImpl;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.SQLObject;
import com.alibaba.druid.sql.ast.SQLObjectImpl;
import com.alibaba.druid.sql.ast.expr.SQLAggregateExpr;
import com.alibaba.druid.sql.ast.expr.SQLAllExpr;
import com.alibaba.druid.sql.ast.expr.SQLAnyExpr;
import com.alibaba.druid.sql.ast.expr.SQLBetweenExpr;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator;
import com.alibaba.druid.sql.ast.expr.SQLCharExpr;
import com.alibaba.druid.sql.ast.expr.SQLExistsExpr;
import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
import com.alibaba.druid.sql.ast.expr.SQLInListExpr;
import com.alibaba.druid.sql.ast.expr.SQLInSubQueryExpr;
import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr;
import com.alibaba.druid.sql.ast.expr.SQLQueryExpr;
import com.alibaba.druid.sql.ast.expr.SQLSomeExpr;
import com.alibaba.druid.sql.ast.statement.SQLAlterTableItem;
import com.alibaba.druid.sql.ast.statement.SQLAlterTableStatement;
import com.alibaba.druid.sql.ast.statement.SQLDeleteStatement;
import com.alibaba.druid.sql.ast.statement.SQLExprTableSource;
import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.ast.statement.SQLSelectItem;
import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource;
import com.alibaba.druid.sql.ast.statement.SQLUpdateStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlCreateTableStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlDeleteStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlHintStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlInsertStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlSchemaStatVisitor;
import com.alibaba.druid.stat.TableStat;
import com.alibaba.druid.stat.TableStat.Column;
import com.alibaba.druid.stat.TableStat.Condition;
import com.alibaba.druid.stat.TableStat.Mode;

import io.mycat.route.util.RouterUtil;
import io.mycat.util.StringUtil;

/**
 * Druid解析器中用来从ast语法中提取表名、条件、字段等的vistor
 * @author wang.dw
 *
 */
public class MycatSchemaStatVisitor extends MySqlSchemaStatVisitor {
	private boolean hasOrCondition = false;
	private List whereUnits = new ArrayList();
	private List storedwhereUnits = new ArrayList<>();
    private List orBinaryExprs = new ArrayList();
    private Queue subQuerys = new LinkedList<>();  //子查询集合
	private boolean hasChange = false; // 是否有改写sql
	private boolean subqueryRelationOr = false;   //子查询存在关联条件的情况下，是否有 or 条件
	private Map aliasMap = new LinkedHashMap<>();
	private List selectTableList = new ArrayList<>();
	private String currentTable;
	// 标识一个元素是否已经被VISIT，防止多次重复访问导致添加aliasMap时报表冲突异常
	private final String TABLE_HAS_VISIT_FLAG = "TABLE_HAS_VISIT_FLAG";
	// 记录哪些表已经被visit
	private List visitedTableSourceExpr = new ArrayList();

	private void reset() {
		this.conditions.clear();
		this.whereUnits.clear();
        this.orBinaryExprs.clear();
		this.hasOrCondition = false;
	}
	
	public List getWhereUnits() {
		return whereUnits;
	}

	public boolean hasOrCondition() {
		return hasOrCondition;
	}
	
    @Override
    public boolean visit(SQLSelectStatement x) {
		// setAliasMap();
//        getAliasMap().put("DUAL", null);
		aliasMap.clear();
		selectTableList.clear();
		clearTableVisitFlag();
        return true;
    }

    @Override
    public boolean visit(SQLBetweenExpr x) {
        String begin = null;
        if(x.beginExpr instanceof SQLCharExpr)
        {
            begin= (String) ( (SQLCharExpr)x.beginExpr).getValue();
        }  else {
            begin = x.beginExpr.toString();
        }
        String end = null;
        if(x.endExpr instanceof SQLCharExpr)
        {
            end= (String) ( (SQLCharExpr)x.endExpr).getValue();
        }  else {
            end = x.endExpr.toString();
        }
        Column column = getColumn(x);
        if (column == null) {
            return true;
        }

        Condition condition = null;
        for (Condition item : this.getConditions()) {
            if (item.getColumn().equals(column) && item.getOperator().equals("between")) {
                condition = item;
                break;
            }
        }

        if (condition == null) {
			condition = new Condition(column, "between");
            this.conditions.add(condition);
        }


        condition.getValues().add(begin);
        condition.getValues().add(end);


        return true;
    }

    @Override
    protected Column getColumn(SQLExpr expr) {
		Map aliasMap = getAliasMap();
		if (aliasMap == null) {
			return null;
		}

		if (expr instanceof SQLBetweenExpr) {
			return getColumnByExpr((SQLBetweenExpr) expr);
		} else if (expr instanceof SQLIdentifierExpr) {
			return getColumnByExpr((SQLIdentifierExpr) expr);
		} else {// SQLPropertyExpr
			return super.getColumn(expr);
		}
		// return null;
    }

	private Column getColumnByExpr(SQLIdentifierExpr expr) {
//        Column attrColumn = (Column) expr.getAttribute(ATTR_COLUMN);
//        if (attrColumn != null) {
//            return attrColumn;
//        }

		String column = expr.getName();
		String table = getCurrentTable();
		if (table != null && aliasMap.containsKey(table)) {
			table = aliasMap.get(table);
			if (table == null) {
				return null;
			}
		}

		if (table != null) {
			return new Column(table, column);
		}

//        if (variants.containsKey(column)) {
//            return null;
//        }

		return new Column("UNKNOWN", column);
	}
	private Column getColumnByExpr(SQLBetweenExpr betweenExpr) {
		if (betweenExpr.getTestExpr() != null) {
			String tableName = null;
			String column = null;
			if (betweenExpr.getTestExpr() instanceof SQLPropertyExpr) { // field
																		// has
																		// alias
				tableName = ((SQLIdentifierExpr) ((SQLPropertyExpr) betweenExpr.getTestExpr()).getOwner()).getName();
				column = ((SQLPropertyExpr) betweenExpr.getTestExpr()).getName();
				tableName = getTableNameByAlias(tableName);

				return new Column(tableName, column);
			} else if (betweenExpr.getTestExpr() instanceof SQLIdentifierExpr) {
				column = ((SQLIdentifierExpr) betweenExpr.getTestExpr()).getName();
				tableName = getOwnerTableName(betweenExpr, column);
			}
			String table = tableName;
			table = getTableNameByAlias(table);

			if (table != null && !"".equals(table)) {
				return new Column(table, column);
			}
		}
		return null;
	}

	/**
	 * 通过别名获取到表的名称.如果获取不到返回alias
	 * 
	 * @param alias
	 * @return
	 */
	private String getTableNameByAlias(String alias) {
		if (alias != null) {
			String upperCaseAlias = alias.toUpperCase();
			if (aliasMap.containsKey(upperCaseAlias)) {
				return aliasMap.get(upperCaseAlias);
			}
		}
		return alias;
	}
    /**
     * 从between语句中获取字段所属的表名。
     * 对于容易出现ambiguous的（字段不知道到底属于哪个表），实际应用中必须使用别名来避免歧义
     * @param betweenExpr
     * @param column
     * @return
     */
    private String getOwnerTableName(SQLBetweenExpr betweenExpr,String column) {
        if(tableStats.size() == 1) {//只有一个表，直接返回这一个表名
            return tableStats.keySet().iterator().next().getName();
        } else if(tableStats.size() == 0) {//一个表都没有，返回空串
            return "";
        } else {//多个表名
			for (Column col : columns.values())
            {
                if(col.getName().equals(column)) {
                    return col.getTable();
                }
            }
//            for(Column col : columns) {//从columns中找表名
//                if(col.getName().equals(column)) {
//                    return col.getTable();
//                }
//            }

            //前面没找到表名的，自己从parent中解析

            SQLObject parent = betweenExpr.getParent();
            if(parent instanceof SQLBinaryOpExpr)
            {
                parent=parent.getParent();
            }

            if(parent instanceof MySqlSelectQueryBlock) {
                MySqlSelectQueryBlock select = (MySqlSelectQueryBlock) parent;
                if(select.getFrom() instanceof SQLJoinTableSource) {//多表连接
                    SQLJoinTableSource joinTableSource = (SQLJoinTableSource)select.getFrom();
                    return joinTableSource.getLeft().toString();//将left作为主表，此处有不严谨处，但也是实在没有办法，如果要准确，字段前带表名或者表的别名即可
                } else if(select.getFrom() instanceof SQLExprTableSource) {//单表
                    return select.getFrom().toString();
                }
            }
            else if(parent instanceof SQLUpdateStatement) {
                SQLUpdateStatement update = (SQLUpdateStatement) parent;
                return update.getTableName().getSimpleName();
            } else if(parent instanceof SQLDeleteStatement) {
                SQLDeleteStatement delete = (SQLDeleteStatement) parent;
                return delete.getTableName().getSimpleName();
            } else {
                
            }
        }
        return "";
    }
    
    private void setSubQueryRelationOrFlag(SQLExprImpl x){
    	MycatSubQueryVisitor subQueryVisitor = new MycatSubQueryVisitor();
    	x.accept(subQueryVisitor);
    	if(subQueryVisitor.isRelationOr()){
    		subqueryRelationOr = true;
    	}
    }
    
    /*
     * 子查询
     * (non-Javadoc)
     * @see com.alibaba.druid.sql.visitor.SQLASTVisitorAdapter#visit(com.alibaba.druid.sql.ast.expr.SQLQueryExpr)
     */
    @Override
    public boolean visit(SQLQueryExpr x) {
    	setSubQueryRelationOrFlag(x);
    	addSubQuerys(x.getSubQuery());
    	return super.visit(x);
    }
    /*
     * (non-Javadoc)
     * @see com.alibaba.druid.sql.visitor.SchemaStatVisitor#visit(com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource)
     */
    @Override
    public boolean visit(SQLSubqueryTableSource x){
    	addSubQuerys(x.getSelect());
    	return super.visit(x);
    }
    
    /*
     * (non-Javadoc)
     * @see com.alibaba.druid.sql.visitor.SQLASTVisitorAdapter#visit(com.alibaba.druid.sql.ast.expr.SQLExistsExpr)
     */
    @Override
    public boolean visit(SQLExistsExpr x) {
    	setSubQueryRelationOrFlag(x);
    	addSubQuerys(x.getSubQuery());
    	return super.visit(x);
    }
    
    @Override
    public boolean visit(SQLInListExpr x) {
    	return super.visit(x);
    }
    
    /*
     *  对 in 子查询的处理
     * (non-Javadoc)
     * @see com.alibaba.druid.sql.visitor.SchemaStatVisitor#visit(com.alibaba.druid.sql.ast.expr.SQLInSubQueryExpr)
     */
    @Override
    public boolean visit(SQLInSubQueryExpr x) {
    	setSubQueryRelationOrFlag(x);
    	addSubQuerys(x.getSubQuery());
    	return super.visit(x);
    }
    
    /* 
     *  遇到 all 将子查询改写成  SELECT MAX(name) FROM subtest1
     *  例如:
     *        select * from subtest where id > all (select name from subtest1);
     *    		>/>= all ----> >/>= max
     *    		    all ----> not in
     *          =    all ----> id = 1 and id = 2
     *          other  不改写
     */    
    @Override
    public boolean visit(SQLAllExpr x) {
    	setSubQueryRelationOrFlag(x);
    	
    	List itemlist = ((SQLSelectQueryBlock)(x.getSubQuery().getQuery())).getSelectList();
    	SQLExpr sexpr = itemlist.get(0).getExpr();
    	
		if(x.getParent() instanceof SQLBinaryOpExpr){
			SQLBinaryOpExpr parentExpr = (SQLBinaryOpExpr)x.getParent();
			SQLAggregateExpr saexpr = null;
			switch (parentExpr.getOperator()) {
			case GreaterThan:
			case GreaterThanOrEqual:
			case NotLessThan:
				this.hasChange = true;
				if(sexpr instanceof SQLIdentifierExpr 
						|| (sexpr instanceof SQLPropertyExpr&&((SQLPropertyExpr)sexpr).getOwner() instanceof SQLIdentifierExpr)){
					saexpr = new SQLAggregateExpr("MAX");
					saexpr.getArguments().add(sexpr);
	        		saexpr.setParent(itemlist.get(0));
	        		itemlist.get(0).setExpr(saexpr);
				}
				SQLQueryExpr maxSubQuery = new SQLQueryExpr(x.getSubQuery());
        		x.getSubQuery().setParent(x.getParent());
        		// 生成新的SQLQueryExpr 替换当前 SQLAllExpr 节点
            	if(x.getParent() instanceof SQLBinaryOpExpr){
            		if(((SQLBinaryOpExpr)x.getParent()).getLeft().equals(x)){
            			((SQLBinaryOpExpr)x.getParent()).setLeft(maxSubQuery);
            		}else if(((SQLBinaryOpExpr)x.getParent()).getRight().equals(x)){
            			((SQLBinaryOpExpr)x.getParent()).setRight(maxSubQuery);
            		}
            	}
            	addSubQuerys(x.getSubQuery());
            	return super.visit(x.getSubQuery());
			case LessThan:
			case LessThanOrEqual:
			case NotGreaterThan:
				this.hasChange = true;
				if(sexpr instanceof SQLIdentifierExpr 
						|| (sexpr instanceof SQLPropertyExpr&&((SQLPropertyExpr)sexpr).getOwner() instanceof SQLIdentifierExpr)){
					saexpr = new SQLAggregateExpr("MIN");
					saexpr.getArguments().add(sexpr);
	        		saexpr.setParent(itemlist.get(0));
	        		itemlist.get(0).setExpr(saexpr);
	        		
	            	x.subQuery.setParent(x.getParent());
				}
				// 生成新的SQLQueryExpr 替换当前 SQLAllExpr 节点
            	SQLQueryExpr minSubQuery = new SQLQueryExpr(x.getSubQuery());
            	if(x.getParent() instanceof SQLBinaryOpExpr){
            		if(((SQLBinaryOpExpr)x.getParent()).getLeft().equals(x)){
            			((SQLBinaryOpExpr)x.getParent()).setLeft(minSubQuery);
            		}else if(((SQLBinaryOpExpr)x.getParent()).getRight().equals(x)){
            			((SQLBinaryOpExpr)x.getParent()).setRight(minSubQuery);
            		}
            	}
            	addSubQuerys(x.getSubQuery());
            	return super.visit(x.getSubQuery());
			 case LessThanOrGreater:
			 case NotEqual:
				this.hasChange = true;
				SQLInSubQueryExpr notInSubQueryExpr = new SQLInSubQueryExpr(x.getSubQuery());
				x.getSubQuery().setParent(notInSubQueryExpr);
				notInSubQueryExpr.setNot(true);
				// 生成新的SQLQueryExpr 替换当前 SQLAllExpr 节点
				if(x.getParent() instanceof SQLBinaryOpExpr){
					SQLBinaryOpExpr xp = (SQLBinaryOpExpr)x.getParent();
					
					if(xp.getLeft().equals(x)){
						notInSubQueryExpr.setExpr(xp.getRight());
					}else if(xp.getRight().equals(x)){
						notInSubQueryExpr.setExpr(xp.getLeft());
					}
					
					if(xp.getParent() instanceof MySqlSelectQueryBlock){
						((MySqlSelectQueryBlock)xp.getParent()).setWhere(notInSubQueryExpr);
					}else if(xp.getParent() instanceof SQLBinaryOpExpr){
						SQLBinaryOpExpr pp = ((SQLBinaryOpExpr)xp.getParent());
						if(pp.getLeft().equals(xp)){
							pp.setLeft(notInSubQueryExpr);
						}else if(pp.getRight().equals(xp)){
							pp.setRight(notInSubQueryExpr);
						}
					}
	            }
				addSubQuerys(x.getSubQuery());
	            return super.visit(notInSubQueryExpr);
			 default:
				break;
			}
		}
		addSubQuerys(x.getSubQuery());
    	return super.visit(x);
    }
    
    /* 
     *  遇到 some 将子查询改写成  SELECT MIN(name) FROM subtest1
     *  例如:
     *        select * from subtest where id > some (select name from subtest1);
     *    >/>= some ----> >/>= min
     *        some ----> not in
     *    =    some ----> in
     *    other  不改写
     */
    @Override
    public boolean visit(SQLSomeExpr x) {
    	
    	setSubQueryRelationOrFlag(x);
    	
    	List itemlist = ((SQLSelectQueryBlock)(x.getSubQuery().getQuery())).getSelectList();
    	SQLExpr sexpr = itemlist.get(0).getExpr();
    	
		if(x.getParent() instanceof SQLBinaryOpExpr){
			SQLBinaryOpExpr parentExpr = (SQLBinaryOpExpr)x.getParent();
			SQLAggregateExpr saexpr = null;
			switch (parentExpr.getOperator()) {
			case GreaterThan:
			case GreaterThanOrEqual:
			case NotLessThan:
				this.hasChange = true;
				if(sexpr instanceof SQLIdentifierExpr 
						|| (sexpr instanceof SQLPropertyExpr&&((SQLPropertyExpr)sexpr).getOwner() instanceof SQLIdentifierExpr)){
					saexpr = new SQLAggregateExpr("MIN");
					saexpr.getArguments().add(sexpr);
	        		saexpr.setParent(itemlist.get(0));
	        		itemlist.get(0).setExpr(saexpr);
				}
				SQLQueryExpr maxSubQuery = new SQLQueryExpr(x.getSubQuery());
        		x.getSubQuery().setParent(maxSubQuery);
        		// 生成新的SQLQueryExpr 替换当前 SQLAllExpr 节点
            	if(x.getParent() instanceof SQLBinaryOpExpr){
            		if(((SQLBinaryOpExpr)x.getParent()).getLeft().equals(x)){
            			((SQLBinaryOpExpr)x.getParent()).setLeft(maxSubQuery);
            		}else if(((SQLBinaryOpExpr)x.getParent()).getRight().equals(x)){
            			((SQLBinaryOpExpr)x.getParent()).setRight(maxSubQuery);
            		}
            	}
            	addSubQuerys(x.getSubQuery());
            	return super.visit(x.getSubQuery());
			case LessThan:
			case LessThanOrEqual:
			case NotGreaterThan:
				this.hasChange = true;
				if(sexpr instanceof SQLIdentifierExpr 
						|| (sexpr instanceof SQLPropertyExpr&&((SQLPropertyExpr)sexpr).getOwner() instanceof SQLIdentifierExpr)){
					saexpr = new SQLAggregateExpr("MAX");
					saexpr.getArguments().add(sexpr);
	        		saexpr.setParent(itemlist.get(0));
	        		itemlist.get(0).setExpr(saexpr);
				}
				// 生成新的SQLQueryExpr 替换当前 SQLAllExpr 节点
            	SQLQueryExpr minSubQuery = new SQLQueryExpr(x.getSubQuery());
        		x.getSubQuery().setParent(minSubQuery);
            	
            	if(x.getParent() instanceof SQLBinaryOpExpr){
            		if(((SQLBinaryOpExpr)x.getParent()).getLeft().equals(x)){
            			((SQLBinaryOpExpr)x.getParent()).setLeft(minSubQuery);
            		}else if(((SQLBinaryOpExpr)x.getParent()).getRight().equals(x)){
            			((SQLBinaryOpExpr)x.getParent()).setRight(minSubQuery);
            		}
            	}
            	addSubQuerys(x.getSubQuery());
            	return super.visit(x.getSubQuery());
			 case LessThanOrGreater:
			 case NotEqual:
				 this.hasChange = true;
					SQLInSubQueryExpr notInSubQueryExpr = new SQLInSubQueryExpr(x.getSubQuery());
					x.getSubQuery().setParent(notInSubQueryExpr);
					notInSubQueryExpr.setNot(true);
					// 生成新的SQLQueryExpr 替换当前 SQLAllExpr 节点
					if(x.getParent() instanceof SQLBinaryOpExpr){
						SQLBinaryOpExpr xp = (SQLBinaryOpExpr)x.getParent();
						
						if(xp.getLeft().equals(x)){
							notInSubQueryExpr.setExpr(xp.getRight());
						}else if(xp.getRight().equals(x)){
							notInSubQueryExpr.setExpr(xp.getLeft());
						}
						
						if(xp.getParent() instanceof MySqlSelectQueryBlock){
							((MySqlSelectQueryBlock)xp.getParent()).setWhere(notInSubQueryExpr);
						}else if(xp.getParent() instanceof SQLBinaryOpExpr){
							SQLBinaryOpExpr pp = ((SQLBinaryOpExpr)xp.getParent());
							if(pp.getLeft().equals(xp)){
								pp.setLeft(notInSubQueryExpr);
							}else if(pp.getRight().equals(xp)){
								pp.setRight(notInSubQueryExpr);
							}
						}
		            }
					addSubQuerys(x.getSubQuery());
		            return super.visit(notInSubQueryExpr);
			 case Equality:
				 this.hasChange = true;
				SQLInSubQueryExpr inSubQueryExpr = new SQLInSubQueryExpr(x.getSubQuery());
				x.getSubQuery().setParent(inSubQueryExpr);
				inSubQueryExpr.setNot(false);
				// 生成新的SQLQueryExpr 替换当前 SQLAllExpr 节点
				if(x.getParent() instanceof SQLBinaryOpExpr){
					SQLBinaryOpExpr xp = (SQLBinaryOpExpr)x.getParent();
					
					if(xp.getLeft().equals(x)){
						inSubQueryExpr.setExpr(xp.getRight());
					}else if(xp.getRight().equals(x)){
						inSubQueryExpr.setExpr(xp.getLeft());
					}
					
					if(xp.getParent() instanceof MySqlSelectQueryBlock){
						((MySqlSelectQueryBlock)xp.getParent()).setWhere(inSubQueryExpr);
					}else if(xp.getParent() instanceof SQLBinaryOpExpr){
						SQLBinaryOpExpr pp = ((SQLBinaryOpExpr)xp.getParent());
						if(pp.getLeft().equals(xp)){
							pp.setLeft(inSubQueryExpr);
						}else if(pp.getRight().equals(xp)){
							pp.setRight(inSubQueryExpr);
						}
					}
	            }
				addSubQuerys(x.getSubQuery());
	            return super.visit(inSubQueryExpr);
			 default:
				break;
			}
		}
		addSubQuerys(x.getSubQuery());
    	return super.visit(x);
    }

    /* 
     *  遇到 any 将子查询改写成  SELECT MIN(name) FROM subtest1
     *  例如:
     *    select * from subtest where id oper any (select name from subtest1);
     *    >/>= any ----> >/>= min
     *        any ----> not in
     *    =    some ----> in
     *    other  不改写
     */
    @Override
    public boolean visit(SQLAnyExpr x) {
    	
    	setSubQueryRelationOrFlag(x);
    	
    	List itemlist = ((SQLSelectQueryBlock)(x.getSubQuery().getQuery())).getSelectList();
    	SQLExpr sexpr = itemlist.get(0).getExpr();
    	
		if(x.getParent() instanceof SQLBinaryOpExpr){
			SQLBinaryOpExpr parentExpr = (SQLBinaryOpExpr)x.getParent();
			SQLAggregateExpr saexpr = null;
			switch (parentExpr.getOperator()) {
			case GreaterThan:
			case GreaterThanOrEqual:
			case NotLessThan:
				this.hasChange = true;
				if(sexpr instanceof SQLIdentifierExpr 
						|| (sexpr instanceof SQLPropertyExpr&&((SQLPropertyExpr)sexpr).getOwner() instanceof SQLIdentifierExpr)){
					saexpr = new SQLAggregateExpr("MIN");
					saexpr.getArguments().add(sexpr);
	        		saexpr.setParent(itemlist.get(0));
	        		itemlist.get(0).setExpr(saexpr);
				}
				SQLQueryExpr maxSubQuery = new SQLQueryExpr(x.getSubQuery());
        		x.getSubQuery().setParent(maxSubQuery);
				// 生成新的SQLQueryExpr 替换当前 SQLAllExpr 节点
            	if(x.getParent() instanceof SQLBinaryOpExpr){
            		if(((SQLBinaryOpExpr)x.getParent()).getLeft().equals(x)){
            			((SQLBinaryOpExpr)x.getParent()).setLeft(maxSubQuery);
            		}else if(((SQLBinaryOpExpr)x.getParent()).getRight().equals(x)){
            			((SQLBinaryOpExpr)x.getParent()).setRight(maxSubQuery);
            		}
            	}
            	addSubQuerys(x.getSubQuery());
            	return super.visit(x.getSubQuery());
			case LessThan:
			case LessThanOrEqual:
			case NotGreaterThan:
				this.hasChange = true;
				if(sexpr instanceof SQLIdentifierExpr 
						|| (sexpr instanceof SQLPropertyExpr&&((SQLPropertyExpr)sexpr).getOwner() instanceof SQLIdentifierExpr)){
					saexpr = new SQLAggregateExpr("MAX");
					saexpr.getArguments().add(sexpr);
	        		saexpr.setParent(itemlist.get(0));
	        		itemlist.get(0).setExpr(saexpr);
				}
				// 生成新的SQLQueryExpr 替换当前 SQLAllExpr 节点
            	SQLQueryExpr minSubQuery = new SQLQueryExpr(x.getSubQuery());
            	x.subQuery.setParent(minSubQuery);
            	if(x.getParent() instanceof SQLBinaryOpExpr){
            		if(((SQLBinaryOpExpr)x.getParent()).getLeft().equals(x)){
            			((SQLBinaryOpExpr)x.getParent()).setLeft(minSubQuery);
            		}else if(((SQLBinaryOpExpr)x.getParent()).getRight().equals(x)){
            			((SQLBinaryOpExpr)x.getParent()).setRight(minSubQuery);
            		}
            	}
            	addSubQuerys(x.getSubQuery());
            	return super.visit(x.getSubQuery());
			 case LessThanOrGreater:
			 case NotEqual:
				 this.hasChange = true;
					SQLInSubQueryExpr notInSubQueryExpr = new SQLInSubQueryExpr(x.getSubQuery());
					x.getSubQuery().setParent(notInSubQueryExpr);
					notInSubQueryExpr.setNot(true);
					// 生成新的SQLQueryExpr 替换当前 SQLAllExpr 节点
					if(x.getParent() instanceof SQLBinaryOpExpr){
						SQLBinaryOpExpr xp = (SQLBinaryOpExpr)x.getParent();
						
						if(xp.getLeft().equals(x)){
							notInSubQueryExpr.setExpr(xp.getRight());
						}else if(xp.getRight().equals(x)){
							notInSubQueryExpr.setExpr(xp.getLeft());
						}
						
						if(xp.getParent() instanceof MySqlSelectQueryBlock){
							((MySqlSelectQueryBlock)xp.getParent()).setWhere(notInSubQueryExpr);
						}else if(xp.getParent() instanceof SQLBinaryOpExpr){
							SQLBinaryOpExpr pp = ((SQLBinaryOpExpr)xp.getParent());
							if(pp.getLeft().equals(xp)){
								pp.setLeft(notInSubQueryExpr);
							}else if(pp.getRight().equals(xp)){
								pp.setRight(notInSubQueryExpr);
							}
						}
		            }
					addSubQuerys(x.getSubQuery());
		            return super.visit(notInSubQueryExpr);
			 case Equality:
				 this.hasChange = true;
				SQLInSubQueryExpr inSubQueryExpr = new SQLInSubQueryExpr(x.getSubQuery());
				x.getSubQuery().setParent(inSubQueryExpr);
				inSubQueryExpr.setNot(false);
				// 生成新的SQLQueryExpr 替换当前 SQLAllExpr 节点
				if(x.getParent() instanceof SQLBinaryOpExpr){
					SQLBinaryOpExpr xp = (SQLBinaryOpExpr)x.getParent();
					
					if(xp.getLeft().equals(x)){
						inSubQueryExpr.setExpr(xp.getRight());
					}else if(xp.getRight().equals(x)){
						inSubQueryExpr.setExpr(xp.getLeft());
					}
					
					if(xp.getParent() instanceof MySqlSelectQueryBlock){
						((MySqlSelectQueryBlock)xp.getParent()).setWhere(inSubQueryExpr);
					}else if(xp.getParent() instanceof SQLBinaryOpExpr){
						SQLBinaryOpExpr pp = ((SQLBinaryOpExpr)xp.getParent());
						if(pp.getLeft().equals(xp)){
							pp.setLeft(inSubQueryExpr);
						}else if(pp.getRight().equals(xp)){
							pp.setRight(inSubQueryExpr);
						}
					}
	            }
				addSubQuerys(x.getSubQuery());
	            return super.visit(inSubQueryExpr);
			 default:
				break;
			}
		}
		addSubQuerys(x.getSubQuery());
    	return super.visit(x);
    }
    
    @Override
	public boolean visit(SQLBinaryOpExpr x) {
        x.getLeft().setParent(x);
        x.getRight().setParent(x);
        
        /*
         * fix bug 当 selectlist 存在多个子查询时, 主表没有别名的情况下.主表的查询条件 被错误的附加到子查询上.
         *  eg. select (select id from subtest2 where id = 1), (select id from subtest3 where id = 2) from subtest1 where id =4;
         *  像这样的子查询, subtest1 的 过滤条件  id = 4 .  被 加入到  subtest3 上. 加别名的情况下正常,不加别名,就会存在这个问题.
         *  这里设置好操作的是哪张表后,再进行判断.
         */
		// druid新版本没有这个变量，先注释掉。
//        String currenttable = x.getParent()==null?null: (String) x.getParent().getAttribute(SchemaStatVisitor.ATTR_TABLE);
//        if(currenttable!=null){
//        	this.setCurrentTable(currenttable);
//        }
//        
        switch (x.getOperator()) {
            case Equality:
            case LessThanOrEqualOrGreaterThan:
            case Is:
            case IsNot:
            case GreaterThan:
            case GreaterThanOrEqual:
            case LessThan:
            case LessThanOrEqual:
            case NotLessThan:
            case LessThanOrGreater:
			case NotEqual:
			case NotGreaterThan:
                handleCondition(x.getLeft(), x.getOperator().name, x.getRight());
                handleCondition(x.getRight(), x.getOperator().name, x.getLeft());
                handleRelationship(x.getLeft(), x.getOperator().name, x.getRight());
                break;
            case BooleanOr:
            	//永真条件，where条件抛弃
                if (!RouterUtil.isConditionAlwaysTrue(x)) {
                  hasOrCondition = true;
                  orBinaryExprs.add(x);
                }
            	return false;
            case Like:
            case NotLike:
            default:
                break;
        }

		super.statExpr(x.getLeft());
		super.statExpr(x.getRight());
		return false;
		// return true;
    }
	
    /**
     * 根据带or的expr创建 WhereUnits
     */
    private void buildWhereUnits() {
        for (SQLBinaryOpExpr orExpr : orBinaryExprs) {
            WhereUnit whereUnit = new WhereUnit();
            whereUnit.setFinishedParse(true);
            whereUnit.addOutConditions(getConditions());
            WhereUnit innerWhereUnit = new WhereUnit(orExpr);
            whereUnit.addSubWhereUnit(innerWhereUnit);
            whereUnits.add(whereUnit);
        }
    }

    /**
     * 分解条件
     */
    public List> splitConditions() {
        buildWhereUnits();

        // 按照or拆分
		for(WhereUnit whereUnit : whereUnits) {
			splitUntilNoOr(whereUnit);
		}
		
		this.storedwhereUnits.addAll(whereUnits);
		
		loopFindSubWhereUnit(whereUnits);
		
		//拆分后的条件块解析成Condition列表
		for(WhereUnit whereUnit : storedwhereUnits) {
			this.getConditionsFromWhereUnit(whereUnit);
		}
		
		//多个WhereUnit组合:多层集合的组合
		return mergedConditions();
	}
	
	/**
	 * 循环寻找子WhereUnit（实际是嵌套的or）
	 * @param whereUnitList
	 */
	private void loopFindSubWhereUnit(List whereUnitList) {
		List subWhereUnits = new ArrayList();
		for(WhereUnit whereUnit : new ArrayList<>(whereUnitList)) {
			if(whereUnit.getSplitedExprList().size() > 0) {
				List removeSplitedList = new ArrayList();
				for(SQLExpr sqlExpr : whereUnit.getSplitedExprList()) {
					reset();

					if(isExprHasOr(sqlExpr)) {
                        this.buildWhereUnits();
                        removeSplitedList.add(sqlExpr);
						WhereUnit subWhereUnit = this.whereUnits.get(0);
						splitUntilNoOr(subWhereUnit);
						whereUnit.addSubWhereUnit(subWhereUnit);
						subWhereUnits.add(subWhereUnit);
					} else {
						this.conditions.clear();
					}
				}
				if(removeSplitedList.size() > 0) {
					whereUnit.getSplitedExprList().removeAll(removeSplitedList);
				}
			}
			subWhereUnits.addAll(whereUnit.getSubWhereUnit());
		}
		if(subWhereUnits.size() > 0) {
			loopFindSubWhereUnit(subWhereUnits);
		}
	}
	
	private boolean isExprHasOr(SQLExpr expr) {
		expr.accept(this);
		return hasOrCondition;
	}
	
	private List> mergedConditions() {
		if(storedwhereUnits.size() == 0) {
			return new ArrayList>();
		}
		for(WhereUnit whereUnit : storedwhereUnits) {
			mergeOneWhereUnit(whereUnit);
		}
		return getMergedConditionList(storedwhereUnits);
		
	}
	
	/**
	 * 一个WhereUnit内递归
	 * @param whereUnit
	 */
	private void mergeOneWhereUnit(WhereUnit whereUnit) {
		if(whereUnit.getSubWhereUnit().size() > 0) {
			for(WhereUnit sub : whereUnit.getSubWhereUnit()) {
				mergeOneWhereUnit(sub);
			}
			
			if(whereUnit.getSubWhereUnit().size() > 1) {
				List> mergedConditionList = getMergedConditionList(whereUnit.getSubWhereUnit());
				if(whereUnit.getOutConditions().size() > 0) {
					for(int i = 0; i < mergedConditionList.size() ; i++) {
						mergedConditionList.get(i).addAll(whereUnit.getOutConditions());
					}
				}
				whereUnit.setConditionList(mergedConditionList);
			} else if(whereUnit.getSubWhereUnit().size() == 1) {
				if(whereUnit.getOutConditions().size() > 0 && whereUnit.getSubWhereUnit().get(0).getConditionList().size() > 0) {
					for(int i = 0; i < whereUnit.getSubWhereUnit().get(0).getConditionList().size() ; i++) {
						whereUnit.getSubWhereUnit().get(0).getConditionList().get(i).addAll(whereUnit.getOutConditions());
					}
				}
				whereUnit.getConditionList().addAll(whereUnit.getSubWhereUnit().get(0).getConditionList());
			}
		} else {
			//do nothing
		}
	}
	
	/**
	 * 条件合并：多个WhereUnit中的条件组合
	 * @return
	 */
	private List> getMergedConditionList(List whereUnitList) {
		List> mergedConditionList = new ArrayList>();
		if(whereUnitList.size() == 0) {
			return mergedConditionList; 
		}
		mergedConditionList.addAll(whereUnitList.get(0).getConditionList());
		
		for(int i = 1; i < whereUnitList.size(); i++) {
			mergedConditionList = merge(mergedConditionList, whereUnitList.get(i).getConditionList());
		}
		return mergedConditionList;
	}
	
	/**
	 * 两个list中的条件组合
	 * @param list1
	 * @param list2
	 * @return
	 */
	    private List> merge(List> list1, List> list2) {
        if(list1.size() == 0) {
            return list2;
        } else if (list2.size() == 0) {
            return list1;
        }
        
		List> retList = new ArrayList>();
		for(int i = 0; i < list1.size(); i++) {
			for(int j = 0; j < list2.size(); j++) {
//				List listTmp = new ArrayList();
//				listTmp.addAll(list1.get(i));
//				listTmp.addAll(list2.get(j));
//				retList.add(listTmp);
			    /**
		         * 单纯做笛卡尔积运算，会导致非常多不必要的条件列表，

		         * 当whereUnit和条件相对多时，会急剧增长条件列表项，内存直线上升，导致假死状态

		         * 因此，修改算法为 

		         * 1、先合并两个条件列表的元素为一个条件列表

		         * 2、计算合并后的条件列表，在结果retList中：

		         *  2-1、如果当前的条件列表 是 另外一个条件列表的 超集，更新，并标识已存在

		         *  2-2、如果当前的条件列表 是 另外一个条件列表的 子集，标识已存在

		         * 3、最后，如果被标识不存在，加入结果retList，否则丢弃。

		         * 
		         * @author SvenAugustus
		         */
  			    // 合并两个条件列表的元素为一个条件列表
                List listTmp = mergeSqlConditionList(list1.get(i), list2.get(j));
      
                // 判定当前的条件列表 是否 另外一个条件列表的 子集
                boolean exists = false;
                Iterator> it = retList.iterator();
                while (it.hasNext()) {
                  List result = (List) it.next();
                  if (result != null && listTmp != null && listTmp.size() > result.size()) {
                    // 如果当前的条件列表 是 另外一个条件列表的 超集，更新，并标识已存在
                    if (sqlConditionListInOther(result, listTmp)) {
                      result.clear();
                      result.addAll(listTmp);
                      exists = true;
                      break;
                    }
                  } else {
                    // 如果当前的条件列表 是 另外一个条件列表的 子集，标识已存在
                    if (sqlConditionListInOther(listTmp, result)) {
                      exists = true;
                      break;
                    }
                  }
                }
                if (!exists) {// 被标识不存在，加入
                  retList.add(listTmp);
                } // 否则丢弃
			}
		}
        return retList;
    }
	
	private void getConditionsFromWhereUnit(WhereUnit whereUnit) {
		List> retList = new ArrayList>();
		//or语句外层的条件:如where condition1 and (condition2 or condition3),condition1就会在外层条件中,因为之前提取
		List outSideCondition = new ArrayList();
//		stashOutSideConditions();
		outSideCondition.addAll(conditions);
		this.conditions.clear();
		for(SQLExpr sqlExpr : whereUnit.getSplitedExprList()) {
			sqlExpr.accept(this);
//            List conditions = new ArrayList();
//            conditions.addAll(getConditions()); conditions.addAll(outSideCondition);
          /**
           * 合并两个条件列表的元素为一个条件列表，减少不必要多的条件项

           * 
           * @author SvenAugustus
           */
          List conditions = mergeSqlConditionList(getConditions(), outSideCondition);
			retList.add(conditions);
			this.conditions.clear();
		}
		whereUnit.setConditionList(retList);
		
		for(WhereUnit subWhere : whereUnit.getSubWhereUnit()) {
			getConditionsFromWhereUnit(subWhere);
		}
	}
	
	/**
	 * 递归拆分OR
	 * 
	 * @param whereUnit
	 * TODO:考虑嵌套or语句，条件中有子查询、 exists等很多种复杂情况是否能兼容
	 */
	private void splitUntilNoOr(WhereUnit whereUnit) {
		if(whereUnit.isFinishedParse()) {
			if(whereUnit.getSubWhereUnit().size() > 0) {
				for(int i = 0; i < whereUnit.getSubWhereUnit().size(); i++) {
					splitUntilNoOr(whereUnit.getSubWhereUnit().get(i));
				}
			} 
		} else {
			SQLBinaryOpExpr expr = whereUnit.getCanSplitExpr();
			if(expr.getOperator() == SQLBinaryOperator.BooleanOr) {
//				whereUnit.addSplitedExpr(expr.getRight());
				addExprIfNotFalse(whereUnit, expr.getRight());
				if(expr.getLeft() instanceof SQLBinaryOpExpr) {
					whereUnit.setCanSplitExpr((SQLBinaryOpExpr)expr.getLeft());
					splitUntilNoOr(whereUnit);
				} else {
					addExprIfNotFalse(whereUnit, expr.getLeft());
				}
			} else {
				addExprIfNotFalse(whereUnit, expr);
				whereUnit.setFinishedParse(true);
			}
		}
    }

	private void addExprIfNotFalse(WhereUnit whereUnit, SQLExpr expr) {
		//非永假条件加入路由计算
		if(!RouterUtil.isConditionAlwaysFalse(expr)) {
			whereUnit.addSplitedExpr(expr);
		}
	}
	
	@Override
    public boolean visit(SQLAlterTableStatement x) {
        String tableName = x.getName().toString();
		TableStat stat = getTableStat(tableName);
        stat.incrementAlterCount();

		setCurrentTable(tableName);
		// setCurrentTable(x, tableName);

        for (SQLAlterTableItem item : x.getItems()) {
            item.setParent(x);
            item.accept(this);
        }

        return false;
    }
    public boolean visit(MySqlCreateTableStatement x) {
        SQLName sqlName=  x.getName();
        if(sqlName!=null)
        {
            String table = sqlName.toString();
            if(table.startsWith("`"))
            {
                table=table.substring(1,table.length()-1);
            }
            setCurrentTable(table);
        }
        return false;
    }
    public boolean visit(MySqlInsertStatement x) {
        SQLName sqlName=  x.getTableName();
        if(sqlName!=null)
        {
            String table = sqlName.toString();
            if(table.startsWith("`"))
            {
                table=table.substring(1,table.length()-1);
            }
            setCurrentTable(sqlName.toString());
        }
        return false;
    }
	// DUAL
    public boolean visit(MySqlDeleteStatement x) {
		// setAliasMap();
		aliasMap.clear();
		this.clearTableVisitFlag();

        setMode(x, Mode.Delete);

        accept(x.getFrom());
        accept(x.getUsing());
        x.getTableSource().accept(this);

        if (x.getTableSource() instanceof SQLExprTableSource) {
            SQLName tableName = (SQLName) ((SQLExprTableSource) x.getTableSource()).getExpr();
            String ident = tableName.toString();
			// setCurrentTable(x, ident);
			setCurrentTable(ident);

			TableStat stat = this.getTableStat(ident);
            stat.incrementDeleteCount();
        }

        accept(x.getWhere());

        accept(x.getOrderBy());
        accept(x.getLimit());

        return false;
    }
    
    public void endVisit(MySqlDeleteStatement x) {
    }
    
    public boolean visit(SQLUpdateStatement x) {
		// setAliasMap();
		aliasMap.clear();
		this.clearTableVisitFlag();

        setMode(x, Mode.Update);

        SQLName identName = x.getTableName();
        if (identName != null) {
            String ident = identName.toString();
            String alias = x.getTableSource().getAlias();
            setCurrentTable(ident);

            TableStat stat = getTableStat(ident);
            stat.incrementUpdateCount();

			// Map aliasMap = getAliasMap();
			putAliasToMap(ident, ident);
            if(alias != null) {
				putAliasToMap(alias, ident);
            }
        } else {
            x.getTableSource().accept(this);
        }

        accept(x.getItems());
        accept(x.getWhere());

        return false;
    }
    
    @Override
    public void endVisit(MySqlHintStatement x) {
    	super.endVisit(x);
    }
    
    @Override
    public boolean visit(MySqlHintStatement x) {
    	List hits = x.getHints();
    	if(hits != null && !hits.isEmpty()) {
    		String schema = parseSchema(hits);
    		if(schema != null ) {
				// setCurrentTable(x, schema + ".");
				setCurrentTable(schema + ".");
    			return true;
    		}
    	}
    	return true;
    }

	@Override
	public boolean visit(SQLExprTableSource x) {
		super.visit(x);

		if (this.isSimpleExprTableSource(x)) {
			String ident = x.getExpr().toString();
			setCurrentTable(ident);
			selectTableList.add(ident);
			String alias = x.getAlias();

			if (x.getAttribute(TABLE_HAS_VISIT_FLAG) == null || !((Boolean) x.getAttribute(TABLE_HAS_VISIT_FLAG))) {
				x.putAttribute(TABLE_HAS_VISIT_FLAG, true);
				this.visitedTableSourceExpr.add(x);
				if (alias != null) {
					putAliasToMap(alias, ident);
				} else {
					putAliasToMap(ident, ident);
				}
			}
		} else {
			this.accept(x.getExpr());
		}

		return false;
	}

	private String parseSchema(List hits) {
    	String regx = "\\!mycat:schema\\s*=([\\s\\w]*)$";
    	for(SQLCommentHint hit : hits ) {
    		Pattern pattern = Pattern.compile(regx);
    		Matcher m = pattern.matcher(hit.getText());
    		if(m.matches()) {
    			return m.group(1).trim();
    		}
    	}
		return null;
    }

    public Queue getSubQuerys() {
		return subQuerys;
	}
	
	private void addSubQuerys(SQLSelect sqlselect){
		/* 多个 sqlselect 之间  , equals 和 hashcode 是相同的.去重时 都被过滤掉了. */
		if(subQuerys.isEmpty()){
			subQuerys.add(sqlselect);
			return;
		}
        boolean exists = false;
		Iterator iter = subQuerys.iterator();
		while(iter.hasNext()){
			SQLSelect ss = iter.next();
			if(ss.getQuery() instanceof SQLSelectQueryBlock
					&&sqlselect.getQuery() instanceof SQLSelectQueryBlock){
				SQLSelectQueryBlock current = (SQLSelectQueryBlock)sqlselect.getQuery();
				SQLSelectQueryBlock ssqb = (SQLSelectQueryBlock)ss.getQuery();
//                  if(!sqlSelectQueryBlockEquals(ssqb,current)){
//                    subQuerys.add(sqlselect);
//                  }
                /**
                 * 修正判定逻辑，应改为全不在subQuerys中才加入

                 * 
                 * @author SvenAugustus
                 */
                if(sqlSelectQueryBlockEquals(current,ssqb)){
                   exists = true;
                   break;
                }
				}
			}
        if(!exists) {
          subQuerys.add(sqlselect);
		}
	}
	
	/* 多个 sqlselect 之间  , equals 和 hashcode 是相同的.去重时 使用 SQLSelectQueryBlock equals 方法 */
    private boolean sqlSelectQueryBlockEquals(SQLSelectQueryBlock obj1,SQLSelectQueryBlock obj2) {
        if (obj1 == obj2) return true;
        if (obj2 == null) return false;
        if (obj1.getClass() != obj2.getClass()) return false;
        if (obj1.isParenthesized() ^ obj2.isParenthesized()) return false;
        if (obj1.getDistionOption() != obj2.getDistionOption()) return false;
        if (obj1.getFrom() == null) {
            if (obj2.getFrom() != null) return false;
        } else if (!obj1.getFrom().equals(obj2.getFrom())) return false;
        if (obj1.getGroupBy() == null) {
            if (obj2.getGroupBy() != null) return false;
        } else if (!obj1.getGroupBy().equals(obj2.getGroupBy())) return false;
        if (obj1.getInto() == null) {
            if (obj2.getInto() != null) return false;
        } else if (!obj1.getInto().equals(obj2.getInto())) return false;
        if (obj1.getSelectList() == null) {
            if (obj2.getSelectList() != null) return false;
        } else if (!obj1.getSelectList().equals(obj2.getSelectList())) return false;
        if (obj1.getWhere() == null) {
            if (obj2.getWhere() != null) return false;
        } else if (!obj1.getWhere().equals(obj2.getWhere())) return false;
        return true;
    }

	public boolean isHasChange() {
		return hasChange;
	}

	public boolean isSubqueryRelationOr() {
		return subqueryRelationOr;
	}
    
    /**
     * 判定当前的条件列表 是否 另外一个条件列表的 子集
     * 
     * @author SvenAugustus
     * @param current 当前的条件列表 
     * @param other 另外一个条件列表
     * @return
     */
    private boolean sqlConditionListInOther(List current, List other) {
      if (current == null) {
        if (other != null) {
          return false;
        }
        return true;
      }
      if (current.size() > other.size()) {
        return false;
      }
      if (other.size() == current.size()) {
        // 判定两个条件列表的元素是否内容相等
        return sqlConditionListEquals(current, other);
      }
      for (int j = 0; j < current.size(); j++) {
        boolean exists = false;
        for (int i = 0; i < other.size(); i++) {
          // 判定两个条件是否相等
          if (sqlConditionEquals(current.get(j), other.get(i))) {
            exists = true;
            break;
          }
        }
        if (!exists) {
          return false;
        }
      }
      return true;
    }
    
    /**
     * 判定两个条件列表的元素是否内容相等
     * 
     * @author SvenAugustus
     * @param list1
     * @param list2
     * @return
     */
    private boolean sqlConditionListEquals(List list1, List list2) {
      if (list1 == null) {
        if (list2 != null) {
          return false;
        }
        return true;
      }
      if (list2.size() != list1.size()) {
        return false;
      }
      int len = list1.size();
      for (int j = 0; j < len; j++) {
        boolean exists = false;
        for (int i = 0; i < len; i++) {
          // 判定两个条件是否相等
          if (sqlConditionEquals(list2.get(j), list1.get(i))) {
            exists = true;
            break;
          }
        }
        if (!exists) {
          return false;
        }
      }
      return true;
    }

    /**
     * 合并两个条件列表的元素为一个条件列表
     * 
     * @author SvenAugustus
     * @param list1 条件列表1
     * @param list2 条件列表2
     * @return
     */
    private List mergeSqlConditionList(List list1, List list2) {
      if (list1 == null) {
        list1 = new ArrayList();
      }
      if (list2 == null) {
        list2 = new ArrayList();
      }
      List retList = new ArrayList();
      if (!list1.isEmpty() && !(list1.get(0) instanceof Condition)) {
        return retList;
      }
      if (!list2.isEmpty() && !(list2.get(0) instanceof Condition)) {
        return retList;
      }
      retList.addAll(list1);
      for (int j = 0; j < list2.size(); j++) {
        boolean exists = false;
        for (int i = 0; i < list1.size(); i++) {
          if (sqlConditionEquals(list2.get(j), list1.get(i))) {
            exists = true;
            break;
          }
        }
        if (!exists) {
          retList.add(list2.get(j));
        }
      }
      return retList;
    }
    
    /**
     * 判定两个条件是否相等
     * 
     * @author SvenAugustus
     * @param obj1
     * @param obj2
     * @return
     */
    private boolean sqlConditionEquals(Condition obj1, Condition obj2) {
      if (obj1 == obj2) {
        return true;
      }
      if (obj2 == null) {
        return false;
      }
      if (obj1.getClass() != obj2.getClass()) {
        return false;
      }
      Condition other = (Condition) obj2;
      if (obj1.getColumn() == null) {
        if (other.getColumn() != null) {
          return false;
        }
      } else if (!obj1.getColumn().equals(other.getColumn())) {
        return false;
      }
      if (obj1.getOperator() == null) {
        if (other.getOperator() != null) {
          return false;
        }
      } else if (!obj1.getOperator().equals(other.getOperator())) {
        return false;
      }
      if (obj1.getValues() == null) {
        if (other.getValues() != null) {
          return false;
        }
      } else {
        boolean notEquals=false;
        for (Object val1: obj1.getValues()) {
          for (Object val2: obj2.getValues()) {
            if(val1==null) {
              if(val2!=null) {
                notEquals=true;
                break;
              }
            }else if(!val1.equals(val2)) {
              notEquals=true;
              break;
            }
          }
          if(notEquals)break;
        }
        if(notEquals)
        return false;
      }
      return true;
    }

	public Map getAliasMap() {
		return aliasMap;
	}

	public List getSelectTableList() {
		return selectTableList;
	}

	public String getCurrentTable() {
		return currentTable;
	}

	public void setCurrentTable(String currentTable) {
		this.currentTable = SQLUtils.normalize(currentTable);
	}

	/**
	 * 清理掉表已经被visit标识
	 */
	private void clearTableVisitFlag() {
		for (SQLObjectImpl tableSourceExpr : visitedTableSourceExpr) {
			if (tableSourceExpr.getAttribute(TABLE_HAS_VISIT_FLAG) != null) {
				tableSourceExpr.putAttribute(TABLE_HAS_VISIT_FLAG, false);
			}
		}
		visitedTableSourceExpr.clear();
	}

	/**
	 * key全部转变成大写
	 * 
	 * @param key
	 * @param value
	 */
	private void putAliasToMap(String key, String value) {
		if (key != null) {
			key = StringUtil.removeBackquote(key.toUpperCase());
			if (!aliasMap.containsKey(key)) {
				aliasMap.put(key, StringUtil.removeBackquote(value));
			}

//			if (aliasMap.containsKey(key)) {
//				
//				String msg = String.format(
//						"The alias[%s] of the table[%s] shoud be unique.If the table has no alias, the alias is equal to the name of the table!",
//						key, value);
//				throw new NotSupportException(msg);
//			} else {

			// }
		}
	}

}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/MycatSelectParser.java
================================================
package io.mycat.route.parser.druid;

import com.alibaba.druid.sql.ast.statement.SQLSelectItem;
import com.alibaba.druid.sql.ast.statement.SQLSelectQuery;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlSelectParser;
import com.alibaba.druid.sql.parser.SQLExprParser;
import com.alibaba.druid.sql.parser.Token;

/**
 * Created by nange on 2015/3/13.
 */
public class MycatSelectParser extends MySqlSelectParser
{
    public MycatSelectParser(SQLExprParser exprParser)
    {
        super(exprParser);
    }

    public MycatSelectParser(String sql)
    {
        super(sql);
    }


//public SQLSelectQuery query()
//{
//    parseTop();
//    return super.query();
//}

    public void parseTop()
    {
        if (lexer.token() == Token.TOP)
        {
            lexer.nextToken();

            boolean paren = false;
            if (lexer.token() == Token.LPAREN)
            {
                paren = true;
                lexer.nextToken();
            }

            if (paren)
            {
                accept(Token.RPAREN);
            }

            if (lexer.token() == Token.LITERAL_INT)
            {
                lexer.mark();
                lexer.nextToken();
            }
            if (lexer.token() == Token.IDENTIFIER)
            {
                lexer.nextToken();

            }
            if (lexer.token() == Token.EQ||lexer.token() == Token.DOT)
            {
                lexer.nextToken();
            } else  if(lexer.token() != Token.STAR)
            {
                lexer.reset();
            }
            if (lexer.token() == Token.PERCENT)
            {
                lexer.nextToken();
            }


        }


    }
}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/MycatStatementParser.java
================================================
package io.mycat.route.parser.druid;

import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.expr.SQLCharExpr;
import com.alibaba.druid.sql.ast.expr.SQLLiteralExpr;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlLoadDataInFileStatement;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.druid.sql.parser.*;
import com.alibaba.druid.util.JdbcConstants;

/**
 * Created by nange on 2015/3/13.
 */
public class MycatStatementParser extends MySqlStatementParser
{
    private static final String LOW_PRIORITY   = "LOW_PRIORITY";
    private static final String LOCAL          = "LOCAL";
    private static final String IGNORE         = "IGNORE";
    private static final String CHARACTER      = "CHARACTER";
    public MycatStatementParser(String sql)
    {
        super(sql);
        selectExprParser = new MycatExprParser(sql);
    }

    public MycatStatementParser(Lexer lexer)
    {
        super(lexer);
        selectExprParser = new MycatExprParser(lexer);
    }

    protected SQLExprParser selectExprParser;
    @Override
    public SQLSelectStatement parseSelect()
    {

        MycatSelectParser selectParser = new MycatSelectParser(this.selectExprParser);
        return new SQLSelectStatement(selectParser.select(), JdbcConstants.MYSQL);
    }


    //此处注释掉，以修正后端jdbc方式时，delete语句解析出错的情况
    //
//    public SQLSelectParser createSQLSelectParser()
//    {
//        return new MycatSelectParser(this.selectExprParser);
//    }

    @Override
    protected MySqlLoadDataInFileStatement parseLoadDataInFile()
    {
        acceptIdentifier("DATA");

        LoadDataStatement stmt = new LoadDataStatement();

        if (identifierEquals(LOW_PRIORITY)) {
            stmt.setLowPriority(true);
            lexer.nextToken();
        }

        if (identifierEquals("CONCURRENT")) {
            stmt.setConcurrent(true);
            lexer.nextToken();
        }

        if (identifierEquals(LOCAL)) {
            stmt.setLocal(true);
            lexer.nextToken();
        }

        acceptIdentifier("INFILE");

        SQLLiteralExpr fileName = (SQLLiteralExpr) exprParser.expr();
        stmt.setFileName(fileName);

        if (lexer.token() == Token.REPLACE) {
            stmt.setReplicate(true);
            lexer.nextToken();
        }

        if (identifierEquals(IGNORE)) {
            stmt.setIgnore(true);
            lexer.nextToken();
        }

        accept(Token.INTO);
        accept(Token.TABLE);

        SQLName tableName = exprParser.name();
        stmt.setTableName(tableName);

        if (identifierEquals(CHARACTER)) {
            lexer.nextToken();
            accept(Token.SET);

            if (lexer.token() != Token.LITERAL_CHARS) {
                throw new ParserException("syntax error, illegal charset");
            }

            String charset = lexer.stringVal();
            lexer.nextToken();
            stmt.setCharset(charset);
        }

        if (identifierEquals("FIELDS") || identifierEquals("COLUMNS")) {
            lexer.nextToken();
            if (identifierEquals("TERMINATED")) {
                lexer.nextToken();
                accept(Token.BY);
                stmt.setColumnsTerminatedBy(new SQLCharExpr(lexer.stringVal()));
                lexer.nextToken();
            }

            if (identifierEquals("OPTIONALLY")) {
                stmt.setColumnsEnclosedOptionally(true);
                lexer.nextToken();
            }

            if (identifierEquals("ENCLOSED")) {
                lexer.nextToken();
                accept(Token.BY);
                stmt.setColumnsEnclosedBy(new SQLCharExpr(lexer.stringVal()));
                lexer.nextToken();
            }

            if (identifierEquals("ESCAPED")) {
                lexer.nextToken();
                accept(Token.BY);
                stmt.setColumnsEscaped(new SQLCharExpr(lexer.stringVal()));
                lexer.nextToken();
            }
        }

        if (identifierEquals("LINES")) {
            lexer.nextToken();
            if (identifierEquals("STARTING")) {
                lexer.nextToken();
                accept(Token.BY);
                stmt.setLinesStartingBy(new SQLCharExpr(lexer.stringVal()));
                lexer.nextToken();
            }

            if (identifierEquals("TERMINATED")) {
                lexer.nextToken();
                accept(Token.BY);
                stmt.setLinesTerminatedBy(new SQLCharExpr(lexer.stringVal()));
                lexer.nextToken();
            }
        }

        if (identifierEquals(IGNORE)) {
            lexer.nextToken();
            stmt.setIgnoreLinesNumber( this.exprParser.expr());
            acceptIdentifier("LINES");
        }

        if (lexer.token() == Token.LPAREN) {
            lexer.nextToken();
            this.exprParser.exprList(stmt.getColumns(), stmt);
            accept(Token.RPAREN);
        }

        if (lexer.token() == Token.SET) {
            lexer.nextToken();
            this.exprParser.exprList(stmt.getSetList(), stmt);
        }

        return stmt;
    }
}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/MycatSubQueryVisitor.java
================================================
package io.mycat.route.parser.druid;

import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr;
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlSchemaStatVisitor;

/**
 * 子查询访问器
 */
public class MycatSubQueryVisitor extends MySqlSchemaStatVisitor{
	
	private boolean relationOr;
	
	@Override
	public boolean visit(SQLBinaryOpExpr x) {

		switch (x.getOperator()) {
	            case Equality:
	            case LessThanOrEqualOrGreaterThan:
	            case GreaterThan:
	            case GreaterThanOrEqual:
	            case LessThan:
	            case LessThanOrEqual:
	            case NotLessThan:
	            case LessThanOrGreater:
   			 	case NotEqual:
   			 	case NotGreaterThan:	            	
	                break;
	            case BooleanOr:
	            	relationOr = true;
	            	break;
	            case Like:
	            case NotLike:
	            default:
	                break;
	        }
	        return true;
	}

	public boolean isRelationOr() {
		return relationOr;
	}
	
	
}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/RouteCalculateUnit.java
================================================
package io.mycat.route.parser.druid;

import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;

import io.mycat.sqlengine.mpp.ColumnRoutePair;
import io.mycat.sqlengine.mpp.RangeValue;

/**
 * 路由计算单元
 * 
 * @author wang.dw
 * @date 2015-3-14 下午6:24:54
 * @version 0.1.0 
 * @copyright wonhigh.cn
 */
public class RouteCalculateUnit {
	private Map>> tablesAndConditions = new LinkedHashMap>>();

	public Map>> getTablesAndConditions() {
		return tablesAndConditions;
	}

	public void addShardingExpr(String tableName, String columnName, Object value) {
		Map> tableColumnsMap = tablesAndConditions.get(tableName);
		
		if (value == null) {
			// where a=null
			return;
		}
		
		if (tableColumnsMap == null) {
			tableColumnsMap = new LinkedHashMap>();
			tablesAndConditions.put(tableName, tableColumnsMap);
		}
		
		String uperColName = columnName.toUpperCase();
		Set columValues = tableColumnsMap.get(uperColName);

		if (columValues == null) {
			columValues = new LinkedHashSet();
			tablesAndConditions.get(tableName).put(uperColName, columValues);
		}

		if (value instanceof Object[]) {
			for (Object item : (Object[]) value) {
				if(item == null) {
					continue;
				}
				columValues.add(new ColumnRoutePair(item.toString()));
			}
		} else if (value instanceof RangeValue) {
			columValues.add(new ColumnRoutePair((RangeValue) value));
		} else {
			columValues.add(new ColumnRoutePair(value.toString()));
		}
	}
	
	public void clear() {
		tablesAndConditions.clear();
	}
	
	
}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/SchemaStatVisitorFactory.java
================================================
package io.mycat.route.parser.druid;

import io.mycat.config.model.SchemaConfig;

/**
 * 为防止SchemaStatVisitor被污染，采用factory创建
 *
 * Date：2017年12月1日
 * 
 * @author SvenAugustus
 * @version 1.0
 * @since JDK 1.7
 */
public class SchemaStatVisitorFactory {

  /**
   * 创建
   * 
   * @return
   */
	public static MycatSchemaStatVisitor create(SchemaConfig schema) {
		MycatSchemaStatVisitor visitor = new MycatSchemaStatVisitor();
		return visitor;
  }
}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/SqlMethodInvocationHandler.java
================================================
package io.mycat.route.parser.druid;

import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr;

import java.sql.SQLNonTransientException;

/**
 * 调用sql函数如now()等
 *
 * @author zhuyiqiang
 * @version 2018/9/3
 */
public interface SqlMethodInvocationHandler {
    /**
     * 调用sql函数，返回结果
     * @param expr 函数表达式
     * @return 执行结果
     * @throws SQLNonTransientException
     */
    String invoke(SQLMethodInvokeExpr expr) throws SQLNonTransientException;
}

================================================
FILE: src/main/java/io/mycat/route/parser/druid/SqlMethodInvocationHandlerFactory.java
================================================
package io.mycat.route.parser.druid;

import io.mycat.route.parser.druid.impl.MysqlMethodInvocationHandler;
import io.mycat.route.parser.druid.impl.OracleMethodInvocationHandler;
import io.mycat.route.parser.druid.impl.PgsqlMethodInvocationHandler;

/**
 * 按不同的db生成对应的sql函数处理器
 *
 * @author zhuyiqiang
 * @version 2018/9/5
 */
public class SqlMethodInvocationHandlerFactory {
    private static MysqlMethodInvocationHandler mysql = new MysqlMethodInvocationHandler();
    private static OracleMethodInvocationHandler oracle = new OracleMethodInvocationHandler();
    private static PgsqlMethodInvocationHandler pgsql = new PgsqlMethodInvocationHandler();

    public static MysqlMethodInvocationHandler getForMysql() {
        return mysql;
    }

    public static OracleMethodInvocationHandler getForOracle() {
        return oracle;
    }

    public static PgsqlMethodInvocationHandler getForPgsql() {
        return pgsql;
    }
}

================================================
FILE: src/main/java/io/mycat/route/parser/druid/WhereUnit.java
================================================
package io.mycat.route.parser.druid;

import java.util.ArrayList;
import java.util.List;

import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr;
import com.alibaba.druid.stat.TableStat.Condition;

/**
 * Where条件单元
 * 
 * @author wang.dw
 * @date 2015-3-17 下午4:21:21
 * @version 0.1.0 
 * @copyright wonhigh.cn
 * 
 * 示例：
SELECT id,traveldate
FROM   travelrecord
WHERE  id = 1
       AND ( fee > 0
              OR days > 0
              OR ( traveldate > '2015-05-04 00:00:07.375'
                   AND ( user_id <= 2
                          OR fee = days
                          OR fee > 0 ) ) )
       AND name = 'zhangsan'
ORDER  BY traveldate DESC
LIMIT  20 
 * 
 * 
 * 
 */
public class WhereUnit {
	/**
	 * 完整的where条件
	 */
	private SQLBinaryOpExpr whereExpr;
	
	/**
	 * 还能继续再分的表达式:可能还有or关键字
	 */
	private SQLBinaryOpExpr canSplitExpr;
	
	private List splitedExprList = new ArrayList();
	
	private List> conditionList = new ArrayList>();
	
	/**
	 * whereExpr并不是一个where的全部，有部分条件在outConditions
	 */
	private List outConditions = new ArrayList();

	/**
	 * 按照or拆分后的条件片段中可能还有or语句，这样的片段实际上是嵌套的or语句，将其作为内层子whereUnit，不管嵌套多少层，循环处理
	 */
	private List subWhereUnits = new ArrayList();
	
	private boolean finishedParse = false;
	
	public List getOutConditions() {
		return outConditions;
	}

	public void addOutConditions(List outConditions) {
		this.outConditions.addAll(outConditions);
	}
	
	public boolean isFinishedParse() {
		return finishedParse;
	}

	public void setFinishedParse(boolean finishedParse) {
		this.finishedParse = finishedParse;
	}

	public WhereUnit() {
	}
	
	public WhereUnit(SQLBinaryOpExpr whereExpr) {
		this.whereExpr = whereExpr;
		this.canSplitExpr = whereExpr;
	}

	public SQLBinaryOpExpr getWhereExpr() {
		return whereExpr;
	}

	public void setWhereExpr(SQLBinaryOpExpr whereExpr) {
		this.whereExpr = whereExpr;
	}

	public SQLBinaryOpExpr getCanSplitExpr() {
		return canSplitExpr;
	}

	public void setCanSplitExpr(SQLBinaryOpExpr canSplitExpr) {
		this.canSplitExpr = canSplitExpr;
	}

	public List getSplitedExprList() {
		return splitedExprList;
	}

	public void addSplitedExpr(SQLExpr splitedExpr) {
		this.splitedExprList.add(splitedExpr);
	}

	public List> getConditionList() {
		return conditionList;
	}

	public void setConditionList(List> conditionList) {
		this.conditionList = conditionList;
	}
	
	public void addSubWhereUnit(WhereUnit whereUnit) {
		this.subWhereUnits.add(whereUnit);
	}
	
	public List getSubWhereUnit() {
		return this.subWhereUnits;
	}
}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/impl/DefaultDruidParser.java
================================================
package io.mycat.route.parser.druid.impl;

import java.sql.SQLNonTransientException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr;
import com.alibaba.druid.sql.ast.statement.SQLSelectQuery;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
import com.alibaba.druid.stat.TableStat.Condition;

import io.mycat.cache.LayerCachePool;
import io.mycat.config.model.SchemaConfig;
import io.mycat.route.RouteResultset;
import io.mycat.route.parser.druid.DruidParser;
import io.mycat.route.parser.druid.DruidShardingParseInfo;
import io.mycat.route.parser.druid.MycatSchemaStatVisitor;
import io.mycat.route.parser.druid.RouteCalculateUnit;
import io.mycat.route.parser.druid.SqlMethodInvocationHandler;
import io.mycat.route.parser.druid.SqlMethodInvocationHandlerFactory;
import io.mycat.sqlengine.mpp.RangeValue;
import io.mycat.util.StringUtil;

/**
 * 对SQLStatement解析
 * 主要通过visitor解析和statement解析：有些类型的SQLStatement通过visitor解析足够了，
 *  有些只能通过statement解析才能得到所有信息
 *  有些需要通过两种方式解析才能得到完整信息
 * @author wang.dw
 *
 */
public class DefaultDruidParser implements DruidParser {
	protected static final Logger LOGGER = LoggerFactory.getLogger(DefaultDruidParser.class);
	/**
	 * 解析得到的结果
	 */
	protected DruidShardingParseInfo ctx;
	
	private Map tableAliasMap = new HashMap();

	private List conditions = new ArrayList();

	protected SqlMethodInvocationHandler invocationHandler;
	
	public Map getTableAliasMap() {
		return tableAliasMap;
	}

	public List getConditions() {
		return conditions;
	}

	public DefaultDruidParser() {
		invocationHandler = SqlMethodInvocationHandlerFactory.getForMysql();
	}

	/**
	 * 使用MycatSchemaStatVisitor解析,得到tables、tableAliasMap、conditions等
	 * @param schema
	 * @param stmt
	 */
	public void parser(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt, String originSql,LayerCachePool cachePool,MycatSchemaStatVisitor schemaStatVisitor) throws SQLNonTransientException {
		ctx = new DruidShardingParseInfo();
		//设置为原始sql，如果有需要改写sql的，可以通过修改SQLStatement中的属性，然后调用SQLStatement.toString()得到改写的sql
		ctx.setSql(originSql);
		//通过visitor解析
		visitorParse(schema, rrs, stmt, schemaStatVisitor);

		//通过Statement解析
		statementParse(schema, rrs, stmt);
	}
	
	/**
	 * 是否终止解析,子类可覆盖此方法控制解析进程.
	 * 存在子查询的情况下,如果子查询需要先执行获取返回结果后,进一步改写sql后,再执行 在这种情况下,不再需要statement 和changeSql 解析。增加此模板方法
	 * @param schemaStatVisitor
	 * @return
	 */
	public boolean afterVisitorParser(RouteResultset rrs, SQLStatement stmt,MycatSchemaStatVisitor schemaStatVisitor){
		return false;
	}
	
	/**
	 * 子类可覆盖（如果visitorParse解析得不到表名、字段等信息的，就通过覆盖该方法来解析）
	 * 子类覆盖该方法一般是将SQLStatement转型后再解析（如转型为MySqlInsertStatement）
	 */
	@Override
	public void statementParse(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt) throws SQLNonTransientException {
		
	}
	
	/**
	 * 改写sql：如insert是
	 */
	@Override
	public void changeSql(SchemaConfig schema, RouteResultset rrs,
			SQLStatement stmt,LayerCachePool cachePool) throws SQLNonTransientException {
		
	}

	/**
	 * 子类可覆盖（如果该方法解析得不到表名、字段等信息的，就覆盖该方法，覆盖成空方法，然后通过statementPparse去解析）
	 * 通过visitor解析：有些类型的Statement通过visitor解析得不到表名、
	 * @param stmt
	 */
	@Override
	public void visitorParse(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt, MycatSchemaStatVisitor visitor)
			throws SQLNonTransientException {

		stmt.accept(visitor);
		ctx.setVisitor(visitor);

		if(stmt instanceof SQLSelectStatement){
			SQLSelectQuery query = ((SQLSelectStatement) stmt).getSelect().getQuery();
			if(query instanceof MySqlSelectQueryBlock){
				if(((MySqlSelectQueryBlock)query).isForUpdate()){
					rrs.setSelectForUpdate(true);
				}
			}
		}

		List> mergedConditionList = new ArrayList>();
		if(visitor.hasOrCondition()) {//包含or语句
			//TODO
			//根据or拆分
			mergedConditionList = visitor.splitConditions();
		} else {//不包含OR语句
			mergedConditionList.add(visitor.getConditions());
		}
		
		if(visitor.isHasChange()){	// 在解析的过程中子查询被改写了.需要更新ctx.
			ctx.setSql(stmt.toString());
			rrs.setStatement(ctx.getSql());
		}
		
		buildTableAliasMap(schema, visitor);
		ctx.setRouteCalculateUnits(this.buildRouteCalculateUnits(visitor, mergedConditionList));
	}

	private Map buildTableAliasMap(SchemaConfig schema, MycatSchemaStatVisitor visitor) {
		if (visitor.getAliasMap() == null) {
			return null;
		}
		Map tableAliasMap = new HashMap<>();
		for (Map.Entry entry : visitor.getAliasMap().entrySet()) {
			String key = entry.getKey();
			String value = entry.getValue();
			if (key != null) {
				int pos = key.indexOf(".");
				if (pos > 0) {
					key = key.substring(pos + 1);
				}
			}
			if (value != null) {
				int pos = value.indexOf(".");
				if (pos > 0) {
					value = value.substring(pos + 1);
				}

			}
			if (key != null && key.charAt(0) == '`') {
				key = key.substring(1, key.length() - 1);
			}
			if (value != null && value.charAt(0) == '`') {
				value = value.substring(1, value.length() - 1);
			}
			// remove database in database.table
			if (key != null) {
               int pos = key.indexOf(".");
					if(pos> 0) {
						String schemaInSQL = key.substring(0, pos);
						// 只有sql里面的schema名称和逻辑schema相同时，才去掉schema，防止"物理schema.物理表名"被错判为同名的逻辑表，产生路由判断错误
						if (schemaInSQL.equalsIgnoreCase(schema.getName())) {
							key = key.substring(pos + 1);
						}
					}
				String upperkey = key.toUpperCase();
				// String upperValue = value.toUpperCase();

				if (!tableAliasMap.containsKey(upperkey)) {
					tableAliasMap.put(upperkey, value);
				}

//				if (!ctx.getTables().contains(upperValue)) {
//					ctx.addTable(upperValue);
//				}
			}
		}


		// visitor.getAliasMap().putAll(tableAliasMap);
        ctx.addTables(visitor.getTables(), schema.getName());
		ctx.setTableAliasMap(visitor.getAliasMap());
		return tableAliasMap;
	}

	private List buildRouteCalculateUnits(MycatSchemaStatVisitor visitor,
			List> conditionList) {
		List retList = new ArrayList();

		//遍历condition ，找分片字段
		for(int i = 0; i < conditionList.size(); i++) {
			RouteCalculateUnit routeCalculateUnit = new RouteCalculateUnit();
			for(Condition condition : conditionList.get(i)) {
				List values = condition.getValues();
				if(values.size() == 0) {
					continue;  
				}
				if(checkConditionValues(values)) {
					String columnName = StringUtil.removeBackquote(condition.getColumn().getName().toUpperCase());
					String tableName = StringUtil.removeBackquote(condition.getColumn().getTable().toUpperCase());
					int index = 0;

						if(visitor.getAliasMap() != null && visitor.getAliasMap().get(tableName) != null
							&& !visitor.getAliasMap().get(tableName).equals(tableName)) {
						tableName = visitor.getAliasMap().get(tableName);
					}
					//处理schema.table的情况
					if ((index = tableName.indexOf(".")) != -1) {
						tableName = tableName.substring(index + 1);
					}
					tableName = tableName.toUpperCase();
//					//确保表名是大写
//					if(visitor.getAliasMap() != null && visitor.getAliasMap().get(tableName) == null) {//子查询的别名条件忽略掉,不参数路由计算，否则后面找不到表
//						continue;
//					}
					boolean noFindTable = true;
					if (visitor.getAliasMap() != null) {
						for (String value : visitor.getAliasMap().values()) {
							if (value.equalsIgnoreCase(tableName)) {
								noFindTable = false;
								break;
							}
						}
					}
					if (noFindTable) {// 子查询的别名条件忽略掉,不参数路由计算，否则后面找不到表
						continue;
					}
					
					String operator = condition.getOperator();
					
					//只处理between ,in和=3中操作符
					if(operator.equals("between")) {
						RangeValue rv = new RangeValue(values.get(0), values.get(1), RangeValue.EE);
								routeCalculateUnit.addShardingExpr(tableName.toUpperCase(), columnName, rv);
					} else if(operator.equals("=") || operator.toLowerCase().equals("in")){ //只处理=号和in操作符,其他忽略
								routeCalculateUnit.addShardingExpr(tableName.toUpperCase(), columnName, values.toArray());
					}
				}
			}
			retList.add(routeCalculateUnit);
		}
		return retList;
	}
	
	private boolean checkConditionValues(List values) {
		for(Object value : values) {
			if(value != null && !value.toString().equals("")) {
				return true;
			}
		}
		return false;
	}
	
	public DruidShardingParseInfo getCtx() {
		return ctx;
	}

	public void setInvocationHandler(SqlMethodInvocationHandler invocationHandler) {
		this.invocationHandler = invocationHandler;
	}

	/**
	 * 尝试解析某些SQL函数，如now(), sysdate()等
	 */
	protected String tryInvokeSQLMethod(SQLMethodInvokeExpr expr) throws SQLNonTransientException {
		return invocationHandler.invoke(expr);
	}
}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/impl/DruidAlterTableParser.java
================================================
package io.mycat.route.parser.druid.impl;

import java.sql.SQLNonTransientException;

import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.statement.SQLAlterTableStatement;

import io.mycat.config.model.SchemaConfig;
import io.mycat.route.RouteResultset;
import io.mycat.route.parser.druid.MycatSchemaStatVisitor;
import io.mycat.util.StringUtil;

/**
 * alter table 语句解析
 * @author wang.dw
 *
 */
public class DruidAlterTableParser extends DefaultDruidParser {
	@Override
	public void visitorParse(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt, MycatSchemaStatVisitor visitor)
			throws SQLNonTransientException {
		
	}
	@Override
	public void statementParse(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt) throws SQLNonTransientException {
        SQLAlterTableStatement alterTable = (SQLAlterTableStatement)stmt;
	String tableName = StringUtil.removeBackquote(alterTable.getTableSource().toString().toUpperCase());
//
	ctx.addTable(tableName);
		
	}

//    public static void main(String[] args)
//    {
//        String s="SELECT Customer,SUM(OrderPrice) FROM Orders\n" +
//                "GROUP BY Customer";
//        SQLStatementParser parser = new MySqlStatementParser(s);
//        SQLStatement statement = parser.parseStatement();
//        System.out.println();
//    }
}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/impl/DruidCreateTableParser.java
================================================
package io.mycat.route.parser.druid.impl;

import java.sql.SQLNonTransientException;

import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLCharExpr;
import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
import com.alibaba.druid.sql.ast.statement.SQLCharacterDataType;
import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition;
import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlCreateTableStatement;

import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.TableConfig;
import io.mycat.route.RouteResultset;
import io.mycat.route.function.AbstractPartitionAlgorithm;
import io.mycat.route.function.SlotFunction;
import io.mycat.route.parser.druid.MycatSchemaStatVisitor;
import io.mycat.util.StringUtil;


public class DruidCreateTableParser extends DefaultDruidParser {

	@Override
	public void visitorParse(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt,
			MycatSchemaStatVisitor visitor) {
	}
	
	@Override
	public void statementParse(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt) throws SQLNonTransientException {
		MySqlCreateTableStatement createStmt = (MySqlCreateTableStatement)stmt;
		if(createStmt.getQuery() != null) {
			String msg = "create table from other table not supported :" + stmt;
			LOGGER.warn(msg);
			throw new SQLNonTransientException(msg);
		}
		String tableName = StringUtil.removeBackquote(createStmt.getTableSource().toString().toUpperCase());
		if(schema.getTables().containsKey(tableName)) {
			TableConfig tableConfig = schema.getTables().get(tableName);
			AbstractPartitionAlgorithm algorithm = tableConfig.getRule().getRuleAlgorithm();
			if(algorithm instanceof SlotFunction){
				SQLColumnDefinition column = new SQLColumnDefinition();
				column.setDataType(new SQLCharacterDataType("int"));
				column.setName(new SQLIdentifierExpr("_slot"));
				column.setComment(new SQLCharExpr("自动迁移算法slot,禁止修改"));
				((SQLCreateTableStatement)stmt).getTableElementList().add(column);
				String sql = createStmt.toString();
				rrs.setStatement(sql);
				ctx.setSql(sql);
			}
		}
		ctx.addTable(tableName);
		
	}
}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/impl/DruidDeleteParser.java
================================================
package io.mycat.route.parser.druid.impl;

import java.sql.SQLNonTransientException;

import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlDeleteStatement;

import io.mycat.MycatServer;
import io.mycat.cache.CachePool;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.TableConfig;
import io.mycat.route.RouteResultset;
import io.mycat.util.StringUtil;

public class DruidDeleteParser extends DefaultDruidParser {
	@Override
	public void statementParse(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt) throws SQLNonTransientException {
		MySqlDeleteStatement delete = (MySqlDeleteStatement)stmt;
		String tableName = StringUtil.removeBackquote(delete.getTableName().getSimpleName().toUpperCase());
		if (!ctx.getTables().contains(tableName)) {
			ctx.addTable(tableName);
		}

		//在解析SQL时清空该表的主键缓存
		TableConfig tableConfig = schema.getTables().get(tableName);
		if (tableConfig != null && !tableConfig.primaryKeyIsPartionKey()) {
			String cacheName = schema.getName() + "_" + tableName;
			cacheName = cacheName.toUpperCase();
			for (CachePool value : MycatServer.getInstance().getCacheService().getAllCachePools().values()) {
				value.clearCache(cacheName);
				value.getCacheStatic().reset();
			}
		}
	}
}



================================================
FILE: src/main/java/io/mycat/route/parser/druid/impl/DruidInsertParser.java
================================================
package io.mycat.route.parser.druid.impl;

import java.sql.SQLNonTransientException;
import java.sql.SQLSyntaxErrorException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr;
import com.alibaba.druid.sql.ast.expr.SQLCharExpr;
import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr;
import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr;
import com.alibaba.druid.sql.ast.statement.SQLExprTableSource;
import com.alibaba.druid.sql.ast.statement.SQLInsertStatement;
import com.alibaba.druid.sql.ast.statement.SQLInsertStatement.ValuesClause;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlInsertStatement;

import io.mycat.backend.mysql.nio.handler.FetchStoreNodeOfChildTableHandler;
import io.mycat.backend.mysql.nio.handler.JDBCFetchStoreNodeOfChildTableHandler;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.TableConfig;
import io.mycat.route.RouteResultset;
import io.mycat.route.RouteResultsetNode;
import io.mycat.route.function.AbstractPartitionAlgorithm;
import io.mycat.route.function.SlotFunction;
import io.mycat.route.parser.druid.MycatSchemaStatVisitor;
import io.mycat.route.parser.druid.RouteCalculateUnit;
import io.mycat.route.parser.util.ParseUtil;
import io.mycat.route.util.RouterUtil;
import io.mycat.server.parser.ServerParse;
import io.mycat.util.StringUtil;

public class DruidInsertParser extends DefaultDruidParser {
	@Override
	public void visitorParse(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt, MycatSchemaStatVisitor visitor)
			throws SQLNonTransientException {
		
	}
	
	/**
	 * 考虑因素：isChildTable、批量、是否分片
	 */
	@Override
	public void statementParse(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt) throws SQLNonTransientException {
		MySqlInsertStatement insert = (MySqlInsertStatement)stmt;
		String tableName = StringUtil.removeBackquote(insert.getTableName().getSimpleName()).toUpperCase();

		ctx.addTable(tableName);
		if(RouterUtil.isNoSharding(schema,tableName)) {//整个schema都不分库或者该表不拆分
			RouterUtil.routeForTableMeta(rrs, schema, tableName, rrs.getStatement());
			rrs.setFinishedRoute(true);
			return;
		}

		TableConfig tc = schema.getTables().get(tableName);
		if(tc == null) {
			String msg = "can't find table define in schema "
					+ tableName + " schema:" + schema.getName();
			LOGGER.warn(msg);
			throw new SQLNonTransientException(msg);
		} else {
			//childTable的insert直接在解析过程中完成路由
			if (tc.isChildTable()) {
				parserChildTable(schema, rrs, tableName, insert);
				return;
			}
			
			String partitionColumn = tc.getPartitionColumn();
			
			if(partitionColumn != null) {//分片表
				//拆分表必须给出column list,否则无法寻找分片字段的值
				if(insert.getColumns() == null || insert.getColumns().size() == 0) {
					throw new SQLSyntaxErrorException("partition table, insert must provide ColumnList");
				}
				
				//批量insert
				if(isMultiInsert(insert)) {
//					String msg = "multi insert not provided" ;
//					LOGGER.warn(msg);
//					throw new SQLNonTransientException(msg);
					parserBatchInsert(schema, rrs, partitionColumn, tableName, insert);
				} else {
					parserSingleInsert(schema, rrs, partitionColumn, tableName, insert);
				}
				
			}
		}
	}
	
	/**
	 * 寻找joinKey的索引
	 * @param columns
	 * @param joinKey
	 * @return -1表示没找到，>=0表示找到了
	 */
	private int getJoinKeyIndex(List columns, String joinKey) {
		for(int i = 0; i < columns.size(); i++) {
			String col = StringUtil.removeBackquote(columns.get(i).toString()).toUpperCase();
			if(col.equals(joinKey)) {
				return i;
			}
		}
		return -1;
	}
	
	/**
	 * 是否为批量插入：insert into ...values (),()...或 insert into ...select.....
	 * @param insertStmt
	 * @return
	 */
	private boolean isMultiInsert(MySqlInsertStatement insertStmt) {
		return (insertStmt.getValuesList() != null && insertStmt.getValuesList().size() > 1) || insertStmt.getQuery() != null;
	}
	
	private RouteResultset parserChildTable(SchemaConfig schema, RouteResultset rrs,
			String tableName, MySqlInsertStatement insertStmt) throws SQLNonTransientException {
		TableConfig tc = schema.getTables().get(tableName);
		
		String joinKey = tc.getJoinKey();
		int joinKeyIndex = getJoinKeyIndex(insertStmt.getColumns(), joinKey);
		if(joinKeyIndex == -1) {
			String inf = "joinKey not provided :" + tc.getJoinKey()+ "," + insertStmt;
			LOGGER.warn(inf);
			throw new SQLNonTransientException(inf);
		}
		if(isMultiInsert(insertStmt)) {
			String msg = "ChildTable multi insert not provided" ;
			LOGGER.warn(msg);
			throw new SQLNonTransientException(msg);
		}
		
		String joinKeyVal = insertStmt.getValues().getValues().get(joinKeyIndex).toString();

		
		String sql = insertStmt.toString();
		
		// try to route by ER parent partion key
		RouteResultset theRrs = RouterUtil.routeByERParentKey(null,schema, ServerParse.INSERT,sql, rrs, tc,joinKeyVal);
		if (theRrs != null) {
			rrs.setFinishedRoute(true);
			return theRrs;
		}

		// route by sql query root parent's datanode
		String findRootTBSql = tc.getLocateRTableKeySql().toLowerCase() + joinKeyVal;
		if (LOGGER.isDebugEnabled()) {
			LOGGER.debug("find root parent's node sql "+ findRootTBSql);
		}

		String dn = null;
		if (tc.getRootParent().getFetchStoreNodeByJdbc()) {
			JDBCFetchStoreNodeOfChildTableHandler jdbcFetchHandler = new JDBCFetchStoreNodeOfChildTableHandler();
			dn = jdbcFetchHandler.execute(schema.getName(),findRootTBSql, tc.getRootParent().getDataNodes());
		} else {
			FetchStoreNodeOfChildTableHandler FetchHandler = new FetchStoreNodeOfChildTableHandler();
			FetchHandler.execute(schema.getName(),findRootTBSql, tc.getRootParent().getDataNodes());
		}

		if (dn == null) {
			throw new SQLNonTransientException("can't find (root) parent sharding node for sql:"+ sql);
		}
		if (LOGGER.isDebugEnabled()) {
			LOGGER.debug("found partion node for child table to insert "+ dn + " sql :" + sql);
		}
		return RouterUtil.routeToSingleNode(rrs, dn, sql);
	}
	
	/**
	 * 单条insert（非批量）
	 * @param schema
	 * @param rrs
	 * @param partitionColumn
	 * @param tableName
	 * @param insertStmt
	 * @throws SQLNonTransientException
	 */
	private void parserSingleInsert(SchemaConfig schema, RouteResultset rrs, String partitionColumn,
			String tableName, MySqlInsertStatement insertStmt) throws SQLNonTransientException {
		boolean isFound = false;
		for(int i = 0; i < insertStmt.getColumns().size(); i++) {
			if(partitionColumn.equalsIgnoreCase(StringUtil.removeBackquote(insertStmt.getColumns().get(i).toString()))) {//找到分片字段
				isFound = true;
				String column = StringUtil.removeBackquote(insertStmt.getColumns().get(i).toString());

				String shardingValue = StringUtil.removeBackquote(getShardingValue(insertStmt.getValues().getValues().get(i)));
				insertStmt.getValues().getValues().set(i,new SQLCharExpr(shardingValue));
				ctx.setSql(insertStmt.toString());

				RouteCalculateUnit routeCalculateUnit = new RouteCalculateUnit();
				routeCalculateUnit.addShardingExpr(tableName, column, shardingValue);
				ctx.addRouteCalculateUnit(routeCalculateUnit);
				//mycat是单分片键，找到了就返回
				break;
			}
		}
		if(!isFound) {//分片表的
			String msg = "bad insert sql (sharding column:"+ partitionColumn + " not provided," + insertStmt;
			LOGGER.warn(msg);
			throw new SQLNonTransientException(msg);
		}
		// insert into .... on duplicateKey 
		//such as :INSERT INTO TABLEName (a,b,c) VALUES (1,2,3) ON DUPLICATE KEY UPDATE b=VALUES(b); 
		//INSERT INTO TABLEName (a,b,c) VALUES (1,2,3) ON DUPLICATE KEY UPDATE c=c+1;
		if(insertStmt.getDuplicateKeyUpdate() != null) {
			List updateList = insertStmt.getDuplicateKeyUpdate();
			for(SQLExpr expr : updateList) {
				SQLBinaryOpExpr opExpr = (SQLBinaryOpExpr)expr;
				String column = StringUtil.removeBackquote(opExpr.getLeft().toString().toUpperCase());
				if(column.equals(partitionColumn)) {
					String msg = "Sharding column can't be updated: " + tableName + " -> " + partitionColumn;
					LOGGER.warn(msg);
					throw new SQLNonTransientException(msg);
				}
			}
		}
	}
	
	/**
	 * insert into .... select .... 或insert into table() values (),(),....
	 * @param schema
	 * @param rrs
	 * @param insertStmt
	 * @throws SQLNonTransientException
	 */
	private void parserBatchInsert(SchemaConfig schema, RouteResultset rrs, String partitionColumn, 
			String tableName, MySqlInsertStatement insertStmt) throws SQLNonTransientException {
		//insert into table() values (),(),....
		if(insertStmt.getValuesList().size() > 1) {
			//字段列数
			int columnNum = insertStmt.getColumns().size();
			int shardingColIndex = getShardingColIndex(insertStmt, partitionColumn);
			if(shardingColIndex == -1) {
				String msg = "bad insert sql (sharding column:"+ partitionColumn + " not provided," + insertStmt;
				LOGGER.warn(msg);
				throw new SQLNonTransientException(msg);
			} else {
				List valueClauseList = insertStmt.getValuesList();
				
				Map> nodeValuesMap = new HashMap>();
				Map slotsMap = new HashMap<>();
				TableConfig tableConfig = schema.getTables().get(tableName);
				AbstractPartitionAlgorithm algorithm = tableConfig.getRule().getRuleAlgorithm();
				for(ValuesClause valueClause : valueClauseList) {
					if(valueClause.getValues().size() != columnNum) {
						String msg = "bad insert sql columnSize != valueSize:"
					             + columnNum + " != " + valueClause.getValues().size() 
					             + "values:" + valueClause;
						LOGGER.warn(msg);
						throw new SQLNonTransientException(msg);
					}
					SQLExpr expr = valueClause.getValues().get(shardingColIndex);
					String shardingValue = StringUtil.removeBackquote(getShardingValue(expr));
					valueClause.getValues().set(shardingColIndex, new SQLCharExpr(shardingValue));

					Integer nodeIndex = algorithm.calculate(StringUtil.removeBackquote(shardingValue));
					if(algorithm instanceof SlotFunction){
						slotsMap.put(nodeIndex,((SlotFunction) algorithm).slotValue()) ;
					}
					//没找到插入的分片
					if(nodeIndex == null) {
						String msg = "can't find any valid datanode :" + tableName 
								+ " -> " + partitionColumn + " -> " + shardingValue;
						LOGGER.warn(msg);
						throw new SQLNonTransientException(msg);
					}
					if(nodeValuesMap.get(nodeIndex) == null) {
						nodeValuesMap.put(nodeIndex, new ArrayList());
					}
					nodeValuesMap.get(nodeIndex).add(valueClause);
				}


				RouteResultsetNode[] nodes = new RouteResultsetNode[nodeValuesMap.size()];
				int count = 0;
				for(Map.Entry> node : nodeValuesMap.entrySet()) {
					Integer nodeIndex = node.getKey();
					List valuesList = node.getValue();
					insertStmt.getValuesList().clear();
					insertStmt.getValuesList().addAll(valuesList);
					// insertStmt.setValuesList(valuesList);
					if(tableConfig.isDistTable()) {
						nodes[count] = new RouteResultsetNode(tableConfig.getDataNodes().get(0),
								rrs.getSqlType(),insertStmt.toString());
						if(tableConfig.getDistTables()==null){
							String msg = " sub table not exists for " + nodes[count].getName() + " on " + tableName;
							LOGGER.error("DruidMycatRouteStrategyError " + msg);
							throw new SQLSyntaxErrorException(msg);
						}
						String subTableName = tableConfig.getDistTables().get(nodeIndex);
						
						nodes[count].setSubTableName(subTableName);
						SQLInsertStatement insertStatement = (SQLInsertStatement) insertStmt;
						SQLExprTableSource tableSource = insertStatement.getTableSource();
						//getDisTable 修改表名称
						SQLIdentifierExpr sqlIdentifierExpr = new SQLIdentifierExpr();
						sqlIdentifierExpr.setParent(tableSource.getParent());
						sqlIdentifierExpr.setName(subTableName);
						SQLExprTableSource from2 = new SQLExprTableSource(sqlIdentifierExpr);
						insertStatement.setTableSource(from2);
						nodes[count].setStatement(insertStatement.toString());
					} else {
						nodes[count] = new RouteResultsetNode(tableConfig.getDataNodes().get(nodeIndex),
								rrs.getSqlType(),insertStmt.toString());
					}
					
					if(algorithm instanceof SlotFunction) {
						nodes[count].setSlot(slotsMap.get(nodeIndex));
						nodes[count].setStatement(ParseUtil.changeInsertAddSlot(nodes[count].getStatement(),nodes[count].getSlot()));
					}
					nodes[count++].setSource(rrs);

				}
				rrs.setNodes(nodes);
				rrs.setFinishedRoute(true);

			}
		} else if(insertStmt.getQuery() != null) { // insert into .... select ....
			String msg = "TODO:insert into .... select .... not supported!";
			LOGGER.warn(msg);
			throw new SQLNonTransientException(msg);
		}
	}

	private String getShardingValue(SQLExpr expr) throws SQLNonTransientException {
		String shardingValue = null;
		if(expr instanceof SQLIntegerExpr) {
			SQLIntegerExpr intExpr = (SQLIntegerExpr)expr;
			shardingValue = intExpr.getNumber() + "";
		} else if (expr instanceof SQLCharExpr) {
			SQLCharExpr charExpr = (SQLCharExpr)expr;
			shardingValue = charExpr.getText();
		} else if (expr instanceof SQLMethodInvokeExpr) {
			SQLMethodInvokeExpr methodInvokeExpr = (SQLMethodInvokeExpr)expr;
			try {
				shardingValue = tryInvokeSQLMethod(methodInvokeExpr);
			}catch (Exception e){
				LOGGER.error("",e);
			}
			if (shardingValue == null){
				shardingValue = expr.toString();
			}
		} else {
			shardingValue = expr.toString();
		}
		return shardingValue;
	}
	/**
	 * 寻找拆分字段在 columnList中的索引
	 * @param insertStmt
	 * @param partitionColumn
	 * @return
	 */
	private int getShardingColIndex(MySqlInsertStatement insertStmt,String partitionColumn) {
		int shardingColIndex = -1;
		for(int i = 0; i < insertStmt.getColumns().size(); i++) {
			if(partitionColumn.equalsIgnoreCase(StringUtil.removeBackquote(insertStmt.getColumns().get(i).toString()))) {//找到分片字段
				shardingColIndex = i;
				return shardingColIndex;
			}
		}
		return shardingColIndex;
	}
}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/impl/DruidLockTableParser.java
================================================
package io.mycat.route.parser.druid.impl;

import java.sql.SQLNonTransientException;
import java.util.List;

import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlLockTableStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlLockTableStatement.LockType;

import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.TableConfig;
import io.mycat.route.RouteResultset;
import io.mycat.route.RouteResultsetNode;
import io.mycat.route.parser.druid.DruidParser;
import io.mycat.route.parser.druid.MycatSchemaStatVisitor;
import io.mycat.server.parser.ServerParse;
import io.mycat.util.SplitUtil;

/**
 * lock tables [table] [write|read]语句解析器
 * @author songdabin
 */
public class DruidLockTableParser extends DefaultDruidParser implements DruidParser {
	@Override
	public void statementParse(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt)
			throws SQLNonTransientException {
		MySqlLockTableStatement lockTableStat = (MySqlLockTableStatement)stmt;
		String table = lockTableStat.getItems().get(0).getTableSource().toString().toUpperCase();
		TableConfig tableConfig = schema.getTables().get(table);
		if (tableConfig == null) {
			String msg = "can't find table define of " + table + " in schema:" + schema.getName();
			LOGGER.warn(msg);
			throw new SQLNonTransientException(msg);
		}
		LockType lockType = lockTableStat.getItems().get(0).getLockType();
		if (LockType.WRITE != lockType && LockType.READ != lockType) {
			String msg = "lock type must be write or read";
			LOGGER.warn(msg);
			throw new SQLNonTransientException(msg);
		}
		List dataNodes = tableConfig.getDataNodes();
		RouteResultsetNode[] nodes = new RouteResultsetNode[dataNodes.size()];
		for (int i = 0; i < dataNodes.size(); i ++) {
			nodes[i] = new RouteResultsetNode(dataNodes.get(i), ServerParse.LOCK, stmt.toString());
		}
		rrs.setNodes(nodes);
		rrs.setFinishedRoute(true);
	}
	
	@Override
	public void visitorParse(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt, MycatSchemaStatVisitor visitor)
			throws SQLNonTransientException {
		// 对于lock tables table1 write, table2 read类型的多表锁语句，DruidParser只能解析出table1，
		// 由于多表锁在分布式场景处理逻辑繁琐，且应用场景较少，因此在此处对这种锁表语句进行拦截。
		// 多表锁的语句在语义上会有","，这里以此为判断依据
		String sql = rrs.getStatement();
		sql = sql.replaceAll("\n", " ").replaceAll("\t", " ");
		String[] stmts = SplitUtil.split(sql, ',', true);
		// 如果命令中存在","，则按多表锁的语句来处理
		if (stmts.length > 1) {
			String tmpStmt = null;
			String tmpWords[] = null;
			for (int i = 1; i < stmts.length; i ++) {
				tmpStmt = stmts[i];
				tmpWords = SplitUtil.split(tmpStmt, ' ', true);
				if (tmpWords.length==2 && ("READ".equalsIgnoreCase(tmpWords[1]) || "WRITE".equalsIgnoreCase(tmpWords[1]))) {
					// 如果符合多表锁的语法，则继续，并在最后提示不能多表锁！
					continue;
				} else {
					// 如果不符合多表锁的语法，则提示语法错误和不能多表锁！
					throw new SQLNonTransientException("You have an error in your SQL syntax, don't support lock multi tables!");
				}
			}
			LOGGER.error("can't lock multi-table");
			throw new SQLNonTransientException("can't lock multi-table");
		}
	}
}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/impl/DruidSelectDb2Parser.java
================================================
package io.mycat.route.parser.druid.impl;

import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLOrderBy;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLAggregateExpr;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator;
import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.ast.statement.SQLSelectItem;
import com.alibaba.druid.sql.ast.statement.SQLSelectQuery;
import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource;
import com.alibaba.druid.sql.ast.statement.SQLTableSource;
import com.alibaba.druid.sql.dialect.oracle.ast.stmt.OracleSelectQueryBlock;
import com.alibaba.druid.util.JdbcConstants;

import io.mycat.config.model.SchemaConfig;
import io.mycat.route.RouteResultset;


/**
 * 由于druid的db2解析部分不够完整，且使用oracle的解析基本能满足需求
 * 所以基于oracle的扩展
 */
public class DruidSelectDb2Parser extends DruidSelectOracleParser
{

    protected void parseNativePageSql(SQLStatement stmt, RouteResultset rrs, OracleSelectQueryBlock mysqlSelectQuery, SchemaConfig schema)
    {
        //第一层子查询
        SQLExpr where=  mysqlSelectQuery.getWhere();
        SQLTableSource from= mysqlSelectQuery.getFrom();
        if(where instanceof SQLBinaryOpExpr &&from instanceof SQLSubqueryTableSource)
        {

            SQLBinaryOpExpr one= (SQLBinaryOpExpr) where;
            SQLExpr left=one.getLeft();
            SQLBinaryOperator operator =one.getOperator();

                    SQLSelectQuery subSelect = ((SQLSubqueryTableSource) from).getSelect().getQuery();
                    SQLOrderBy orderBy=null;
                    if (subSelect instanceof OracleSelectQueryBlock)
                    {
                        boolean hasRowNumber=false;
                        OracleSelectQueryBlock subSelectOracle = (OracleSelectQueryBlock) subSelect;
                        List sqlSelectItems=    subSelectOracle.getSelectList();
                        for (SQLSelectItem sqlSelectItem : sqlSelectItems)
                        {
                            SQLExpr sqlExpr=  sqlSelectItem.getExpr()   ;
                            if(sqlExpr instanceof  SQLAggregateExpr )
                            {
                                SQLAggregateExpr agg= (SQLAggregateExpr) sqlExpr;
                                if("row_number".equalsIgnoreCase(agg.getMethodName())&&agg.getOver()!=null)
                                {
                                    hasRowNumber=true;
                                    orderBy= agg.getOver().getOrderBy();
                                }

                            }
                        }

                        if(hasRowNumber)
                        {
                            if((operator==SQLBinaryOperator.LessThan||operator==SQLBinaryOperator.LessThanOrEqual) && one.getRight() instanceof SQLIntegerExpr )
                            {
                                SQLIntegerExpr right = (SQLIntegerExpr) one.getRight();
                                int firstrownum = right.getNumber().intValue();
                                if (operator == SQLBinaryOperator.LessThan&&firstrownum!=0) {
                                    firstrownum = firstrownum - 1;
                                }
                                if (subSelect instanceof OracleSelectQueryBlock)
                                {
                                    rrs.setLimitStart(0);
                                    rrs.setLimitSize(firstrownum);
                                    mysqlSelectQuery = (OracleSelectQueryBlock) subSelect;    //为了继续解出order by 等
                                    if(orderBy!=null)
                                    {
										SQLSelect oracleSelect = (SQLSelect) subSelect.getParent();
                                        oracleSelect.setOrderBy(orderBy);
                                    }
                                    parseOrderAggGroupOracle(stmt,rrs, mysqlSelectQuery, schema);
                                    isNeedParseOrderAgg=false;
                                }
                            }
                            else
                            if(operator==SQLBinaryOperator.BooleanAnd && left instanceof SQLBinaryOpExpr&&one.getRight() instanceof SQLBinaryOpExpr )
                            {
                                SQLBinaryOpExpr leftE= (SQLBinaryOpExpr) left;
                                SQLBinaryOpExpr rightE= (SQLBinaryOpExpr) one.getRight();
                                SQLBinaryOpExpr small=null ;
                                SQLBinaryOpExpr larger=null ;
                                int firstrownum =0;
                                int lastrownum =0;
                                if(leftE.getRight() instanceof SQLIntegerExpr&&(leftE.getOperator()==SQLBinaryOperator.GreaterThan||leftE.getOperator()==SQLBinaryOperator.GreaterThanOrEqual))
                                {
                                    small=leftE;
                                    firstrownum=((SQLIntegerExpr) leftE.getRight()).getNumber().intValue();
                                    if(leftE.getOperator()==SQLBinaryOperator.GreaterThanOrEqual &&firstrownum!=0) {
                                        firstrownum = firstrownum - 1;
                                    }
                                } else
                                if(leftE.getRight() instanceof SQLIntegerExpr&&(leftE.getOperator()==SQLBinaryOperator.LessThan||leftE.getOperator()==SQLBinaryOperator.LessThanOrEqual))
                                {
                                    larger=leftE;
                                    lastrownum=((SQLIntegerExpr) leftE.getRight()).getNumber().intValue();
                                    if(leftE.getOperator()==SQLBinaryOperator.LessThan&&lastrownum!=0) {
                                        lastrownum = lastrownum - 1;
                                    }
                                }

                                if(rightE.getRight() instanceof SQLIntegerExpr&&(rightE.getOperator()==SQLBinaryOperator.GreaterThan||rightE.getOperator()==SQLBinaryOperator.GreaterThanOrEqual))
                                {
                                    small=rightE;
                                    firstrownum=((SQLIntegerExpr) rightE.getRight()).getNumber().intValue();
                                    if(rightE.getOperator()==SQLBinaryOperator.GreaterThanOrEqual&&firstrownum!=0) {
                                        firstrownum = firstrownum - 1;
                                    }
                                } else
                                if(rightE.getRight() instanceof SQLIntegerExpr&&(rightE.getOperator()==SQLBinaryOperator.LessThan||rightE.getOperator()==SQLBinaryOperator.LessThanOrEqual))
                                {
                                    larger=rightE;
                                    lastrownum=((SQLIntegerExpr) rightE.getRight()).getNumber().intValue();
                                    if(rightE.getOperator()==SQLBinaryOperator.LessThan&&lastrownum!=0) {
                                        lastrownum = lastrownum - 1;
                                    }
                                }
                                if(small!=null&&larger!=null)
                                {
                                    setLimitIFChange(stmt, rrs, schema, small, firstrownum, lastrownum);
									if (orderBy != null) {
										SQLSelect oracleSelect = (SQLSelect) subSelect.getParent();
										oracleSelect.setOrderBy(orderBy);
									}
                                    parseOrderAggGroupOracle(stmt,rrs, (OracleSelectQueryBlock) subSelect, schema);
                                    isNeedParseOrderAgg=false;
                                }

                            }


                        } else
                        {
                            parseNativeSql(stmt,rrs,mysqlSelectQuery,schema);
                        }
                    }
        }
        else
        {
            parseNativeSql(stmt,rrs,mysqlSelectQuery,schema);
        }
        if(isNeedParseOrderAgg)
        {
            parseOrderAggGroupOracle(stmt,rrs,  mysqlSelectQuery, schema);
        }
    }


    private static final  Pattern pattern = Pattern.compile("FETCH(?:\\s)+FIRST(?:\\s)+(\\d+)(?:\\s)+ROWS(?:\\s)+ONLY",Pattern.CASE_INSENSITIVE);
    protected void parseNativeSql(SQLStatement stmt,RouteResultset rrs, OracleSelectQueryBlock mysqlSelectQuery,SchemaConfig schema)
    {

        Matcher matcher = pattern.matcher(getCtx().getSql());
        while (matcher.find())
        {

          String  row=    matcher.group(1);
            rrs.setLimitStart(0);
            rrs.setLimitSize(Integer.parseInt(row));
        }
    }

    protected String getCurentDbType()
    {
		return JdbcConstants.DB2.name();
    }




}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/impl/DruidSelectOracleParser.java
================================================
package io.mycat.route.parser.druid.impl;

import java.util.List;
import java.util.Map;

import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLLimit;
import com.alibaba.druid.sql.ast.SQLOrderBy;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLAggregateExpr;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator;
import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.ast.statement.SQLSelectItem;
import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem;
import com.alibaba.druid.sql.ast.statement.SQLSelectQuery;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource;
import com.alibaba.druid.sql.ast.statement.SQLTableSource;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
import com.alibaba.druid.sql.dialect.oracle.ast.stmt.OracleSelectQueryBlock;
import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser;
import com.alibaba.druid.util.JdbcConstants;

import io.mycat.config.model.SchemaConfig;
import io.mycat.route.RouteResultset;

public class DruidSelectOracleParser extends DruidSelectParser {

    @Override
	public void statementParse(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt) {
		SQLSelectStatement selectStmt = (SQLSelectStatement)stmt;
		SQLSelectQuery sqlSelectQuery = selectStmt.getSelect().getQuery();
       //从mysql解析过来
		if(sqlSelectQuery instanceof MySqlSelectQueryBlock) {
			MySqlSelectQueryBlock mysqlSelectQuery = (MySqlSelectQueryBlock)selectStmt.getSelect().getQuery();
			SQLLimit limit = mysqlSelectQuery.getLimit();
			if(limit==null)
			{
				  //使用oracle的解析，否则会有部分oracle语法识别错误
				  OracleStatementParser oracleParser = new OracleStatementParser(getCtx().getSql());
				  SQLSelectStatement oracleStmt = (SQLSelectStatement) oracleParser.parseStatement();
                selectStmt= oracleStmt;
				  SQLSelectQuery oracleSqlSelectQuery = oracleStmt.getSelect().getQuery();
				  if(oracleSqlSelectQuery instanceof OracleSelectQueryBlock)
				  {
					  parseNativePageSql(oracleStmt, rrs, (OracleSelectQueryBlock) oracleSqlSelectQuery, schema);
				  }



			  }
			if(isNeedParseOrderAgg)
			{
				parseOrderAggGroupMysql(schema, selectStmt,rrs, mysqlSelectQuery);
				//更改canRunInReadDB属性
				if ((mysqlSelectQuery.isForUpdate() || mysqlSelectQuery.isLockInShareMode()) && rrs.isAutocommit() == false)
				{
					rrs.setCanRunInReadDB(false);
				}
			}

		}


	}


	protected void parseOrderAggGroupOracle(SQLStatement stmt, RouteResultset rrs, OracleSelectQueryBlock mysqlSelectQuery, SchemaConfig schema)
	{
		Map aliaColumns = parseAggGroupCommon(schema, stmt,rrs, mysqlSelectQuery);

		SQLSelect oracleSelect = (SQLSelect) mysqlSelectQuery.getParent();
		if(oracleSelect.getOrderBy() != null) {
			List orderByItems = oracleSelect.getOrderBy().getItems();
			rrs.setOrderByCols(buildOrderByCols(orderByItems,aliaColumns));
		}
        isNeedParseOrderAgg=false;
	}


	protected void parseNativePageSql(SQLStatement stmt, RouteResultset rrs, OracleSelectQueryBlock mysqlSelectQuery, SchemaConfig schema)
	{
		//第一层子查询
		SQLExpr where=  mysqlSelectQuery.getWhere();
		SQLTableSource from= mysqlSelectQuery.getFrom();
		if(where instanceof SQLBinaryOpExpr &&from instanceof SQLSubqueryTableSource)
        {

            SQLBinaryOpExpr one= (SQLBinaryOpExpr) where;
            SQLExpr left=one.getLeft();
            SQLBinaryOperator operator =one.getOperator();

              //解析只有一层rownum限制大小
			if(one.getRight() instanceof SQLIntegerExpr &&"rownum".equalsIgnoreCase(left.toString())
					&&(operator==SQLBinaryOperator.LessThanOrEqual||operator==SQLBinaryOperator.LessThan))
			{
				SQLIntegerExpr right = (SQLIntegerExpr) one.getRight();
				int firstrownum = right.getNumber().intValue();
				if (operator == SQLBinaryOperator.LessThan&&firstrownum!=0) {
					firstrownum = firstrownum - 1;
				}
				SQLSelectQuery subSelect = ((SQLSubqueryTableSource) from).getSelect().getQuery();
				if (subSelect instanceof OracleSelectQueryBlock)
				{
					rrs.setLimitStart(0);
					rrs.setLimitSize(firstrownum);
					mysqlSelectQuery = (OracleSelectQueryBlock) subSelect;    //为了继续解出order by 等
					parseOrderAggGroupOracle(stmt,rrs, mysqlSelectQuery, schema);
					isNeedParseOrderAgg=false;
				}
			}
			else //解析oracle三层嵌套分页
            if(one.getRight() instanceof SQLIntegerExpr &&!"rownum".equalsIgnoreCase(left.toString())
                    &&(operator==SQLBinaryOperator.GreaterThan||operator==SQLBinaryOperator.GreaterThanOrEqual))
           {
			   parseThreeLevelPageSql(stmt, rrs, schema, (SQLSubqueryTableSource) from, one, operator);
			   }
            else //解析oracle rownumber over分页
			{

                    SQLSelectQuery subSelect = ((SQLSubqueryTableSource) from).getSelect().getQuery();
					SQLOrderBy orderBy=null;
                    if (subSelect instanceof OracleSelectQueryBlock)
                    {
                        boolean hasRowNumber=false;
                        OracleSelectQueryBlock subSelectOracle = (OracleSelectQueryBlock) subSelect;
                        List sqlSelectItems=    subSelectOracle.getSelectList();
                        for (SQLSelectItem sqlSelectItem : sqlSelectItems)
                        {
                            SQLExpr sqlExpr=  sqlSelectItem.getExpr()   ;
                            if(sqlExpr instanceof  SQLAggregateExpr )
                            {
                                SQLAggregateExpr agg= (SQLAggregateExpr) sqlExpr;
                                if("row_number".equalsIgnoreCase(agg.getMethodName())&&agg.getOver()!=null)
                                {
                                    hasRowNumber=true;
                                     orderBy= agg.getOver().getOrderBy();
                                }

                            }
                        }

                        if(hasRowNumber)
                        {
							if((operator==SQLBinaryOperator.LessThan||operator==SQLBinaryOperator.LessThanOrEqual) && one.getRight() instanceof SQLIntegerExpr )
							{
								SQLIntegerExpr right = (SQLIntegerExpr) one.getRight();
								int firstrownum = right.getNumber().intValue();
								if (operator == SQLBinaryOperator.LessThan&&firstrownum!=0) {
									firstrownum = firstrownum - 1;
								}
								if (subSelect instanceof OracleSelectQueryBlock)
								{
									rrs.setLimitStart(0);
									rrs.setLimitSize(firstrownum);
									mysqlSelectQuery = (OracleSelectQueryBlock) subSelect;
									if(orderBy!=null)
									{
										SQLSelect oracleSelect = (SQLSelect) subSelect.getParent();
										oracleSelect.setOrderBy(orderBy);
									}
									parseOrderAggGroupOracle(stmt,rrs, mysqlSelectQuery, schema);
									isNeedParseOrderAgg=false;
								}
							}
							  else
							if(operator==SQLBinaryOperator.BooleanAnd && left instanceof SQLBinaryOpExpr&&one.getRight() instanceof SQLBinaryOpExpr )
							{
							SQLBinaryOpExpr leftE= (SQLBinaryOpExpr) left;
							SQLBinaryOpExpr rightE= (SQLBinaryOpExpr) one.getRight();
							SQLBinaryOpExpr small=null ;
							SQLBinaryOpExpr larger=null ;
							int firstrownum =0;
							int lastrownum =0;
							if(leftE.getRight() instanceof SQLIntegerExpr&&(leftE.getOperator()==SQLBinaryOperator.GreaterThan||leftE.getOperator()==SQLBinaryOperator.GreaterThanOrEqual))
							{
								small=leftE;
								firstrownum=((SQLIntegerExpr) leftE.getRight()).getNumber().intValue();
								if(leftE.getOperator()==SQLBinaryOperator.GreaterThanOrEqual &&firstrownum!=0) {
									firstrownum = firstrownum - 1;
								}
							} else
							if(leftE.getRight() instanceof SQLIntegerExpr&&(leftE.getOperator()==SQLBinaryOperator.LessThan||leftE.getOperator()==SQLBinaryOperator.LessThanOrEqual))
							{
								larger=leftE;
								lastrownum=((SQLIntegerExpr) leftE.getRight()).getNumber().intValue();
								if(leftE.getOperator()==SQLBinaryOperator.LessThan&&lastrownum!=0) {
									lastrownum = lastrownum - 1;
								}
							}

							if(rightE.getRight() instanceof SQLIntegerExpr&&(rightE.getOperator()==SQLBinaryOperator.GreaterThan||rightE.getOperator()==SQLBinaryOperator.GreaterThanOrEqual))
							{
								small=rightE;
								firstrownum=((SQLIntegerExpr) rightE.getRight()).getNumber().intValue();
								if(rightE.getOperator()==SQLBinaryOperator.GreaterThanOrEqual&&firstrownum!=0) {
									firstrownum = firstrownum - 1;
								}
							} else
							if(rightE.getRight() instanceof SQLIntegerExpr&&(rightE.getOperator()==SQLBinaryOperator.LessThan||rightE.getOperator()==SQLBinaryOperator.LessThanOrEqual))
							{
								larger=rightE;
								lastrownum=((SQLIntegerExpr) rightE.getRight()).getNumber().intValue();
								if(rightE.getOperator()==SQLBinaryOperator.LessThan&&lastrownum!=0) {
									lastrownum = lastrownum - 1;
								}
							}
							if(small!=null&&larger!=null)
							{
								setLimitIFChange(stmt, rrs, schema, small, firstrownum, lastrownum);
								if(orderBy!=null)
								{
									SQLSelect oracleSelect = (SQLSelect) subSelect.getParent();
									oracleSelect.setOrderBy(orderBy);
								}
								parseOrderAggGroupOracle(stmt,rrs, (OracleSelectQueryBlock) subSelect, schema);
								isNeedParseOrderAgg=false;
							}

                        }


                    } else
                        {
                            parseNativeSql(stmt,rrs,mysqlSelectQuery,schema);
                        }



                }

        }

		}
        else
        {
            parseNativeSql(stmt,rrs,mysqlSelectQuery,schema);
        }
        if(isNeedParseOrderAgg)
        {
            parseOrderAggGroupOracle(stmt,rrs,  mysqlSelectQuery, schema);
        }
    }

	protected String getCurentDbType()
	{
		return JdbcConstants.ORACLE.name();
	}
    protected void parseNativeSql(SQLStatement stmt,RouteResultset rrs, OracleSelectQueryBlock mysqlSelectQuery,SchemaConfig schema)
    {
		 //解析分页以外的语法
    }

	private void parseThreeLevelPageSql(SQLStatement stmt, RouteResultset rrs, SchemaConfig schema, SQLSubqueryTableSource from, SQLBinaryOpExpr one, SQLBinaryOperator operator)
	{
        SQLIntegerExpr right = (SQLIntegerExpr) one.getRight();
		int firstrownum = right.getNumber().intValue();
		if (operator == SQLBinaryOperator.GreaterThanOrEqual&&firstrownum!=0) {
			firstrownum = firstrownum - 1;
		}
		SQLSelectQuery subSelect = from.getSelect().getQuery();
		if (subSelect instanceof OracleSelectQueryBlock)
        {  //第二层子查询
            OracleSelectQueryBlock twoSubSelect = (OracleSelectQueryBlock) subSelect;
            if (twoSubSelect.getWhere() instanceof SQLBinaryOpExpr && twoSubSelect.getFrom() instanceof SQLSubqueryTableSource)
            {
                SQLBinaryOpExpr twoWhere = (SQLBinaryOpExpr) twoSubSelect.getWhere();
                boolean isRowNum = "rownum".equalsIgnoreCase(twoWhere.getLeft().toString());
                boolean isLess = twoWhere.getOperator() == SQLBinaryOperator.LessThanOrEqual || twoWhere.getOperator() == SQLBinaryOperator.LessThan;
                if (isRowNum && twoWhere.getRight() instanceof SQLIntegerExpr && isLess)
                {
                    int lastrownum = ((SQLIntegerExpr) twoWhere.getRight()).getNumber().intValue();
                    if (operator == SQLBinaryOperator.LessThan&&lastrownum!=0) {
						lastrownum = lastrownum - 1;
					}
                    SQLSelectQuery finalQuery = ((SQLSubqueryTableSource) twoSubSelect.getFrom()).getSelect().getQuery();
                    if (finalQuery instanceof OracleSelectQueryBlock)
                    {
						setLimitIFChange(stmt, rrs, schema, one, firstrownum, lastrownum);
                        parseOrderAggGroupOracle(stmt,rrs, (OracleSelectQueryBlock) finalQuery, schema);
                        isNeedParseOrderAgg=false;
                    }

                }

            }

        }
	}





	

}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/impl/DruidSelectParser.java
================================================
package io.mycat.route.parser.druid.impl;

import java.sql.SQLNonTransientException;
import java.sql.SQLSyntaxErrorException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;

import org.apache.logging.log4j.util.Strings;

import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLLimit;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.SQLOrderingSpecification;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLAggregateExpr;
import com.alibaba.druid.sql.ast.expr.SQLAllColumnExpr;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator;
import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr;
import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr;
import com.alibaba.druid.sql.ast.expr.SQLNumericLiteralExpr;
import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr;
import com.alibaba.druid.sql.ast.expr.SQLTextLiteralExpr;
import com.alibaba.druid.sql.ast.statement.SQLExprTableSource;
import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.ast.statement.SQLSelectGroupByClause;
import com.alibaba.druid.sql.ast.statement.SQLSelectItem;
import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem;
import com.alibaba.druid.sql.ast.statement.SQLSelectQuery;
import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource;
import com.alibaba.druid.sql.ast.statement.SQLTableSource;
import com.alibaba.druid.sql.ast.statement.SQLUnionQuery;
import com.alibaba.druid.sql.dialect.db2.ast.stmt.DB2SelectQueryBlock;
import com.alibaba.druid.sql.dialect.db2.visitor.DB2OutputVisitor;
import com.alibaba.druid.sql.dialect.mysql.ast.expr.MySqlOrderingExpr;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlOutputVisitor;
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlSchemaStatVisitor;
import com.alibaba.druid.sql.dialect.oracle.ast.stmt.OracleSelectQueryBlock;
import com.alibaba.druid.sql.dialect.oracle.visitor.OracleOutputVisitor;
import com.alibaba.druid.sql.dialect.postgresql.ast.stmt.PGSelectQueryBlock;
import com.alibaba.druid.sql.dialect.postgresql.visitor.PGOutputVisitor;
import com.alibaba.druid.sql.dialect.sqlserver.ast.SQLServerSelectQueryBlock;
import com.alibaba.druid.sql.parser.SQLStatementParser;
import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
import com.alibaba.druid.util.JdbcConstants;
import com.alibaba.druid.wall.spi.WallVisitorUtils;

import io.mycat.MycatServer;
import io.mycat.cache.LayerCachePool;
import io.mycat.config.ErrorCode;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.TableConfig;
import io.mycat.route.RouteResultset;
import io.mycat.route.RouteResultsetNode;
import io.mycat.route.parser.druid.MycatSchemaStatVisitor;
import io.mycat.route.parser.druid.MycatStatementParser;
import io.mycat.route.parser.druid.RouteCalculateUnit;
import io.mycat.route.parser.util.PageSQLUtil;
import io.mycat.route.parser.util.WildcardUtil;
import io.mycat.route.util.RouterUtil;
import io.mycat.sqlengine.mpp.ColumnRoutePair;
import io.mycat.sqlengine.mpp.HavingCols;
import io.mycat.sqlengine.mpp.MergeCol;
import io.mycat.sqlengine.mpp.OrderCol;
import io.mycat.util.ObjectUtil;
import io.mycat.util.StringUtil;

public class DruidSelectParser extends DefaultDruidParser {


	protected boolean isNeedParseOrderAgg=true;

	@Override
	public void statementParse(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt) {
		SQLSelectStatement selectStmt = (SQLSelectStatement)stmt;
		SQLSelectQuery sqlSelectQuery = selectStmt.getSelect().getQuery();
		if(sqlSelectQuery instanceof MySqlSelectQueryBlock) {
			MySqlSelectQueryBlock mysqlSelectQuery = (MySqlSelectQueryBlock)selectStmt.getSelect().getQuery();

			parseOrderAggGroupMysql(schema, stmt,rrs, mysqlSelectQuery);
			//更改canRunInReadDB属性
			if ((mysqlSelectQuery.isForUpdate() || mysqlSelectQuery.isLockInShareMode()) && rrs.isAutocommit() == false)
			{
				rrs.setCanRunInReadDB(false);
			}

		} else if (sqlSelectQuery instanceof SQLUnionQuery) {
//			MySqlUnionQuery unionQuery = (MySqlUnionQuery)sqlSelectQuery;
//			MySqlSelectQueryBlock left = (MySqlSelectQueryBlock)unionQuery.getLeft();
//			MySqlSelectQueryBlock right = (MySqlSelectQueryBlock)unionQuery.getLeft();
//			System.out.println();
		}
	}
	protected void parseOrderAggGroupMysql(SchemaConfig schema, SQLStatement stmt, RouteResultset rrs, MySqlSelectQueryBlock mysqlSelectQuery)
	{
		MySqlSchemaStatVisitor visitor = new MySqlSchemaStatVisitor();
		stmt.accept(visitor);
//		rrs.setGroupByCols((String[])visitor.getGroupByColumns().toArray());
		if(!isNeedParseOrderAgg)
		{
			return;
		}
		Map aliaColumns = parseAggGroupCommon(schema, stmt, rrs, mysqlSelectQuery);

		//setOrderByCols
		if(mysqlSelectQuery.getOrderBy() != null) {
			List orderByItems = mysqlSelectQuery.getOrderBy().getItems();
			rrs.setOrderByCols(buildOrderByCols(orderByItems,aliaColumns));
		}
		isNeedParseOrderAgg=false;
	}
	protected Map parseAggGroupCommon(SchemaConfig schema, SQLStatement stmt, RouteResultset rrs, SQLSelectQueryBlock mysqlSelectQuery)
	{
		Map aliaColumns = new HashMap();
		Map aggrColumns = new HashMap();
		// Added by winbill, 20160314, for having clause, Begin ==>
		List havingColsName = new ArrayList();
		// Added by winbill, 20160314, for having clause, End  <==
		List selectList = mysqlSelectQuery.getSelectList();
		boolean isNeedChangeSql=false;
		int size = selectList.size();
		boolean isDistinct=mysqlSelectQuery.getDistionOption()==2;
		for (int i = 0; i < size; i++)
		{
			SQLSelectItem item = selectList.get(i);

			if (item.getExpr() instanceof SQLAggregateExpr)
			{
				SQLAggregateExpr expr = (SQLAggregateExpr) item.getExpr();
				String method = expr.getMethodName().toUpperCase();
				boolean isHasArgument=!expr.getArguments().isEmpty();
				if(isHasArgument)
				{
					String aggrColName = method + "(" + expr.getArguments().get(0) + ")";   // Added by winbill, 20160314, for having clause
					havingColsName.add(aggrColName);       // Added by winbill, 20160314, for having clause
				}
				//只处理有别名的情况，无别名添加别名，否则某些数据库会得不到正确结果处理
				int mergeType = MergeCol.getMergeType(method);
				if (MergeCol.MERGE_AVG == mergeType&&isRoutMultiNode(schema,rrs))
				{    //跨分片avg需要特殊处理，直接avg结果是不对的
					String colName = item.getAlias() != null ? item.getAlias() : method + i;
					SQLSelectItem sum =new SQLSelectItem();
					String sumColName = colName + "SUM";
					sum.setAlias(sumColName);
					SQLAggregateExpr sumExp =new SQLAggregateExpr("SUM");
					ObjectUtil.copyProperties(expr,sumExp);
					sumExp.getArguments().addAll(expr.getArguments());
					sumExp.setMethodName("SUM");
					sum.setExpr(sumExp);
					selectList.set(i, sum);
					aggrColumns.put(sumColName, MergeCol.MERGE_SUM);
					havingColsName.add(sumColName);    // Added by winbill, 20160314, for having clause
					havingColsName.add(item.getAlias() != null ? item.getAlias() : "");    // Added by winbill, 20160314, two aliases for AVG

					SQLSelectItem count =new SQLSelectItem();
					String countColName = colName + "COUNT";
					count.setAlias(countColName);
					SQLAggregateExpr countExp = new SQLAggregateExpr("COUNT");
					ObjectUtil.copyProperties(expr,countExp);
					countExp.getArguments().addAll(expr.getArguments());
					countExp.setMethodName("COUNT");
					count.setExpr(countExp);
					selectList.add(count);
					aggrColumns.put(countColName, MergeCol.MERGE_COUNT);

					isNeedChangeSql=true;
					aggrColumns.put(colName, mergeType);
					rrs.setHasAggrColumn(true);
				} else if (MergeCol.MERGE_UNSUPPORT != mergeType){
					String aggColName = null;
					StringBuilder sb = new StringBuilder();
					if(mysqlSelectQuery instanceof MySqlSelectQueryBlock) {
						expr.accept(new MySqlOutputVisitor(sb));
					} else if(mysqlSelectQuery instanceof OracleSelectQueryBlock) {
						expr.accept(new OracleOutputVisitor(sb));
					} else if(mysqlSelectQuery instanceof PGSelectQueryBlock){
						expr.accept(new PGOutputVisitor(sb));
					} else if(mysqlSelectQuery instanceof SQLServerSelectQueryBlock) {
						expr.accept(new SQLASTOutputVisitor(sb));
					} else if(mysqlSelectQuery instanceof DB2SelectQueryBlock) {
						expr.accept(new DB2OutputVisitor(sb));
					}
					aggColName = sb.toString();

					if (item.getAlias() != null && item.getAlias().length() > 0)
					{
						aggrColumns.put(item.getAlias(), mergeType);
						aliaColumns.put(aggColName,item.getAlias());
					} else
					{   //如果不加，jdbc方式时取不到正确结果   ;修改添加别名
						item.setAlias(method + i);
						aggrColumns.put(method + i, mergeType);
						aliaColumns.put(aggColName, method + i);
						isNeedChangeSql=true;
					}
					rrs.setHasAggrColumn(true);
					havingColsName.add(item.getAlias());   // Added by winbill, 20160314, for having clause
					havingColsName.add("");                // Added by winbill, 20160314, one alias for non-AVG
				}
			} else
			{
				if (!(item.getExpr() instanceof SQLAllColumnExpr))
				{
					String alia = item.getAlias();
					String field = getFieldName(item);
					if (alia == null)
					{
						alia = field;
					}
					aliaColumns.put(field, alia);
				}
			}

		}
		if(aggrColumns.size() > 0) {
			rrs.setMergeCols(aggrColumns);
		}

		//通过优化转换成group by来实现
		if(isDistinct)
		{
			mysqlSelectQuery.setDistionOption(0);
			SQLSelectGroupByClause   groupBy=new SQLSelectGroupByClause();
			for (String fieldName : aliaColumns.keySet())
			{
				groupBy.addItem(new SQLIdentifierExpr(fieldName));
			}
			mysqlSelectQuery.setGroupBy(groupBy);
			isNeedChangeSql=true;
		}


		//setGroupByCols
		if(mysqlSelectQuery.getGroupBy() != null) {
			List groupByItems = mysqlSelectQuery.getGroupBy().getItems();
			String[] groupByCols = buildGroupByCols(groupByItems,aliaColumns);
			rrs.setGroupByCols(groupByCols);
			rrs.setHavings(buildGroupByHaving(mysqlSelectQuery.getGroupBy().getHaving(),aliaColumns));
			rrs.setHasAggrColumn(true);
			rrs.setHavingColsName(havingColsName.toArray()); // Added by winbill, 20160314, for having clause
		}


		if (isNeedChangeSql)
		{
			String sql = stmt.toString();
			rrs.changeNodeSqlAfterAddLimit(schema,getCurentDbType(),sql,0,-1, false);
			getCtx().setSql(sql);
		}
		return aliaColumns;
	}

	private HavingCols buildGroupByHaving(SQLExpr having,Map aliaColumns ){
		if (having == null) {
			return null;
		}

		SQLBinaryOpExpr expr  = ((SQLBinaryOpExpr) having);
		SQLExpr left = expr.getLeft();
		SQLBinaryOperator operator = expr.getOperator();
		SQLExpr right = expr.getRight();

		String leftValue = null;;
		if (left instanceof SQLAggregateExpr) {
			leftValue = ((SQLAggregateExpr) left).getMethodName() + "("
					+ ((SQLAggregateExpr) left).getArguments().get(0) + ")";
			String aggrColumnAlias = getAliaColumn(aliaColumns,leftValue);
			if(aggrColumnAlias != null) { // having聚合函数存在别名
				expr.setLeft(new SQLIdentifierExpr(aggrColumnAlias));
				leftValue = aggrColumnAlias;
			}
		} else if (left instanceof SQLIdentifierExpr) {
			leftValue = ((SQLIdentifierExpr) left).getName();
		}

		String rightValue = null;
		if (right instanceof  SQLNumericLiteralExpr) {
			rightValue = right.toString();
		}else if(right instanceof SQLTextLiteralExpr){
			rightValue = StringUtil.removeBackquote(right.toString());
		}

		return new HavingCols(leftValue,rightValue,operator.getName());
	}

	private boolean isRoutMultiNode(SchemaConfig schema,  RouteResultset rrs)
	{
		if(rrs.getNodes()!=null&&rrs.getNodes().length>1)
		{
			return true;
		}
		LayerCachePool tableId2DataNodeCache = (LayerCachePool) MycatServer.getInstance().getCacheService().getCachePool("TableID2DataNodeCache");
		try
		{
			tryRoute(schema, rrs, tableId2DataNodeCache);
			if(rrs.getNodes()!=null&&rrs.getNodes().length>1)
			{
				return true;
			}
		} catch (SQLNonTransientException e)
		{
			throw new RuntimeException(e);
		}
		return false;
	}

	private String getFieldName(SQLSelectItem item){
		if ((item.getExpr() instanceof SQLPropertyExpr)||(item.getExpr() instanceof SQLMethodInvokeExpr)
				|| (item.getExpr() instanceof SQLIdentifierExpr) || item.getExpr() instanceof SQLBinaryOpExpr) {
			return item.getExpr().toString();//字段别名
		}
		else if (!StringUtil.isEmpty(item.getAlias())) { // add by hehuang 20181205 如果SelectItem存在别名，则认为表达式为字段名，sql语法支持常量作为字段
			return item.getExpr().toString();
		} else {
			return item.toString();
		}
	}

	/**
	 * 现阶段目标为 有一个只涉及到一张表的子查询时,先执行子查询,获得返回结果后,改写原有sql继续执行,得到最终结果.
	 * 在这种情况下,原sql不需要继续解析.
	 * 使用catlet 的情况也不再继续解析.
	 */
	@Override
	public boolean afterVisitorParser(RouteResultset rrs, SQLStatement stmt, MycatSchemaStatVisitor visitor) {
		int subQuerySize = visitor.getSubQuerys().size();

		if(subQuerySize==0&&ctx.getTables().size()==2){ //两表关联,考虑使用catlet
			if(ctx.getVisitor().getConditions() !=null && ctx.getVisitor().getConditions().size()>0){
				return true;
			}
		}else if(subQuerySize==1){     //只涉及一张表的子查询,使用  MiddlerResultHandler 获取中间结果后,改写原有 sql 继续执行 TODO 后期可能会考虑多个.
			SQLSelectQuery sqlSelectQuery = visitor.getSubQuerys().iterator().next().getQuery();
			if(((MySqlSelectQueryBlock)sqlSelectQuery).getFrom() instanceof SQLExprTableSource) {
				return true;
			}
		}

		return super.afterVisitorParser(rrs, stmt, visitor);
	}

	/**
	 * 改写sql：需要加limit的加上
	 */
	@Override
	public void changeSql(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt,LayerCachePool cachePool) throws SQLNonTransientException {

		tryRoute(schema, rrs, cachePool);

		rrs.copyLimitToNodes();

		SQLSelectStatement selectStmt = (SQLSelectStatement)stmt;
		SQLSelectQuery sqlSelectQuery = selectStmt.getSelect().getQuery();
		if(sqlSelectQuery instanceof MySqlSelectQueryBlock) {
			MySqlSelectQueryBlock mysqlSelectQuery = (MySqlSelectQueryBlock)selectStmt.getSelect().getQuery();
			int limitStart = 0;
			int limitSize = schema.getDefaultMaxLimit();

			//clear group having
			SQLSelectGroupByClause groupByClause = mysqlSelectQuery.getGroupBy();
			// Modified by winbill, 20160614, do NOT include having clause when routing to multiple nodes
			if(groupByClause != null && groupByClause.getHaving() != null && isRoutMultiNode(schema,rrs)){
				groupByClause.setHaving(null);
			}

			Map>> allConditions = getAllConditions();
			boolean isNeedAddLimit = isNeedAddLimit(schema, rrs, mysqlSelectQuery, allConditions);
			if(isNeedAddLimit) {
				SQLLimit limit = new SQLLimit();
				limit.setRowCount(new SQLIntegerExpr(limitSize));
				mysqlSelectQuery.setLimit(limit);
				rrs.setLimitSize(limitSize);
				String sql= getSql(rrs, stmt, isNeedAddLimit);
				rrs.changeNodeSqlAfterAddLimit(schema, getCurentDbType(), sql, 0, limitSize, true);

			}
			SQLLimit limit = mysqlSelectQuery.getLimit();
			if(limit != null&&!isNeedAddLimit) {
				SQLIntegerExpr offset = (SQLIntegerExpr)limit.getOffset();
				SQLIntegerExpr count = (SQLIntegerExpr)limit.getRowCount();
				if(offset != null) {
					limitStart = offset.getNumber().intValue();
					rrs.setLimitStart(limitStart);
				}
				if(count != null) {
					limitSize = count.getNumber().intValue();
					rrs.setLimitSize(limitSize);
				}

				if(isNeedChangeLimit(rrs)) {
					SQLLimit changedLimit = new SQLLimit();
					changedLimit.setRowCount(new SQLIntegerExpr(limitStart + limitSize));

					if(offset != null) {
						if(limitStart < 0) {
							String msg = "You have an error in your SQL syntax; check the manual that " +
									"corresponds to your MySQL server version for the right syntax to use near '" + limitStart + "'";
							throw new SQLNonTransientException(ErrorCode.ER_PARSE_ERROR + " - " + msg);
						} else {
							changedLimit.setOffset(new SQLIntegerExpr(0));

						}
					}

					mysqlSelectQuery.setLimit(changedLimit);

					String sql= getSql(rrs, stmt, isNeedAddLimit);
					rrs.changeNodeSqlAfterAddLimit(schema,getCurentDbType(),sql,0, limitStart + limitSize, true);

					//设置改写后的sql
					ctx.setSql(sql);

				}   else
				{

					rrs.changeNodeSqlAfterAddLimit(schema,getCurentDbType(),getCtx().getSql(),rrs.getLimitStart(), rrs.getLimitSize(), true);
					//	ctx.setSql(nativeSql);

				}


			}

			if(rrs.isDistTable()){
              
                for (RouteResultsetNode node : rrs.getNodes()) {
                    String sql = selectStmt.toString();
                    SQLStatementParser parser = null;
                    if (schema.isNeedSupportMultiDBType()) {
                        parser = new MycatStatementParser(sql);
                    } else {
                        parser = new MySqlStatementParser(sql); 
                    }
                    
                    SQLSelectStatement newStmt = null;
                    try {
                        newStmt = (SQLSelectStatement) parser.parseStatement();
                    } catch (Exception t) {
                        LOGGER.error("DruidMycatRouteStrategyError", t);
                        throw new SQLSyntaxErrorException(t);
                    }
                    
                    MySqlSelectQueryBlock query = (MySqlSelectQueryBlock) newStmt.getSelect().getQuery();
                    SQLTableSource from2 = query.getFrom();
                    if (from2 instanceof SQLSubqueryTableSource) {
                        SQLSubqueryTableSource from = (SQLSubqueryTableSource) from2;
                        MySqlSelectQueryBlock query2 = (MySqlSelectQueryBlock) from.getSelect().getQuery();
                        repairExpr(query2.getFrom(), node);
                        node.setStatement(newStmt.toString());
                    } else {
                        repairExpr(from2, node);
                        node.setStatement(newStmt.toString());
                    }
                }
                
            }

			rrs.setCacheAble(isNeedCache(schema, rrs, mysqlSelectQuery, allConditions));
		}

	}
	
	private void repairExpr(SQLTableSource source, RouteResultsetNode node) throws SQLNonTransientException {
        Map subTableNames = node.getSubTableNames();
        
        if (source instanceof SQLJoinTableSource) {
            SQLJoinTableSource joinsource = (SQLJoinTableSource) source;
            SQLTableSource right = joinsource.getRight();
            String alias = right.getAlias();
            
            if(right instanceof SQLExprTableSource) {
                SQLIdentifierExpr expr = (SQLIdentifierExpr) ((SQLExprTableSource) right).getExpr();
                alias = Strings.isBlank(alias) ? expr.getName() : alias;
                String subTableName = subTableNames.get(WildcardUtil.wildcard(expr.getName().toUpperCase()));
                if (Strings.isNotBlank(subTableName)) {
                    String tableName = expr.getName();
                    alias = Strings.isBlank(alias) ? tableName : alias;
                    right.setAlias(alias);
                    
                    expr.setName(subTableName);
                }
            } 
            
            if (right instanceof SQLSubqueryTableSource) {
                SQLSubqueryTableSource from = (SQLSubqueryTableSource) right;
                MySqlSelectQueryBlock query = (MySqlSelectQueryBlock) from.getSelect().getQuery();
                repairExpr(query.getFrom(), node);
            }
            
            repairExpr(joinsource.getLeft(), node);
        } else if (source instanceof SQLExprTableSource) {
            SQLExprTableSource exprTableSource = (SQLExprTableSource) source;
            String alias = exprTableSource.getAlias();
            
            SQLIdentifierExpr expr = (SQLIdentifierExpr) exprTableSource.getExpr();
            alias = Strings.isBlank(alias) ? expr.getName() : alias;
            String subTableName = subTableNames.get(WildcardUtil.wildcard(expr.getName().toUpperCase()));
            if (Strings.isNotBlank(subTableName)) {
                String tableName = expr.getName();
                alias = Strings.isBlank(alias) ? tableName : alias;
                exprTableSource.setAlias(alias);
                expr.setName(subTableName);
            }
            
        } else if (source instanceof  SQLSubqueryTableSource) {
            SQLSelect select = ((SQLSubqueryTableSource) source).getSelect();
            MySqlSelectQueryBlock query = (MySqlSelectQueryBlock) select.getQuery();
            repairExpr(query.getFrom(), node);
        }
    }

	private void fixLimit(MySqlSelectQueryBlock mysqlSelectQuery, RouteResultsetNode node) {
		if(!getCurentDbType().equalsIgnoreCase("mysql")) {
			SQLLimit _limit = mysqlSelectQuery.getLimit();
			if (_limit != null) {
				SQLIntegerExpr offset = (SQLIntegerExpr) _limit.getOffset();
				SQLIntegerExpr count = (SQLIntegerExpr) _limit.getRowCount();
				if (offset != null && count != null) {
					String nativeSql = PageSQLUtil
							.convertLimitToNativePageSql(getCurentDbType(), node.getStatement(),
									offset.getNumber().intValue(), count.getNumber().intValue());
					node.setStatement(nativeSql);
				}
			}
		}
	}

	/**
	 * 获取所有的条件：因为可能被or语句拆分成多个RouteCalculateUnit，条件分散了
	 * @return
	 */
	private Map>> getAllConditions() {
		Map>> map = new HashMap>>();
		for(RouteCalculateUnit unit : ctx.getRouteCalculateUnits()) {
			if(unit != null && unit.getTablesAndConditions() != null) {
				map.putAll(unit.getTablesAndConditions());
			}
		}

		return map;
	}

	private void tryRoute(SchemaConfig schema, RouteResultset rrs, LayerCachePool cachePool) throws SQLNonTransientException {
		if(rrs.isFinishedRoute())
		{
			return;//避免重复路由
		}

		//无表的select语句直接路由带任一节点
		if((ctx.getTables() == null || ctx.getTables().size() == 0)&&(ctx.getTableAliasMap()==null||ctx.getTableAliasMap().isEmpty())) {
			rrs = RouterUtil.routeToSingleNode(rrs, schema.getRandomDataNode(), ctx.getSql());
			rrs.setFinishedRoute(true);
			return;
		}
//		RouterUtil.tryRouteForTables(schema, ctx, rrs, true, cachePool);
		SortedSet nodeSet = new TreeSet();
		boolean isAllGlobalTable = RouterUtil.isAllGlobalTable(ctx, schema);
		for (RouteCalculateUnit unit : ctx.getRouteCalculateUnits()) {
			RouteResultset rrsTmp = RouterUtil.tryRouteForTables(schema, ctx, unit, rrs, true, cachePool);
			if (rrsTmp != null&&rrsTmp.getNodes()!=null) {
				for (RouteResultsetNode node : rrsTmp.getNodes()) {
					nodeSet.add(node);
				}
			}
			if(isAllGlobalTable) {//都是全局表时只计算一遍路由
				break;
			}
		}

		if(nodeSet.size() == 0) {

			Collection stringCollection= ctx.getTableAliasMap().values() ;
			for (String table : stringCollection)
			{
				if(table!=null&&table.toLowerCase().contains("information_schema."))
				{
					rrs = RouterUtil.routeToSingleNode(rrs, schema.getRandomDataNode(), ctx.getSql());
					rrs.setFinishedRoute(true);
					return;
				}
			}

			//add@byron 支持通过表指定schema路由到名称相同的datanode
			if(ctx.getTableAliasMap().size()>0){
				for(String key: ctx.getTableAliasMap().keySet()){
					TableConfig tb_config = schema.getTables().get(key.toUpperCase());
					if(tb_config!=null){
						String table_name = ctx.getTableAliasMap().get(key);
						for(String dataNode:tb_config.getDataNodes()){
							if(tb_config.getDataNodes().size()==1 || table_name.startsWith(dataNode+'.')){
								rrs = RouterUtil.routeToSingleNode(rrs, dataNode, ctx.getSql());
								rrs.setFinishedRoute(true);
								return;
							}
						}
					}
				}
			}
			//end@byron

			String msg = " find no Route:" + ctx.getSql();
			LOGGER.warn(msg);
			throw new SQLNonTransientException(msg);
		}

		RouteResultsetNode[] nodes = new RouteResultsetNode[nodeSet.size()];
		int i = 0;
		for (Iterator iterator = nodeSet.iterator(); iterator.hasNext();) {
			nodes[i] = (RouteResultsetNode) iterator.next();
			i++;

		}

		rrs.setNodes(nodes);
		rrs.setFinishedRoute(true);
	}


	protected String getCurentDbType()
	{
		return JdbcConstants.MYSQL.name();
	}




	protected String getSql( RouteResultset rrs,SQLStatement stmt, boolean isNeedAddLimit)
	{
		if(getCurentDbType().equalsIgnoreCase("mysql")&&(isNeedChangeLimit(rrs)||isNeedAddLimit))
		{

			return stmt.toString();

		}

		return getCtx().getSql();
	}



	protected boolean isNeedChangeLimit(RouteResultset rrs) {
		if(rrs.getNodes() == null) {
			return false;
		} else {
			if(rrs.getNodes().length > 1) {
				return true;
			}
			return false;

		}
	}

	private boolean isNeedCache(SchemaConfig schema, RouteResultset rrs,
								MySqlSelectQueryBlock mysqlSelectQuery, Map>> allConditions) {
		if(ctx.getTables() == null || ctx.getTables().size() == 0 ) {
			return false;
		}
		TableConfig tc = schema.getTables().get(ctx.getTables().get(0));
		if(tc==null ||(ctx.getTables().size() == 1 && tc.isGlobalTable())
				) {//|| (ctx.getTables().size() == 1) && tc.getRule() == null && tc.getDataNodes().size() == 1
			return false;
		} else {
			//单表主键查询
			if(ctx.getTables().size() == 1) {
				String tableName = ctx.getTables().get(0);
				String primaryKey = schema.getTables().get(tableName).getPrimaryKey();
//				schema.getTables().get(ctx.getTables().get(0)).getParentKey() != null;
				if(ctx.getRouteCalculateUnit().getTablesAndConditions().get(tableName) != null
						&& ctx.getRouteCalculateUnit().getTablesAndConditions().get(tableName).get(primaryKey) != null
						&& tc.getDataNodes().size() > 1) {//有主键条件
					return false;
				}
				//全局表不缓存
			}else if(RouterUtil.isAllGlobalTable(ctx, schema)){
				return false;
			}
			return true;
		}
	}

	/**
	 * 单表且是全局表
	 * 单表且rule为空且nodeNodes只有一个
	 * @param schema
	 * @param rrs
	 * @param mysqlSelectQuery
	 * @return
	 */
	private boolean isNeedAddLimit(SchemaConfig schema, RouteResultset rrs,
								   MySqlSelectQueryBlock mysqlSelectQuery, Map>> allConditions) {
//		ctx.getTablesAndConditions().get(key))
		if(rrs.getLimitSize()>-1)
		{
			return false;
		}else
		if(schema.getDefaultMaxLimit() == -1) {
			return false;
		} else if (mysqlSelectQuery.getLimit() != null) {//语句中已有limit
			return false;
		} else if(ctx.getTables().size() == 1) {
			String tableName = ctx.getTables().get(0);
			TableConfig tableConfig = schema.getTables().get(tableName);
			if(tableConfig==null)
			{
				return    schema.getDefaultMaxLimit() > -1;   //   找不到则取schema的配置
			}

			boolean isNeedAddLimit= tableConfig.isNeedAddLimit();
			if(!isNeedAddLimit)
			{
				return false;//优先从配置文件取
			}

			if(schema.getTables().get(tableName).isGlobalTable()) {
				return true;
			}

			String primaryKey = schema.getTables().get(tableName).getPrimaryKey();

//			schema.getTables().get(ctx.getTables().get(0)).getParentKey() != null;
			if(allConditions.get(tableName) == null) {//无条件
				return true;
			}

			if (allConditions.get(tableName).get(primaryKey) != null) {//条件中带主键
				return false;
			}

			return true;
		} else if(rrs.hasPrimaryKeyToCache() && ctx.getTables().size() == 1){//只有一个表且条件中有主键,不需要limit了,因为主键只能查到一条记录
			return false;
		} else {//多表或无表
			return false;
		}

	}
	private String getAliaColumn(Map aliaColumns,String column ){
		String alia=aliaColumns.get(column);
		if (alia==null){
			if(column.indexOf(".") < 0) {
				String col = "." + column;
				String col2 = ".`" + column+"`";
				//展开aliaColumns，将之类的键值对展开成和
				for(Map.Entry entry : aliaColumns.entrySet()) {
					if(entry.getKey().endsWith(col)||entry.getKey().endsWith(col2)) {
						if(entry.getValue() != null && entry.getValue().indexOf(".") > 0) {
							return column;
						}
						return entry.getValue();
					}
				}
			}

			return column;
		}
		else {
			return alia;
		}
	}

	private String[] buildGroupByCols(List groupByItems,Map aliaColumns) {
		String[] groupByCols = new String[groupByItems.size()];
		for(int i= 0; i < groupByItems.size(); i++) {
			SQLExpr sqlExpr = groupByItems.get(i);
			String column = null;
			if(sqlExpr instanceof SQLIdentifierExpr )
			{
				column=((SQLIdentifierExpr) sqlExpr).getName();
			} else if(sqlExpr instanceof SQLMethodInvokeExpr){
				column = ((SQLMethodInvokeExpr) sqlExpr).toString();
			} else if(sqlExpr instanceof MySqlOrderingExpr){
				//todo czn
				SQLExpr expr = ((MySqlOrderingExpr) sqlExpr).getExpr();

				if (expr instanceof SQLName)
				{
					column = StringUtil.removeBackquote(((SQLName) expr).getSimpleName());//不要转大写 2015-2-10 sohudo StringUtil.removeBackquote(expr.getSimpleName().toUpperCase());
				} else
				{
					column = StringUtil.removeBackquote(expr.toString());
				}
			} else if(sqlExpr instanceof SQLPropertyExpr){
				/**
				 * 针对子查询别名，例如select id from (select h.id from hotnews h  union select h.title from hotnews h ) as t1 group by t1.id;
				 */
				column = sqlExpr.toString();
			}
			if(column == null){
				column = sqlExpr.toString();
			}
			int dotIndex=column.indexOf(".") ;
			int bracketIndex=column.indexOf("(") ;
			//通过判断含有括号来决定是否为函数列
			if(dotIndex!=-1&&bracketIndex==-1)
			{
				//此步骤得到的column必须是不带.的，有别名的用别名，无别名的用字段名
				column=column.substring(dotIndex+1) ;
			}
			groupByCols[i] = getAliaColumn(aliaColumns,column);//column;
		}
		return groupByCols;
	}

	protected LinkedHashMap buildOrderByCols(List orderByItems,Map aliaColumns) {
		LinkedHashMap map = new LinkedHashMap();
		for(int i= 0; i < orderByItems.size(); i++) {
			SQLOrderingSpecification type = orderByItems.get(i).getType();
			//orderColumn只记录字段名称,因为返回的结果集是不带表名的。
			SQLExpr expr =  orderByItems.get(i).getExpr();
			String col;
			if (expr instanceof SQLName) {
				col = ((SQLName)expr).getSimpleName();
			}
			else {
				col =expr.toString();
			}
			if(type == null) {
				type = SQLOrderingSpecification.ASC;
			}
			col=getAliaColumn(aliaColumns,col);//此步骤得到的col必须是不带.的，有别名的用别名，无别名的用字段名
			map.put(col, type == SQLOrderingSpecification.ASC ? OrderCol.COL_ORDER_TYPE_ASC : OrderCol.COL_ORDER_TYPE_DESC);
		}
		return map;
	}

	private boolean isConditionAlwaysTrue(SQLStatement statement) {
		SQLSelectStatement selectStmt = (SQLSelectStatement)statement;
		SQLSelectQuery sqlSelectQuery = selectStmt.getSelect().getQuery();
		if(sqlSelectQuery instanceof MySqlSelectQueryBlock) {
			MySqlSelectQueryBlock mysqlSelectQuery = (MySqlSelectQueryBlock)selectStmt.getSelect().getQuery();
			SQLExpr expr = mysqlSelectQuery.getWhere();

			Object o = WallVisitorUtils.getValue(expr);
			if(Boolean.TRUE.equals(o)) {
				return true;
			}
			return false;
		} else {//union
			return false;
		}

	}

	protected void setLimitIFChange(SQLStatement stmt, RouteResultset rrs, SchemaConfig schema, SQLBinaryOpExpr one, int firstrownum, int lastrownum)
	{
		rrs.setLimitStart(firstrownum);
		rrs.setLimitSize(lastrownum - firstrownum);
		LayerCachePool tableId2DataNodeCache = (LayerCachePool) MycatServer.getInstance().getCacheService().getCachePool("TableID2DataNodeCache");
		try
		{
			tryRoute(schema, rrs, tableId2DataNodeCache);
		} catch (SQLNonTransientException e)
		{
			throw new RuntimeException(e);
		}
		if (isNeedChangeLimit(rrs))
		{
			one.setRight(new SQLIntegerExpr(0));
			String curentDbType ="db2".equalsIgnoreCase(this.getCurentDbType())?"oracle":getCurentDbType();
			String sql =   SQLUtils.toSQLString(stmt, curentDbType);;
			rrs.changeNodeSqlAfterAddLimit(schema,getCurentDbType(), sql,0,lastrownum, false);
			//设置改写后的sql
			getCtx().setSql(sql);
		}
	}
}

================================================
FILE: src/main/java/io/mycat/route/parser/druid/impl/DruidSelectPostgresqlParser.java
================================================
package io.mycat.route.parser.druid.impl;

import com.alibaba.druid.util.JdbcConstants;

public class DruidSelectPostgresqlParser extends DruidSelectParser
{


    protected String getCurentDbType()
    {
		return JdbcConstants.POSTGRESQL.name();
    }


}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/impl/DruidSelectSqlServerParser.java
================================================
package io.mycat.route.parser.druid.impl;

import java.util.List;
import java.util.Map;

import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLLimit;
import com.alibaba.druid.sql.ast.SQLOrderBy;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLAggregateExpr;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator;
import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.ast.statement.SQLSelectItem;
import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem;
import com.alibaba.druid.sql.ast.statement.SQLSelectQuery;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource;
import com.alibaba.druid.sql.ast.statement.SQLTableSource;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
import com.alibaba.druid.sql.dialect.sqlserver.ast.SQLServerSelectQueryBlock;
import com.alibaba.druid.sql.dialect.sqlserver.ast.SQLServerTop;
import com.alibaba.druid.sql.dialect.sqlserver.parser.SQLServerStatementParser;
import com.alibaba.druid.util.JdbcConstants;

import io.mycat.config.model.SchemaConfig;
import io.mycat.route.RouteResultset;

public class DruidSelectSqlServerParser extends DruidSelectParser {

	public DruidSelectSqlServerParser(){
		super();
		isNeedParseOrderAgg=true;
	}

	@Override
	public void statementParse(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt) {
		SQLSelectStatement selectStmt = (SQLSelectStatement)stmt;
		SQLSelectQuery sqlSelectQuery = selectStmt.getSelect().getQuery();
		//从mysql解析过来
		if(sqlSelectQuery instanceof MySqlSelectQueryBlock) {
			MySqlSelectQueryBlock mysqlSelectQuery = (MySqlSelectQueryBlock)selectStmt.getSelect().getQuery();
			SQLLimit limit = mysqlSelectQuery.getLimit();
			if(limit==null)
			{
                sqlserverParse(schema, rrs);


            }
			if(isNeedParseOrderAgg)
			{
				parseOrderAggGroupMysql(schema, stmt,rrs, mysqlSelectQuery);
				//更改canRunInReadDB属性
				if ((mysqlSelectQuery.isForUpdate() || mysqlSelectQuery.isLockInShareMode()) && rrs.isAutocommit() == false)
				{
					rrs.setCanRunInReadDB(false);
				}
			}

		}


	}
	protected String getCurentDbType()
	{
		return JdbcConstants.SQL_SERVER.name();
	}
    private void sqlserverParse(SchemaConfig schema, RouteResultset rrs)
    {
        //使用sqlserver的解析，否则会有部分语法识别错误
        SQLServerStatementParser oracleParser = new SQLServerStatementParser(getCtx().getSql());
        SQLSelectStatement oracleStmt = (SQLSelectStatement) oracleParser.parseStatement();
        SQLSelectQuery oracleSqlSelectQuery = oracleStmt.getSelect().getQuery();
        if(oracleSqlSelectQuery instanceof SQLServerSelectQueryBlock)
        {
            parseSqlServerPageSql(oracleStmt, rrs, (SQLServerSelectQueryBlock) oracleSqlSelectQuery, schema);
            if(isNeedParseOrderAgg)
            {
                parseOrderAggGroupSqlServer(schema, oracleStmt,rrs, (SQLServerSelectQueryBlock) oracleSqlSelectQuery);
            }
        }

    }


    private void parseOrderAggGroupSqlServer(SchemaConfig schema, SQLStatement stmt, RouteResultset rrs, SQLServerSelectQueryBlock mysqlSelectQuery)
	{
		Map aliaColumns = parseAggGroupCommon(schema, stmt,rrs, mysqlSelectQuery);

		SQLSelect oracleSelect = (SQLSelect) mysqlSelectQuery.getParent();
		if(oracleSelect.getOrderBy() != null) {
			List orderByItems = oracleSelect.getOrderBy().getItems();
			rrs.setOrderByCols(buildOrderByCols(orderByItems,aliaColumns));
		}
	}

	private void parseSqlServerPageSql(SQLStatement stmt, RouteResultset rrs, SQLServerSelectQueryBlock sqlserverSelectQuery, SchemaConfig schema)
	{
		//第一层子查询
		SQLExpr where=  sqlserverSelectQuery.getWhere();
		SQLTableSource from= sqlserverSelectQuery.getFrom();
        if(sqlserverSelectQuery.getTop()!=null)
        {
            SQLServerTop top= sqlserverSelectQuery.getTop() ;
            SQLExpr sqlExpr=  top.getExpr()  ;
            if(sqlExpr instanceof SQLIntegerExpr)
            {

                int    topValue=((SQLIntegerExpr) sqlExpr).getNumber().intValue();
                rrs.setLimitStart(0);
                rrs.setLimitSize(topValue);
            }
        }
        else
		if(where instanceof SQLBinaryOpExpr &&from instanceof SQLSubqueryTableSource)
		{

			SQLBinaryOpExpr one= (SQLBinaryOpExpr) where;
			SQLExpr left=one.getLeft();
			SQLBinaryOperator operator =one.getOperator();
			SQLSelectQuery subSelect = ((SQLSubqueryTableSource) from).getSelect().getQuery();
			SQLOrderBy orderBy=null;
			if (subSelect instanceof SQLServerSelectQueryBlock)
			{
				boolean hasRowNumber=false;
                boolean hasSubTop=false;
                int subTop=0;
				SQLServerSelectQueryBlock subSelectOracle = (SQLServerSelectQueryBlock) subSelect;
				List sqlSelectItems=    subSelectOracle.getSelectList();
				for (SQLSelectItem sqlSelectItem : sqlSelectItems)
				{
					SQLExpr sqlExpr=  sqlSelectItem.getExpr()   ;
					if(sqlExpr instanceof  SQLAggregateExpr )
					{
						SQLAggregateExpr agg= (SQLAggregateExpr) sqlExpr;
						if("row_number".equalsIgnoreCase(agg.getMethodName())&&agg.getOver()!=null)
						{
							hasRowNumber=true;
							orderBy= agg.getOver().getOrderBy();
						}

					}
				}
                if(subSelectOracle.getFrom() instanceof SQLSubqueryTableSource)
                {
                    SQLSubqueryTableSource subFrom= (SQLSubqueryTableSource) subSelectOracle.getFrom();
                    if (subFrom.getSelect().getQuery() instanceof SQLServerSelectQueryBlock)
                    {
                        SQLServerSelectQueryBlock sqlSelectQuery = (SQLServerSelectQueryBlock) subFrom.getSelect().getQuery();
                        if(sqlSelectQuery.getTop()!=null)
                        {

                            SQLExpr sqlExpr=  sqlSelectQuery.getTop().getExpr()  ;
                            if(sqlExpr instanceof SQLIntegerExpr)
                            {
                                hasSubTop=true;
                                subTop=((SQLIntegerExpr) sqlExpr).getNumber().intValue();
                                orderBy=  subFrom.getSelect().getOrderBy();
                            }
                        }

                    }
                }

				if(hasRowNumber)
				{
                     if(hasSubTop&&(operator==SQLBinaryOperator.GreaterThan||operator==SQLBinaryOperator.GreaterThanOrEqual)&& one.getRight() instanceof SQLIntegerExpr)
                     {
                         SQLIntegerExpr right = (SQLIntegerExpr) one.getRight();
                         int firstrownum = right.getNumber().intValue();
                         if (operator == SQLBinaryOperator.GreaterThanOrEqual&&firstrownum!=0) {
							 firstrownum = firstrownum - 1;
						 }
                         int lastrownum =subTop;
                         setLimitIFChange(stmt, rrs, schema, one, firstrownum, lastrownum);
                         if(orderBy!=null)
                         {
								SQLSelect oracleSelect = (SQLSelect) subSelect.getParent();
								oracleSelect.setOrderBy(orderBy);
                         }
                         parseOrderAggGroupSqlServer(schema, stmt,rrs, (SQLServerSelectQueryBlock) subSelect);
                         isNeedParseOrderAgg=false;

                     }
                       else

					if((operator==SQLBinaryOperator.LessThan||operator==SQLBinaryOperator.LessThanOrEqual) && one.getRight() instanceof SQLIntegerExpr )
					{
						SQLIntegerExpr right = (SQLIntegerExpr) one.getRight();
						int firstrownum = right.getNumber().intValue();
						if (operator == SQLBinaryOperator.LessThan&&firstrownum!=0) {
							firstrownum = firstrownum - 1;
						}
						if (subSelect instanceof SQLServerSelectQueryBlock)
						{
							rrs.setLimitStart(0);
							rrs.setLimitSize(firstrownum);
							sqlserverSelectQuery = (SQLServerSelectQueryBlock) subSelect;    //为了继续解出order by 等
							if(orderBy!=null)
							{
								SQLSelect oracleSelect = (SQLSelect) subSelect.getParent();
								oracleSelect.setOrderBy(orderBy);
							}
							parseOrderAggGroupSqlServer(schema, stmt,rrs, sqlserverSelectQuery);
							isNeedParseOrderAgg=false;
						}
					}
					else
					if(operator==SQLBinaryOperator.BooleanAnd && left instanceof SQLBinaryOpExpr&&one.getRight() instanceof SQLBinaryOpExpr )
					{
						SQLBinaryOpExpr leftE= (SQLBinaryOpExpr) left;
						SQLBinaryOpExpr rightE= (SQLBinaryOpExpr) one.getRight();
						SQLBinaryOpExpr small=null ;
						SQLBinaryOpExpr larger=null ;
						int firstrownum =0;
						int lastrownum =0;
						if(leftE.getRight() instanceof SQLIntegerExpr&&(leftE.getOperator()==SQLBinaryOperator.GreaterThan||leftE.getOperator()==SQLBinaryOperator.GreaterThanOrEqual))
						{
							small=leftE;
							firstrownum=((SQLIntegerExpr) leftE.getRight()).getNumber().intValue();
							if(leftE.getOperator()==SQLBinaryOperator.GreaterThanOrEqual &&firstrownum!=0) {
								firstrownum = firstrownum - 1;
							}
						} else
						if(leftE.getRight() instanceof SQLIntegerExpr&&(leftE.getOperator()==SQLBinaryOperator.LessThan||leftE.getOperator()==SQLBinaryOperator.LessThanOrEqual))
						{
							larger=leftE;
							lastrownum=((SQLIntegerExpr) leftE.getRight()).getNumber().intValue();
							if(leftE.getOperator()==SQLBinaryOperator.LessThan&&lastrownum!=0) {
								lastrownum = lastrownum - 1;
							}
						}

						if(rightE.getRight() instanceof SQLIntegerExpr&&(rightE.getOperator()==SQLBinaryOperator.GreaterThan||rightE.getOperator()==SQLBinaryOperator.GreaterThanOrEqual))
						{
							small=rightE;
							firstrownum=((SQLIntegerExpr) rightE.getRight()).getNumber().intValue();
							if(rightE.getOperator()==SQLBinaryOperator.GreaterThanOrEqual&&firstrownum!=0) {
								firstrownum = firstrownum - 1;
							}
						} else
						if(rightE.getRight() instanceof SQLIntegerExpr&&(rightE.getOperator()==SQLBinaryOperator.LessThan||rightE.getOperator()==SQLBinaryOperator.LessThanOrEqual))
						{
							larger=rightE;
							lastrownum=((SQLIntegerExpr) rightE.getRight()).getNumber().intValue();
							if(rightE.getOperator()==SQLBinaryOperator.LessThan&&lastrownum!=0) {
								lastrownum = lastrownum - 1;
							}
						}
						if(small!=null&&larger!=null)
						{
							setLimitIFChange(stmt, rrs, schema, small, firstrownum, lastrownum);
							if(orderBy!=null)
							{
								SQLSelect oracleSelect = (SQLSelect) subSelect.getParent();
								oracleSelect.setOrderBy(orderBy);
							}
							parseOrderAggGroupSqlServer(schema, stmt,rrs, (SQLServerSelectQueryBlock) subSelect);
							isNeedParseOrderAgg=false;
						}

					}


				}



			}

		}

	}


	

}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/impl/DruidUpdateParser.java
================================================
package io.mycat.route.parser.druid.impl;

import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.statement.SQLUpdateSetItem;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlUpdateStatement;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.expr.*;
import com.alibaba.druid.sql.ast.statement.SQLUpdateStatement;
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlSchemaStatVisitor;
import com.alibaba.druid.stat.TableStat;
import com.alibaba.druid.stat.TableStat.Name;

import io.mycat.MycatServer;
import io.mycat.cache.CachePool;
import io.mycat.cache.DefaultLayedCachePool;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.TableConfig;
import io.mycat.route.RouteResultset;
import io.mycat.route.util.RouterUtil;
import io.mycat.util.StringUtil;

import java.sql.SQLNonTransientException;
import java.util.List;
import java.util.Map;

public class DruidUpdateParser extends DefaultDruidParser {
    @Override
    public void statementParse(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt) throws SQLNonTransientException {
        //这里限制了update分片表的个数只能有一个
        if (ctx.getTables() != null && getUpdateTableCount() > 1 && !schema.isNoSharding()) {
            String msg = "multi table related update not supported,tables:" + ctx.getTables();
            LOGGER.warn(msg);
            throw new SQLNonTransientException(msg);
        }
        MySqlUpdateStatement update = (MySqlUpdateStatement) stmt;
        String tableName = StringUtil.removeBackquote(update.getTableName().getSimpleName().toUpperCase());

        TableConfig tc = schema.getTables().get(tableName);

        if (RouterUtil.isNoSharding(schema, tableName)) {//整个schema都不分库或者该表不拆分
            RouterUtil.routeForTableMeta(rrs, schema, tableName, rrs.getStatement());
            rrs.setFinishedRoute(true);
            return;
        }

        String partitionColumn = tc.getPartitionColumn();
        String joinKey = tc.getJoinKey();
        if (tc.isGlobalTable() || (partitionColumn == null && joinKey == null)) {
            //修改全局表 update 受影响的行数
            RouterUtil.routeToMultiNode(false, rrs, tc.getDataNodes(), rrs.getStatement(), tc.isGlobalTable());
            rrs.setFinishedRoute(true);
            return;
        }


        confirmShardColumnNotUpdated(update, schema, tableName, partitionColumn, joinKey, rrs);

//		if(ctx.getTablesAndConditions().size() > 0) {
//			Map> map = ctx.getTablesAndConditions().get(tableName);
//			if(map != null) {
//				for(Map.Entry> entry : map.entrySet()) {
//					String column = entry.getKey();
//					Set value = entry.getValue();
//					if(column.toUpperCase().equals(anObject))
//				}
//			}
//			
//		}
//		System.out.println();

        if (schema.getTables().get(tableName).isGlobalTable() && ctx.getRouteCalculateUnit().getTablesAndConditions().size() > 1) {
            throw new SQLNonTransientException("global table is not supported in multi table related update " + tableName);
        }

        //在解析SQL时清空该表的主键缓存
        TableConfig tableConfig = schema.getTables().get(tableName);
        if (tableConfig != null && !tableConfig.primaryKeyIsPartionKey()) {
            String cacheName = schema.getName() +"_" + tableName;
            cacheName = cacheName.toUpperCase();
            for (CachePool value : MycatServer.getInstance().getCacheService().getAllCachePools().values()) {
                value.clearCache(cacheName);
                value.getCacheStatic().reset();
            }
        }
    }
    
    /**
     * 获取更新的表数
     * @author lian
     * @date 2016年11月2日
     * @return
     */
    private int getUpdateTableCount(){
    	Map tableMap = this.ctx.getVisitor().getTables();
    	int updateTableCount = 0;
    	for(Name _name : tableMap.keySet()){
    		
    		TableStat ts = tableMap.get(_name);
    		updateTableCount += ts.getUpdateCount();
    	}
    	return updateTableCount;
    }
    
    /*
    * 判断字段是否在SQL AST的节点中，比如 col 在 col = 'A' 中，这里要注意，一些子句中可能会在字段前加上表的别名，
    * 比如 t.col = 'A'，这两种情况， 操作符(=)左边被druid解析器解析成不同的对象SQLIdentifierExpr(无表别名)和
    * SQLPropertyExpr(有表别名)
     */
    private static boolean columnInExpr(SQLExpr sqlExpr, String colName) throws SQLNonTransientException {
        String column;
        if (sqlExpr instanceof SQLIdentifierExpr) {
            column = StringUtil.removeBackquote(((SQLIdentifierExpr) sqlExpr).getName()).toUpperCase();
        } else if (sqlExpr instanceof SQLPropertyExpr) {
            column = StringUtil.removeBackquote(((SQLPropertyExpr) sqlExpr).getName()).toUpperCase();
        } else {
            throw new SQLNonTransientException("Unhandled SQL AST node type encountered: " + sqlExpr.getClass());
        }

        return column.equals(colName.toUpperCase());
    }

    /*
    * 当前节点是不是一个子查询
    * IN (select...), ANY, EXISTS, ALL等关键字, IN (1,2,3...) 这种对应的是SQLInListExpr
     */
    private static boolean isSubQueryClause(SQLExpr sqlExpr) throws SQLNonTransientException {
        return (sqlExpr instanceof SQLInSubQueryExpr || sqlExpr instanceof SQLAnyExpr || sqlExpr instanceof SQLAllExpr
                || sqlExpr instanceof SQLQueryExpr || sqlExpr instanceof SQLExistsExpr);
    }

    /*
    * 遍历where子句的AST，寻找是否有与update子句中更新分片字段相同的条件，
    * o 如果发现有or或者xor，然后分片字段的条件在or或者xor中的，这种情况update也无法执行，比如
    *   update mytab set ptn_col = val, col1 = val1 where col1 = val11 or ptn_col = val；
    *   但是下面的这种update是可以执行的
    *   update mytab set ptn_col = val, col1 = val1 where ptn_col = val and (col1 = val11 or col2 = val2);
    * o 如果没有发现与update子句中更新分片字段相同的条件，则update也无法执行，比如
    *   update mytab set ptn_col = val， col1 = val1 where col1 = val11 and col2 = val22;
    * o 如果条件之间都是and，且有与更新分片字段相同的条件，这种情况是允许执行的。比如
    *   update mytab set ptn_col = val, col1 = val1 where ptn_col = val and col1 = val11 and col2 = val2;
    * o 对于一些特殊的运算符，比如between，not，或者子查询，遇到这些子句现在不会去检查分片字段是否在此类子句中，
    *  即使分片字段在此类子句中，现在也认为对应的update语句无法执行。
    *
    * @param whereClauseExpr   where子句的语法树AST
    * @param column   分片字段的名字
    * @param value    分片字段要被更新成的值
    * @hasOR          遍历到whereClauseExpr这个节点的时候，其上层路径中是否有OR/XOR关系运算
    *
    * @return         true，表示update不能执行，false表示可以执行
    */
    private boolean shardColCanBeUpdated(SQLExpr whereClauseExpr, String column, SQLExpr value, boolean hasOR)
            throws SQLNonTransientException {
        boolean canUpdate = false;
        boolean parentHasOR = false;

        if (whereClauseExpr == null)
            return false;

        if (whereClauseExpr instanceof SQLBinaryOpExpr) {
            SQLBinaryOpExpr nodeOpExpr = (SQLBinaryOpExpr) whereClauseExpr;
            /*
            * 条件中有or或者xor的，如果分片字段出现在or/xor的一个子句中，则此update
            * 语句无法执行
             */
            if ((nodeOpExpr.getOperator() == SQLBinaryOperator.BooleanOr) ||
                    (nodeOpExpr.getOperator() == SQLBinaryOperator.BooleanXor)) {
                parentHasOR = true;
            }
            // 发现类似 col = value 的子句
            if (nodeOpExpr.getOperator() == SQLBinaryOperator.Equality) {
                boolean foundCol;
                SQLExpr leftExpr = nodeOpExpr.getLeft();
                SQLExpr rightExpr = nodeOpExpr.getRight();

                foundCol = columnInExpr(leftExpr, column);

                // 发现col = value子句，col刚好是分片字段，比较value与update要更新的值是否一样，并且是否在or/xor子句中
                if (foundCol) {
                    if (rightExpr.getClass() != value.getClass()) {
                        throw new SQLNonTransientException("SQL AST nodes type mismatch!");
                    }

                    canUpdate = rightExpr.toString().equals(value.toString()) && (!hasOR) && (!parentHasOR);
                }
            } else if (nodeOpExpr.getOperator().isLogical()) {
                if (nodeOpExpr.getLeft() != null) {
                    if (nodeOpExpr.getLeft() instanceof SQLBinaryOpExpr) {
                        canUpdate = shardColCanBeUpdated(nodeOpExpr.getLeft(), column, value, parentHasOR);
                    }
                    // else
                    // 此子语句不是 =,>,<等关系运算符(对应的类是SQLBinaryOpExpr)。比如between X and Y
                    // 或者 NOT，或者单独的子查询，这些情况，我们不做处理
                }
                if ((!canUpdate) && nodeOpExpr.getRight() != null) {
                    if (nodeOpExpr.getRight() instanceof SQLBinaryOpExpr) {
                        canUpdate = shardColCanBeUpdated(nodeOpExpr.getRight(), column, value, parentHasOR);
                    }
                    // else
                    // 此子语句不是 =,>,<等关系运算符(对应的类是SQLBinaryOpExpr)。比如between X and Y
                    // 或者 NOT，或者单独的子查询，这些情况，我们不做处理
                }
            } else if (isSubQueryClause(nodeOpExpr)){
                // 对于子查询的检查有点复杂，这里暂时不支持
                return false;
            }
            // else
            // 其他类型的子句，忽略, 如果分片字段在这类子句中，此类情况目前不做处理，将返回false
        }
        // else
        //此处说明update的where只有一个条件，并且不是 =,>,<等关系运算符(对应的类是SQLBinaryOpExpr)。比如between X and Y
        // 或者 NOT，或者单独的子查询，这些情况，我们都不做处理

        return canUpdate;
    }

    private void confirmShardColumnNotUpdated(SQLUpdateStatement update,SchemaConfig schema,String tableName,String partitionColumn,String joinKey,RouteResultset rrs) throws SQLNonTransientException {
        List updateSetItem = update.getItems();
        if (updateSetItem != null && updateSetItem.size() > 0) {
            boolean hasParent = (schema.getTables().get(tableName).getParentTC() != null);
            for (SQLUpdateSetItem item : updateSetItem) {
                String column = StringUtil.removeBackquote(item.getColumn().toString().toUpperCase());
                //考虑别名，前面已经限制了update分片表的个数只能有一个，所以这里别名只能是分片表的
                if (column.contains(StringUtil.TABLE_COLUMN_SEPARATOR)) {
                    column = column.substring(column.indexOf(".") + 1).trim().toUpperCase();
                }
                if (partitionColumn != null && partitionColumn.equals(column)) {
                    boolean canUpdate;
                    canUpdate = ((update.getWhere() != null) && shardColCanBeUpdated(update.getWhere(),
                            partitionColumn, item.getValue(), false));

                    if (!canUpdate) {
                        String msg = "Sharding column can't be updated " + tableName + "->" + partitionColumn;
                        LOGGER.warn(msg);
                        throw new SQLNonTransientException(msg);
                    }
                }
                if (hasParent) {
                    if (column.equals(joinKey)) {
                        String msg = "Parent relevant column can't be updated " + tableName + "->" + joinKey;
                        LOGGER.warn(msg);
                        throw new SQLNonTransientException(msg);
                    }
                    rrs.setCacheAble(true);
                }
            }
        }
    }
}


================================================
FILE: src/main/java/io/mycat/route/parser/druid/impl/MysqlMethodInvocationHandler.java
================================================
package io.mycat.route.parser.druid.impl;

import java.sql.SQLNonTransientException;
import java.text.ParseException;
import java.util.Date;
import java.util.List;

import org.apache.commons.lang.time.DateFormatUtils;
import org.apache.commons.lang.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.expr.SQLCharExpr;
import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr;
import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr;

import io.mycat.route.parser.druid.SqlMethodInvocationHandler;

/**
 * mysql函数调用
 *
 * @author zhuyiqiang
 * @version 2018/9/3
 */
public class MysqlMethodInvocationHandler implements SqlMethodInvocationHandler {
    protected static final Logger LOGGER = LoggerFactory.getLogger(MysqlMethodInvocationHandler.class);
    private final String[] SUPPORT_PATTERNS = {
            "yyyy-MM-dd HH:mm:ss",
            "yyyy-MM-dd",
            "yyyy-MM-dd HH:mm"
    };

    @Override
    public String invoke(SQLMethodInvokeExpr expr) throws SQLNonTransientException {
        SQLExpr ret = doInvoke(expr);
        if (ret != null) {
            return ret.toString();
        }
        throw new SQLNonTransientException("unsupported mysql function expression: " + expr.toString());
    }

    private SQLExpr doInvoke(SQLMethodInvokeExpr expr) throws SQLNonTransientException {
        String methodName = expr.getMethodName().toUpperCase();
        switch (methodName) {
            case "NOW":
            case "SYSDATE":
            case "CURRENT_TIMESTAMP":
                return invokeNow();
            case "ADDDATE":
            case "DATE_ADD":
                return invokeAddDate(expr, false);
            case "SUBDATE":
            case "DATE_SUB":
                return invokeAddDate(expr, true);
        }
        return null;
    }

    private SQLExpr invokeNow() {
        String time = DateFormatUtils.format(new Date(), "yyyy-MM-dd HH:mm:ss");
        return new SQLIdentifierExpr(time);
    }

    private SQLExpr invokeAddDate(SQLMethodInvokeExpr expr, boolean negative) throws SQLNonTransientException {
        List parameters = expr.getParameters();
        if (parameters.size() != 2) {
            throwSyntaxError(expr);
        }
        SQLExpr p1 = parameters.get(0);
        SQLExpr p2 = parameters.get(1);
        if (p1 instanceof SQLMethodInvokeExpr) {
            p1 = doInvoke((SQLMethodInvokeExpr) p1);
        }
        if (p1 instanceof SQLCharExpr) {
            String time = ((SQLCharExpr) p1).getText();
            Integer delta = null;
            String unit = null;
            if (p2 instanceof SQLIntegerExpr) {
                delta = (Integer) ((SQLIntegerExpr) p2).getNumber();
                unit = "DAY";
			} else {
				throwSyntaxError(p2);
			}
			// else if (p2 instanceof MySqlIntervalExpr) {
//                SQLIntegerExpr value = (SQLIntegerExpr) ((MySqlIntervalExpr) p2).getValue();
//                delta = (Integer) value.getNumber();
//                unit = ((MySqlIntervalExpr) p2).getUnit().name();
//            } 

            try {
                Date date = DateUtils.parseDate(time, SUPPORT_PATTERNS);
                Date result;
                delta = negative ? -delta : delta;
                if ("MONTH".equals(unit)) {
                    result = DateUtils.addMonths(date, delta);
                } else if ("DAY".equals(unit)) {
                    result = DateUtils.addDays(date, delta);
                } else if ("HOUR".equals(unit)) {
                    result = DateUtils.addHours(date, delta);
                } else if ("MINUTE".equals(unit)) {
                    result = DateUtils.addMinutes(date, delta);
                } else if ("SECOND".equals(unit)) {
                    result = DateUtils.addSeconds(date, delta);
                } else {
                    return null;
                }
                String ret = DateFormatUtils.format(result, "yyyy-MM-dd HH:mm:ss");
                return new SQLIdentifierExpr(ret);
            } catch (ParseException e) {
                LOGGER.error("",e);
            }
        }
        return null;
    }

    private void throwSyntaxError(SQLExpr expr) throws SQLNonTransientException {
        String errMsg = "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '" + expr + "' at line 1";
        throw new SQLNonTransientException(errMsg);
    }
}

================================================
FILE: src/main/java/io/mycat/route/parser/druid/impl/OracleMethodInvocationHandler.java
================================================
package io.mycat.route.parser.druid.impl;

import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr;
import io.mycat.route.parser.druid.SqlMethodInvocationHandler;

import java.sql.SQLNonTransientException;

/**
 * oracle函数调用
 *
 * @author zhuyiqiang
 * @version 2018/9/3
 */
public class OracleMethodInvocationHandler implements SqlMethodInvocationHandler {
    @Override
    public String invoke(SQLMethodInvokeExpr expr) throws SQLNonTransientException {
        return null;
    }
}

================================================
FILE: src/main/java/io/mycat/route/parser/druid/impl/PgsqlMethodInvocationHandler.java
================================================
package io.mycat.route.parser.druid.impl;

import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr;
import io.mycat.route.parser.druid.SqlMethodInvocationHandler;

import java.sql.SQLNonTransientException;

/**
 * PostgreSQL函数调用
 *
 * @author zhuyiqiang
 * @version 2018/9/3
 */
public class PgsqlMethodInvocationHandler implements SqlMethodInvocationHandler {
    @Override
    public String invoke(SQLMethodInvokeExpr expr) throws SQLNonTransientException {
        return null;
    }
}

================================================
FILE: src/main/java/io/mycat/route/parser/primitive/FunctionParser.java
================================================
package io.mycat.route.parser.primitive;

import io.mycat.route.parser.primitive.Model.Commons;
import io.mycat.route.parser.primitive.Model.Field;
import io.mycat.route.parser.primitive.Model.Function;
import io.mycat.route.parser.primitive.Model.Identifier;
import io.mycat.util.StringUtil;

import java.sql.SQLNonTransientException;
import java.util.LinkedList;
import java.util.List;
import java.util.Stack;

/**
 * @author Hash Zhang
 * @version 1.0.0
 * @date 2016/7/26
 */
public class FunctionParser {
    public static Function parseFunction(String function) throws SQLNonTransientException {
        StringBuilder buffer = new StringBuilder();
        Stack functions = new Stack<>();

        int flag = 0;
        for (int i = 0; i < function.length(); i++) {
            char current = function.charAt(i);
            switch (current) {
                case Commons.LEFT_BRACKET:
                    if (flag == 0) {
                        String currentIdentifier = buffer.toString().trim();
                        buffer = new StringBuilder();
                        if (!StringUtil.isEmpty(currentIdentifier)) {
                            Function function1 = new Function(currentIdentifier);
                            if (!functions.empty() && functions.peek() != null) {
                                functions.peek().getArguments().add(function1);
                            }
                            functions.push(function1);
                        }
                        break;
                    }
                    buffer.append(current);
                    break;

                case Commons.ARGUMENT_SEPARATOR:
                    if (flag == 0 || flag == 3) {
                        String currentIdentifier = buffer.toString().trim();
                        buffer = new StringBuilder();
                        if (!StringUtil.isEmpty(currentIdentifier)) {
                            if (flag == 3) {
                                flag = 0;
                                Identifier identifier = new Identifier(currentIdentifier);
                                functions.peek().getArguments().add(identifier);
                            } else {
                                Field field = new Field(currentIdentifier);
                                functions.peek().getArguments().add(field);
                            }
                        }
                        break;
                    }
                    buffer.append(current);
                    break;
                case Commons.RIGHT_BRACKET:
                    if (flag != 1 && flag != 2) {
                        String currentIdentifier = buffer.toString().trim();
                        buffer = new StringBuilder();
                        if (!StringUtil.isEmpty(currentIdentifier)) {
                            if (flag == 3) {
                                flag = 0;
                                Identifier identifier = new Identifier(currentIdentifier);
                                functions.peek().getArguments().add(identifier);
                            } else {
                                Field field = new Field(currentIdentifier);
                                functions.peek().getArguments().add(field);
                            }
                        }
                        if (flag == 0) {
                            if (functions.size() == 1) {
                                return functions.pop();
                            } else {
                                functions.pop();
                            }
                        }
                        break;
                    }
                    buffer.append(current);
                    break;
                case Commons.QUOTE:
                    if (flag == 0) {
                        flag = 1;
                    } else if (flag == 1) {
                        flag = 3;
                    }
                case Commons.DOUBLE_QUOTE:
                    if (flag == 0) {
                        flag = 2;
                    } else if (flag == 2) {
                        flag = 3;
                    }
                default:
                    buffer.append(current);
            }
        }
        throw new SQLNonTransientException("Function is not in right format!");
    }

    public static List getFields(Function function){
        List fields = new LinkedList<>();
        for(Identifier identifier : function.getArguments()){
            if(identifier instanceof Field){
                fields.add(identifier.getName());
            } else if (identifier instanceof Function){
                fields.addAll(getFields((Function) identifier));
            }
        }
        return fields;
    }
    public static void main(String[] args) throws SQLNonTransientException {
        Function function = FunctionParser.parseFunction("function1(arg1,a.t,\"ast()\",function2(c.t,function3(x)))");
        System.out.println(getFields(function));
    }
}


================================================
FILE: src/main/java/io/mycat/route/parser/primitive/Model/Commons.java
================================================
package io.mycat.route.parser.primitive.Model;

/**
 * @author Hash Zhang
 * @version 1.0.0
 * @date 2016/7/26
 */
public class Commons {
    public final static char ARGUMENT_SEPARATOR = ',';
    public final static char DEPENDENCY_SEPARATOR = '.';
    public final static char LEFT_BRACKET = '(';
    public final static char RIGHT_BRACKET = ')';
    public final static char SLASH = '\\';
    public final static char QUOTE = '\'';
    public final static char DOUBLE_QUOTE = '\"';
}


================================================
FILE: src/main/java/io/mycat/route/parser/primitive/Model/Field.java
================================================
package io.mycat.route.parser.primitive.Model;

/**
 * @author Hash Zhang
 * @version 1.0.0
 * @date 2016/7/26
 */
public class Field extends Identifier {
    public Field(String name) {
        super(name);
    }
}


================================================
FILE: src/main/java/io/mycat/route/parser/primitive/Model/Function.java
================================================
package io.mycat.route.parser.primitive.Model;

import java.util.LinkedList;
import java.util.List;

/**
 * @author Hash Zhang
 * @version 1.0.0
 * @date 2016/7/26
 */
public class Function extends Identifier {
    private final List arguments;

    public Function(String name) {
        super(name);
        this.arguments = new LinkedList<>();
    }

    public List getArguments() {
        return arguments;
    }
}


================================================
FILE: src/main/java/io/mycat/route/parser/primitive/Model/Identifier.java
================================================
package io.mycat.route.parser.primitive.Model;

/**
 * @author Hash Zhang
 * @version 1.0.0
 * @date 2016/7/26
 */
public class Identifier {
    private final String name;

    public Identifier(String name) {
        this.name = name;
    }

    public String getName() {
        return name;
    }
}


================================================
FILE: src/main/java/io/mycat/route/parser/util/ArrayUtil.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.parser.util;

/**
 * @author mycat
 */
public class ArrayUtil {
    public static boolean equals(String str1, String str2) {
        if (str1 == null) {
            return str2 == null;
        }
        return str1.equals(str2);
    }

    public static boolean contains(String[] list, String str) {
        if (list == null) {
            return false;
        }
        for (String string : list) {
            if (equals(str, string)) {
                return true;
            }
        }
        return false;
    }

}

================================================
FILE: src/main/java/io/mycat/route/parser/util/CharTypes.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.parser.util;
/**
 * @author mycat
 * @author mycat
 */
public class CharTypes {
    private final static boolean[] hexFlags = new boolean[256];
    static {
        for (char c = 0; c < hexFlags.length; ++c) {
            if (c >= 'A' && c <= 'F') {
                hexFlags[c] = true;
            } else if (c >= 'a' && c <= 'f') {
                hexFlags[c] = true;
            } else if (c >= '0' && c <= '9') {
                hexFlags[c] = true;
            }
        }
    }

    public static boolean isHex(char c) {
        return c < 256 && hexFlags[c];
    }

    public static boolean isDigit(char c) {
        return c >= '0' && c <= '9';
    }

    private final static boolean[] identifierFlags = new boolean[256];
    static {
        for (char c = 0; c < identifierFlags.length; ++c) {
            if (c >= 'A' && c <= 'Z') {
                identifierFlags[c] = true;
            } else if (c >= 'a' && c <= 'z') {
                identifierFlags[c] = true;
            } else if (c >= '0' && c <= '9') {
                identifierFlags[c] = true;
            }
        }
        //  identifierFlags['`'] = true;
        identifierFlags['_'] = true;
        identifierFlags['$'] = true;
    }

    public static boolean isIdentifierChar(char c) {
        return c > identifierFlags.length || identifierFlags[c];
    }

    private final static boolean[] whitespaceFlags = new boolean[256];
    static {
        whitespaceFlags[' '] = true;
        whitespaceFlags['\n'] = true;
        whitespaceFlags['\r'] = true;
        whitespaceFlags['\t'] = true;
        whitespaceFlags['\f'] = true;
        whitespaceFlags['\b'] = true;
    }

    /**
     * @return false if {@link MySQLLexer#EOI}
     */
    public static boolean isWhitespace(char c) {
        return c <= whitespaceFlags.length && whitespaceFlags[c];
    }

}

================================================
FILE: src/main/java/io/mycat/route/parser/util/PageSQLUtil.java
================================================
package io.mycat.route.parser.util;

import java.util.List;

import com.alibaba.druid.sql.PagerUtils;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLOrderBy;
import com.alibaba.druid.sql.ast.SQLOver;
import com.alibaba.druid.sql.ast.expr.SQLAggregateExpr;
import com.alibaba.druid.sql.ast.expr.SQLAllColumnExpr;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator;
import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
import com.alibaba.druid.sql.ast.expr.SQLNumberExpr;
import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.ast.statement.SQLSelectItem;
import com.alibaba.druid.sql.ast.statement.SQLSelectQuery;
import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource;
import com.alibaba.druid.sql.ast.statement.SQLTableSource;
import com.alibaba.druid.sql.dialect.db2.ast.stmt.DB2SelectQueryBlock;
import com.alibaba.druid.sql.dialect.db2.parser.DB2StatementParser;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser;
import com.alibaba.druid.sql.dialect.postgresql.ast.stmt.PGSelectQueryBlock;
import com.alibaba.druid.sql.dialect.postgresql.parser.PGSQLStatementParser;
import com.alibaba.druid.sql.dialect.sqlserver.ast.SQLServerSelectQueryBlock;
import com.alibaba.druid.sql.dialect.sqlserver.parser.SQLServerStatementParser;
import com.alibaba.druid.util.JdbcConstants;

/**
 * Created by magicdoom on 2015/3/15.
 */
public class PageSQLUtil
{
    public static String convertLimitToNativePageSql(String dbType, String sql, int offset, int count)
    {
		if (JdbcConstants.ORACLE.name().equalsIgnoreCase(dbType))
        {
            OracleStatementParser oracleParser = new OracleStatementParser(sql);
            SQLSelectStatement oracleStmt = (SQLSelectStatement) oracleParser.parseStatement();
            return PagerUtils.limit(oracleStmt.getSelect(), JdbcConstants.ORACLE, offset, count);
		} else if (JdbcConstants.SQL_SERVER.name().equalsIgnoreCase(dbType))
        {
            SQLServerStatementParser oracleParser = new SQLServerStatementParser(sql);
            SQLSelectStatement sqlserverStmt = (SQLSelectStatement) oracleParser.parseStatement();
            SQLSelect select = sqlserverStmt.getSelect();
            SQLOrderBy orderBy=  select.getOrderBy() ;
            if(orderBy==null)
            {
                SQLSelectQuery sqlSelectQuery=      select.getQuery();
                if(sqlSelectQuery instanceof SQLServerSelectQueryBlock)
                {
                    SQLServerSelectQueryBlock sqlServerSelectQueryBlock= (SQLServerSelectQueryBlock) sqlSelectQuery;
                    SQLTableSource from=       sqlServerSelectQueryBlock.getFrom();
                    if("limit".equalsIgnoreCase(from.getAlias()))
                    {
                        from.setAlias(null);
                    }
                }
                SQLOrderBy newOrderBy=new SQLOrderBy(new SQLIdentifierExpr("(select 0)"));
                select.setOrderBy(newOrderBy);

            }

            return 	PagerUtils.limit(select, JdbcConstants.SQL_SERVER, offset, count)  ;
        }
		else if (JdbcConstants.DB2.name().equalsIgnoreCase(dbType))
        {
            DB2StatementParser db2Parser = new DB2StatementParser(sql);
            SQLSelectStatement db2Stmt = (SQLSelectStatement) db2Parser.parseStatement();

			return limitDB2(db2Stmt.getSelect(), JdbcConstants.DB2.name(), offset, count);
		} else if (JdbcConstants.POSTGRESQL.name().equalsIgnoreCase(dbType))
        {
            PGSQLStatementParser pgParser = new PGSQLStatementParser(sql);
            SQLSelectStatement pgStmt = (SQLSelectStatement) pgParser.parseStatement();
            SQLSelect select = pgStmt.getSelect();
            SQLSelectQuery query= select.getQuery();
            if(query instanceof PGSelectQueryBlock)
            {
                PGSelectQueryBlock pgSelectQueryBlock= (PGSelectQueryBlock) query;
                pgSelectQueryBlock.setOffset(null);
                pgSelectQueryBlock.setLimit(null);

            }
            return PagerUtils.limit(select, JdbcConstants.POSTGRESQL, offset, count);

		} else if (JdbcConstants.MYSQL.name().equalsIgnoreCase(dbType))
        {
            MySqlStatementParser pgParser = new MySqlStatementParser(sql);
            SQLSelectStatement pgStmt = (SQLSelectStatement) pgParser.parseStatement();
            SQLSelect select = pgStmt.getSelect();
            SQLSelectQuery query= select.getQuery();
            if(query instanceof MySqlSelectQueryBlock)
            {
                MySqlSelectQueryBlock pgSelectQueryBlock= (MySqlSelectQueryBlock) query;
                pgSelectQueryBlock.setLimit(null);
            }
            return PagerUtils.limit(select, JdbcConstants.MYSQL, offset, count);
        }

        return sql;

    }
    private static String limitDB2(SQLSelect select, String dbType, int offset, int count)
    {
        SQLSelectQuery query = select.getQuery();

        SQLBinaryOpExpr gt = new SQLBinaryOpExpr(new SQLIdentifierExpr("ROWNUM"), //
                SQLBinaryOperator.GreaterThan, //
                new SQLNumberExpr(offset), //
                JdbcConstants.DB2);
        SQLBinaryOpExpr lteq = new SQLBinaryOpExpr(new SQLIdentifierExpr("ROWNUM"), //
                SQLBinaryOperator.LessThanOrEqual, //
                new SQLNumberExpr(count + offset), //
                JdbcConstants.DB2);
        SQLBinaryOpExpr pageCondition = new SQLBinaryOpExpr(gt, SQLBinaryOperator.BooleanAnd, lteq, JdbcConstants.DB2);

        if (query instanceof SQLSelectQueryBlock)
        {
            DB2SelectQueryBlock queryBlock = (DB2SelectQueryBlock) query;

            List selectItemList = queryBlock.getSelectList();
            for (int i = 0; i < selectItemList.size(); i++)
            {
                SQLSelectItem sqlSelectItem = selectItemList.get(i);
                SQLExpr expr = sqlSelectItem.getExpr();
                String alias = sqlSelectItem.getAlias();
                if (expr instanceof SQLAllColumnExpr && alias == null)
                {
                    //未加别名会报语法错误
                    sqlSelectItem.setExpr(new SQLPropertyExpr(new SQLIdentifierExpr("XXYY"), "*"));
                    queryBlock.getFrom().setAlias("XXYY");
                }
            }

//      此处生成order by的顺序不对
//   if (offset <= 0) {
//                queryBlock.setFirst(new SQLNumberExpr(count));
//                return SQLUtils.toSQLString(select, dbType);
//            }

            SQLAggregateExpr aggregateExpr = new SQLAggregateExpr("ROW_NUMBER");
            SQLOrderBy orderBy = select.getOrderBy();
            aggregateExpr.setOver(new SQLOver(orderBy));
            select.setOrderBy(null);

            queryBlock.getSelectList().add(new SQLSelectItem(aggregateExpr, "ROWNUM"));

            DB2SelectQueryBlock countQueryBlock = new DB2SelectQueryBlock();
            countQueryBlock.getSelectList().add(new SQLSelectItem(new SQLAllColumnExpr()));

            countQueryBlock.setFrom(new SQLSubqueryTableSource(select, "XX"));

            countQueryBlock.setWhere(pageCondition);

            return SQLUtils.toSQLString(countQueryBlock, dbType);
        }

        DB2SelectQueryBlock countQueryBlock = new DB2SelectQueryBlock();
        countQueryBlock.getSelectList().add(new SQLSelectItem(new SQLPropertyExpr(new SQLIdentifierExpr("XX"), "*")));
        SQLAggregateExpr aggregateExpr = new SQLAggregateExpr("ROW_NUMBER");
        SQLOrderBy orderBy = select.getOrderBy();
        aggregateExpr.setOver(new SQLOver(orderBy));
        select.setOrderBy(null);
        countQueryBlock.getSelectList().add(new SQLSelectItem(aggregateExpr, "ROWNUM"));

        countQueryBlock.setFrom(new SQLSubqueryTableSource(select, "XX"));

        if (offset <= 0)
        {
            return SQLUtils.toSQLString(countQueryBlock, dbType);
        }

        DB2SelectQueryBlock offsetQueryBlock = new DB2SelectQueryBlock();
        offsetQueryBlock.getSelectList().add(new SQLSelectItem(new SQLAllColumnExpr()));
        offsetQueryBlock.setFrom(new SQLSubqueryTableSource(new SQLSelect(countQueryBlock), "XXX"));
        offsetQueryBlock.setWhere(pageCondition);

        return SQLUtils.toSQLString(offsetQueryBlock, dbType);
    }

}


================================================
FILE: src/main/java/io/mycat/route/parser/util/Pair.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.parser.util;

/**
 * (created at 2010-7-21)
 * 
 * @author mycat
 */
public final class Pair {

    private final K key;
    private final V value;

    public Pair(K key, V value) {
        this.key = key;
        this.value = value;
    }

    public K getKey() {
        return key;
    }

    public V getValue() {
        return value;
    }

    @Override
    public String toString() {
        StringBuilder sb = new StringBuilder();
        sb.append("(").append(key).append(", ").append(value).append(")");
        return sb.toString();
    }

    private static final int HASH_CONST = 37;

    @Override
    public int hashCode() {
        int hash = 17;
        if (key == null) {
            hash += HASH_CONST;
        } else {
            hash = hash << 5 + hash << 1 + hash + key.hashCode();
        }
        if (value == null) {
            hash += HASH_CONST;
        } else {
            hash = hash << 5 + hash << 1 + hash + value.hashCode();
        }
        return hash;
    }

    @SuppressWarnings("rawtypes")
    @Override
    public boolean equals(Object obj) {
        if (this == obj) {
            return true;
        }
        if (!(obj instanceof Pair)) {
            return false;
        }
        Pair that = (Pair) obj;
        return isEquals(this.key, that.key) && isEquals(this.value, that.value);
    }

    private boolean isEquals(Object o1, Object o2) {
        if (o1 == o2) {
            return true;
        }
        if (o1 == null) {
            return o2 == null;
        }
        return o1.equals(o2);
    }

}

================================================
FILE: src/main/java/io/mycat/route/parser/util/PairUtil.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.parser.util;

/**
 * @author mycat
 */
public final class PairUtil {
    private static final int DEFAULT_INDEX = -1;

    /**
     * "2" -> (0,2)

     * "1:2" -> (1,2)

     * "1:" -> (1,0)

     * "-1:" -> (-1,0)

     * ":-1" -> (0,-1)

     * ":" -> (0,0)

     */
    public static Pair sequenceSlicing(String slice) {
        int ind = slice.indexOf(':');
        if (ind < 0) {
            int i = Integer.parseInt(slice.trim());
            if (i >= 0) {
                return new Pair(0, i);
            } else {
                return new Pair(i, 0);
            }
        }
        String left = slice.substring(0, ind).trim();
        String right = slice.substring(1 + ind).trim();
        int start, end;
        if (left.length() <= 0) {
            start = 0;
        } else {
            start = Integer.parseInt(left);
        }
        if (right.length() <= 0) {
            end = 0;
        } else {
            end = Integer.parseInt(right);
        }
        return new Pair(start, end);
    }

    /**
     * 
     * 将名字和索引用进行分割 当src = "offer_group[4]", l='[', r=']'时，
     * 返回的Piar("offer", 4);
     * 当src = "offer_group", l='[', r=']'时， 
     * 返回Pair("offer",-1);
     * 

     */
    public static Pair splitIndex(String src, char l, char r) {
        if (src == null) {
            return null;
        }
        int length = src.length();
        if (length == 0) {
            return new Pair("", DEFAULT_INDEX);
        }
        if (src.charAt(length - 1) != r) {
            return new Pair(src, DEFAULT_INDEX);
        }
        int offset = src.lastIndexOf(l);
        if (offset == -1) {
            return new Pair(src, DEFAULT_INDEX);
        }
        int index = DEFAULT_INDEX;
        try {
            index = Integer.parseInt(src.substring(offset + 1, length - 1));
        } catch (NumberFormatException e) {
            return new Pair(src, DEFAULT_INDEX);
        }
        return new Pair(src.substring(0, offset), index);
    }

}

================================================
FILE: src/main/java/io/mycat/route/parser/util/ParseString.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.parser.util;

/**
 * @author mycat
 */
public final class ParseString {

    private static final byte[] EMPTY_BYTE_ARRAY = new byte[0];

    public static byte[] hexString2Bytes(char[] hexString, int offset, int length) {
        if (hexString == null) {
            return null;
        }
        if (length == 0) {
            return EMPTY_BYTE_ARRAY;
        }
        boolean odd = length << 31 == Integer.MIN_VALUE;
        byte[] bs = new byte[odd ? (length + 1) >> 1 : length >> 1];
        for (int i = offset, limit = offset + length; i < limit; ++i) {
            char high, low;
            if (i == offset && odd) {
                high = '0';
                low = hexString[i];
            } else {
                high = hexString[i];
                low = hexString[++i];
            }
            int b;
            switch (high) {
            case '0':
                b = 0;
                break;
            case '1':
                b = 0x10;
                break;
            case '2':
                b = 0x20;
                break;
            case '3':
                b = 0x30;
                break;
            case '4':
                b = 0x40;
                break;
            case '5':
                b = 0x50;
                break;
            case '6':
                b = 0x60;
                break;
            case '7':
                b = 0x70;
                break;
            case '8':
                b = 0x80;
                break;
            case '9':
                b = 0x90;
                break;
            case 'a':
            case 'A':
                b = 0xa0;
                break;
            case 'b':
            case 'B':
                b = 0xb0;
                break;
            case 'c':
            case 'C':
                b = 0xc0;
                break;
            case 'd':
            case 'D':
                b = 0xd0;
                break;
            case 'e':
            case 'E':
                b = 0xe0;
                break;
            case 'f':
            case 'F':
                b = 0xf0;
                break;
            default:
                throw new IllegalArgumentException("illegal hex-string: " + new String(hexString, offset, length));
            }
            switch (low) {
            case '0':
                break;
            case '1':
                b += 1;
                break;
            case '2':
                b += 2;
                break;
            case '3':
                b += 3;
                break;
            case '4':
                b += 4;
                break;
            case '5':
                b += 5;
                break;
            case '6':
                b += 6;
                break;
            case '7':
                b += 7;
                break;
            case '8':
                b += 8;
                break;
            case '9':
                b += 9;
                break;
            case 'a':
            case 'A':
                b += 10;
                break;
            case 'b':
            case 'B':
                b += 11;
                break;
            case 'c':
            case 'C':
                b += 12;
                break;
            case 'd':
            case 'D':
                b += 13;
                break;
            case 'e':
            case 'E':
                b += 14;
                break;
            case 'f':
            case 'F':
                b += 15;
                break;
            default:
                throw new IllegalArgumentException("illegal hex-string: " + new String(hexString, offset, length));
            }
            bs[(i - offset) >> 1] = (byte) b;
        }
        return bs;
    }

}

================================================
FILE: src/main/java/io/mycat/route/parser/util/ParseUtil.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.parser.util;

import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlInsertStatement;
import com.alibaba.druid.sql.parser.SQLStatementParser;
import io.mycat.route.parser.druid.MycatStatementParser;

/**
 * @author mycat
 */
public final class ParseUtil {

    public static boolean isEOF(char c) {
        return (c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == ';');
    }

	public static boolean isEOF(String stmt, int offset) {
		for (; offset < stmt.length(); offset++) {
			if (!ParseUtil.isEOF(stmt.charAt(offset))) {
				return false;
			}
		}
		return true;
	}

    public static String parseString(String stmt) {
    	 int offset = stmt.indexOf('=');
         if (offset != -1 && stmt.length() > ++offset) {
             String txt = stmt.substring(offset).trim();
             return txt;
         }
         return null;
    }
    
    public static long getSQLId(String stmt) {
        int offset = stmt.indexOf('=');
        if (offset != -1 && stmt.length() > ++offset) {
            String id = stmt.substring(offset).trim();
            try {
                return Long.parseLong(id);
            } catch (NumberFormatException e) {
            }
        }
        return 0L;
    }

    public static String changeInsertAddSlot(String sql,int slotValue)
    {
        SQLStatementParser parser = new MycatStatementParser(sql);
        MySqlInsertStatement insert = (MySqlInsertStatement) parser.parseStatement();
        insert.getColumns().add(new SQLIdentifierExpr("_slot") );
        insert.getValues().getValues().add(new SQLIntegerExpr(slotValue))  ;
        return insert.toString();
    }
    /**
     * 'abc'
     * 
     * @param offset stmt.charAt(offset) == first '
     */
    private static String parseString(String stmt, int offset) {
        StringBuilder sb = new StringBuilder();
        loop: for (++offset; offset < stmt.length(); ++offset) {
            char c = stmt.charAt(offset);
            if (c == '\\') {
                switch (c = stmt.charAt(++offset)) {
                case '0':
                    sb.append('\0');
                    break;
                case 'b':
                    sb.append('\b');
                    break;
                case 'n':
                    sb.append('\n');
                    break;
                case 'r':
                    sb.append('\r');
                    break;
                case 't':
                    sb.append('\t');
                    break;
                case 'Z':
                    sb.append((char) 26);
                    break;
                default:
                    sb.append(c);
                }
            } else if (c == '\'') {
                if (offset + 1 < stmt.length() && stmt.charAt(offset + 1) == '\'') {
                    ++offset;
                    sb.append('\'');
                } else {
                    break loop;
                }
            } else {
                sb.append(c);
            }
        }
        return sb.toString();
    }

    /**
     * "abc"
     * 
     * @param offset stmt.charAt(offset) == first "
     */
    private static String parseString2(String stmt, int offset) {
        StringBuilder sb = new StringBuilder();
        loop: for (++offset; offset < stmt.length(); ++offset) {
            char c = stmt.charAt(offset);
            if (c == '\\') {
                switch (c = stmt.charAt(++offset)) {
                case '0':
                    sb.append('\0');
                    break;
                case 'b':
                    sb.append('\b');
                    break;
                case 'n':
                    sb.append('\n');
                    break;
                case 'r':
                    sb.append('\r');
                    break;
                case 't':
                    sb.append('\t');
                    break;
                case 'Z':
                    sb.append((char) 26);
                    break;
                default:
                    sb.append(c);
                }
            } else if (c == '"') {
                if (offset + 1 < stmt.length() && stmt.charAt(offset + 1) == '"') {
                    ++offset;
                    sb.append('"');
                } else {
                    break loop;
                }
            } else {
                sb.append(c);
            }
        }
        return sb.toString();
    }

    /**
     * AS `abc`
     * 
     * @param offset stmt.charAt(offset) == first `
     */
    private static String parseIdentifierEscape(String stmt, int offset) {
        StringBuilder sb = new StringBuilder();
        loop: for (++offset; offset < stmt.length(); ++offset) {
            char c = stmt.charAt(offset);
            if (c == '`') {
                if (offset + 1 < stmt.length() && stmt.charAt(offset + 1) == '`') {
                    ++offset;
                    sb.append('`');
                } else {
                    break loop;
                }
            } else {
                sb.append(c);
            }
        }
        return sb.toString();
    }

    /**
     * @param aliasIndex for AS id, index of 'i'
     */
    public static String parseAlias(String stmt, final int aliasIndex) {
        if (aliasIndex < 0 || aliasIndex >= stmt.length()) {
            return null;
        }
        switch (stmt.charAt(aliasIndex)) {
        case '\'':
            return parseString(stmt, aliasIndex);
        case '"':
            return parseString2(stmt, aliasIndex);
        case '`':
            return parseIdentifierEscape(stmt, aliasIndex);
        default:
            int offset = aliasIndex;
            for (; offset < stmt.length() && CharTypes.isIdentifierChar(stmt.charAt(offset)); ++offset) {
                ;
            }
            return stmt.substring(aliasIndex, offset);
        }
    }

    /**
     * 解析注释，返回stmt中注释结尾的index
     * @param stmt
     * @param offset
     * @return
     */
    public static int comment(String stmt, int offset) {
        int len = stmt.length();
        int n = offset;
        switch (stmt.charAt(n)) {
        case '/':
            if (len > ++n && stmt.charAt(n++) == '*' && len > n + 1) {
                for (int i = n; i < len; ++i) {
                    if (stmt.charAt(i) == '*') {
                        int m = i + 1;
                        if (len > m && stmt.charAt(m) == '/') {
                            return m;
                        }
                    }
                }
            }
            break;
        case '#':
            for (int i = n + 1; i < len; ++i) {
                if (stmt.charAt(i) == '\n') {
                    return i;
                }
            }
            break;
        }
        return offset;
    }

    public static boolean currentCharIsSep(String stmt, int offset) {
        if (stmt.length() > offset) {
            switch (stmt.charAt(offset)) {
            case ' ':
            case '\t':
            case '\r':
            case '\n':
                return true;
            default:
                return false;
            }
        }
        return true;
    }

    /*****
     * 检查下一个字符是否为分隔符，并把偏移量加1
     */
    public static boolean nextCharIsSep(String stmt, int offset) {
        return currentCharIsSep(stmt, ++offset);
    }

    /*****
     * 检查下一个字符串是否为期望的字符串，并把偏移量移到从offset开始计算，expectValue之后的位置
     * 
     * @param stmt 被解析的sql
     * @param offset 被解析的sql的当前位置
     * @param nextExpectedString 在stmt中准备查找的字符串
     * @param checkSepChar 当找到expectValue值时，是否检查其后面字符为分隔符号
     * @return 如果包含指定的字符串，则移动相应的偏移量，否则返回值=offset
     */
    public static int nextStringIsExpectedWithIgnoreSepChar(String stmt,
                                                            int offset,
                                                            String nextExpectedString,
                                                            boolean checkSepChar) {
        if (nextExpectedString == null || nextExpectedString.length() < 1) {
            return offset;
        }
        int i = offset;
        int index = 0;
        char expectedChar;
        char actualChar;
        boolean isSep;
        for (; i < stmt.length() && index < nextExpectedString.length(); ++i) {
            if (index == 0) {
                isSep = currentCharIsSep(stmt, i);
                if (isSep) {
                    continue;
                }
            }
            actualChar = stmt.charAt(i);
            expectedChar = nextExpectedString.charAt(index++);
            if (actualChar != expectedChar) {
                return offset;
            }
        }
        if (index == nextExpectedString.length()) {
            boolean ok = true;
            if (checkSepChar) {
                ok = nextCharIsSep(stmt, i);
            }
            if (ok) {
                return i;
            }
        }
        return offset;
    }

    private static final String JSON = "json";
    private static final String EQ = "=";

    //private static final String WHERE = "where";
    //private static final String SET = "set";

    /**********
     * 检查下一个字符串是否json= *
     * 
     * @param stmt 被解析的sql
     * @param offset 被解析的sql的当前位置
     * @return 如果包含指定的字符串，则移动相应的偏移量，否则返回值=offset
     */
    public static int nextStringIsJsonEq(String stmt, int offset) {
        int i = offset;

        // / drds 之后的符号
        if (!currentCharIsSep(stmt, ++i)) {
            return offset;
        }

        // json 串
        int k = nextStringIsExpectedWithIgnoreSepChar(stmt, i, JSON, false);
        if (k <= i) {
            return offset;
        }
        i = k;

        // 等于符号
        k = nextStringIsExpectedWithIgnoreSepChar(stmt, i, EQ, false);
        if (k <= i) {
            return offset;
        }
        return i;
    }

    public static int move(String stmt, int offset, int length) {
        int i = offset;
        for (; i < stmt.length(); ++i) {
            switch (stmt.charAt(i)) {
            case ' ':
            case '\t':
            case '\r':
            case '\n':
                continue;
            case '/':
            case '#':
                i = comment(stmt, i);
                continue;
            default:
                return i + length;
            }
        }
        return i;
    }

    public static boolean compare(String s, int offset, char[] keyword) {
        if (s.length() >= offset + keyword.length) {
            for (int i = 0; i < keyword.length; ++i, ++offset) {
                if (Character.toUpperCase(s.charAt(offset)) != keyword[i]) {
                    return false;
                }
            }
            return true;
        }
        return false;
    }

	public static boolean isSpace(char space) {
		return space == ' ' || space == '\r' || space == '\n' || space == '\t';
	}

	public static int findNextBreak(String sql) {
		return findNextBreak(sql, false);
	}

	/**
	 * @param sql
	 * @param inProcedure
	 * @return
	 */
	private static int findNextBreak(String sql, boolean inProcedure) {
		// is the char in apostrophe
		boolean inApostrophe = false;
		// is the char after procedure begin
		boolean procedureBegin = false;
		String bodyLeft = null;
		for (int i = 0; i < sql.length(); i++) {
			char c = sql.charAt(i);
			switch (c) {
			case '\\':
				i++;
				break;
			case '\'':
			case '\"':
				if (!inApostrophe) {
					inApostrophe = true;
					bodyLeft = String.valueOf(c);
				} else {
					String rightTest = String.valueOf(c);
					if (rightTest.equals(bodyLeft)) {
						inApostrophe = false;
					}
				}
				break;
			case ';':
				if (!inApostrophe && !procedureBegin) {
					return i;
				}
				break;
			case 'p':
			case 'P':
				if (!inApostrophe) {
					i = getProcedureEndPos(sql, i);
				}
				break;
			case 'B':
				if (inProcedure && !inApostrophe && isBegin(sql, i)) {
					procedureBegin = true;
					i = i + 4;
				}
				break;
			case 'E':
				if (!inApostrophe && procedureBegin && isEnd(sql, i)) {
					procedureBegin = false;
					i = i + 2;
				}
				break;
			default:
			}
		}
		return sql.length() - 1;
	}

	private static boolean isBegin(String stmt, int offset) {
		if (offset <= 0 || !ParseUtil.isSpace(stmt.charAt(offset - 1)) || stmt.length() <= offset + "EGIN ".length()) {
			return false;
		}
		char c1 = stmt.charAt(++offset);
		char c2 = stmt.charAt(++offset);
		char c3 = stmt.charAt(++offset);
		char c4 = stmt.charAt(++offset);
		char c5 = stmt.charAt(++offset);
		return c1 == 'E' && c2 == 'G' && c3 == 'I' && c4 == 'N' && ParseUtil.isSpace(c5);
	}

	private static boolean isEnd(String stmt, int offset) {
		if (offset <= 0 || !ParseUtil.isSpace(stmt.charAt(offset - 1)) || stmt.length() <= offset + "ND".length()) {
			return false;
		}
		char c1 = stmt.charAt(++offset);
		char c2 = stmt.charAt(++offset);
		if (c1 == 'N' && c2 == 'D') {
			offset++;
			for (; offset < stmt.length(); offset++) {
				char tmp = stmt.charAt(offset);
				if (ParseUtil.isSpace(tmp)) {
					continue;
				}
				return tmp == ';';
			}
			return true;
		}
		return false;
	}

	private static int getProcedureEndPos(String stmt, int offset) {
		if (stmt.length() > offset + "ROCEDURE ".length()) {
			char c1 = stmt.charAt(++offset);
			char c2 = stmt.charAt(++offset);
			char c3 = stmt.charAt(++offset);
			char c4 = stmt.charAt(++offset);
			char c5 = stmt.charAt(++offset);
			char c6 = stmt.charAt(++offset);
			char c7 = stmt.charAt(++offset);
			char c8 = stmt.charAt(++offset);
			if ((c1 == 'r' || c1 == 'R') && (c2 == 'o' || c2 == 'O') && (c3 == 'c' || c3 == 'C')
					&& (c4 == 'e' || c4 == 'E') && (c5 == 'd' || c5 == 'D') && (c6 == 'u' || c6 == 'U')
					&& (c7 == 'r' || c7 == 'R') && (c8 == 'e' || c8 == 'E')) {
				String testSql = stmt.substring(++offset).toUpperCase();
				// offset + length -1 for checking ';' outside
				return offset - 1 + findNextBreak(testSql, true);
			}
		}
		return offset;
	}
}


================================================
FILE: src/main/java/io/mycat/route/parser/util/SQLParserUtils.java
================================================
package io.mycat.route.parser.util;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.Stack;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.junit.Assert;

public class SQLParserUtils {
	
	class State{
		Integer state = -1;
	}

	State state = new State();			//当前处于哪个部分 0: select 1: from 2:where 

    Stack stateStack = new Stack();

	Map tables = new HashMap();
	boolean tableFlag = false;		//在From部分出现关键字from join 逗号后的是表名
    
	public Map parse(String sql){
		tables.clear();
		stateStack.clear();
		state = null;
		
	    tableFlag = false;
		
		boolean sFlag = false;			//单引号 
		boolean dFlag = false;			//双引号计数器
		Scanner reader=new Scanner(sql);
		reader.useDelimiter(" ");
		String value;
		while(reader.hasNext()){
			value = reader.next().toLowerCase();
			//前面已经出现单引号，在再次出现单引号之前不做任何处理
			if (sFlag){
				if (value.endsWith("'")&& getCount(value,"'")==1){
					sFlag = false;
					continue;
				}else if (value.indexOf("'")!=-1){
					value = value.substring(value.indexOf("'")+1);
					sFlag = false;
				}else {
					continue;
				}
			}
			//前面已经出现双引号，在再次出现双引号之前不做任何处理
			if (dFlag){
				if (value.endsWith("\"")&& getCount(value,"\"")==1){
					dFlag = false;
					continue;
				}else if (value.indexOf("\"")!=-1){
					value = value.substring(value.indexOf("\"")+1);
					dFlag = false;
				}else {
					continue;
				}
			}
			//单引号在select，where部分不做处理
			if (state != null && state.state !=1 && getCount(value,"'")%2==1){
				sFlag = true;
				continue;
			}
			if (state != null && state.state !=1 && getCount(value,"\"")%2==1){
				dFlag = true;
				continue;
			}
			
			//SELECT关键字
			if (value.equals("select") || value.equals("(select")){
				//if (state != null) 
				state = new State();
				state.state = 0;
				stateStack.push(state);	//入栈
				continue;
			}
			
			
			//FROM关键字
			if (value.equals("from") || value.equals("into")|| value.equals("join")){
				state.state = 1;
				tableFlag = true;
				continue;
			}
			//From部分出现逗号后面是表名
			if (state.state == 1 && value.equals(",")){
				tableFlag = true;
				continue;
			}
			if (state.state == 1 && tableFlag == true){
				getTableName(value);
				continue;
			}

			if (state.state == 1 && tableFlag == false){
				if (!value.startsWith("),") &&(value.equals(",")|| value.endsWith(","))){
					tableFlag = true;
					continue;
				}else if (!value.startsWith("),") && value.indexOf(",")!=-1){
					getTableName(value);
					continue;
				}
					
			}
			
			//WHERE关键字
			if (value.equals("where")){
				state.state = 2;
				continue;
			}
			
			if (value.endsWith("(select")){

				stateStack.push(state);
				state = new State();
				state.state = 0;
				continue;
			}
			if ( value.equals(")")|| value.startsWith("),")){
				stateStack.pop();
				state = stateStack.peek();
				tableFlag = value.endsWith(",")?true:false;
				if (state.state ==1){
					getTableName(value);
				}
				continue;
			}


		}
			
		return tables;
	}
	
	private void getTableName(String str){
		String[] t = str.split(",");
		for (int i=tableFlag?0:1; i0 && !t[i].trim().equals("(")) {
					tables.put(t[i], "");
				}
			}
		}
		if (!str.endsWith(",")) {
			tableFlag = false;
		}
	}
	
	
	
	public static int getCount(String str,String match){
	       int count = 0;
	        int index = 0;
	        while((index=str.indexOf(match,index))!=-1){
	            index = index+match.length();
	            count++;
	        }
	        return count;
	}
	
	
	private boolean test(String sql,String[] tables){
		

		Map result = parse(sql);
		if (result.size() != tables.length) {
			return false;
		}
		for (String tmp : tables){
			if (result.get(tmp.toLowerCase())==null) {
				return false;
			}
		}
		return true;
			
	}
	private static final String sql1 = "select t3.*,ztd3.TypeDetailName as UseStateName\n" +
            "from\n" +
            "( \n" +
            " select t4.*,ztd4.TypeDetailName as AssistantUnitName\n" +
            " from\n" +
            " (\n" +
            "  select t2.*,ztd2.TypeDetailName as UnitName \n" +
            "  from\n" +
            "  (\n" +
            "   select t1.*,ztd1.TypeDetailName as MaterielAttributeName \n" +
            "   from \n" +
            "   (\n" +
            "    select m.*,r.RoutingName,u.username,mc.MoldClassName\n" +
            "    from dbo.D_Materiel as m\n" +
            "    left join dbo.D_Routing as r\n" +
            "    on m.RoutingID=r.RoutingID\n" +
            "    left join dbo.D_MoldClass as mc\n" +
            "    on m.MoldClassID=mc.MoldClassID\n" +
            "    left join dbo.D_User as u\n" +
            "    on u.UserId=m.AddUserID\n" +
            "   )as t1\n" +
            "   left join dbo.D_Type_Detail as ztd1 \n" +
            "   on t1.MaterielAttributeID=ztd1.TypeDetailID\n" +
            "  )as t2\n" +
            "  left join dbo.D_Type_Detail as ztd2 \n" +
            "  on t2.UnitID=ztd2.TypeDetailID\n" +
            " ) as t4\n" +
            " left join dbo.D_Type_Detail as ztd4 \n" +
            " on t4.AssistantUnitID=ztd4.TypeDetailID\n" +
            ")as t3\n" +
            "left join dbo.D_Type_Detail as ztd3 \n" +
            "on t3.UseState=ztd3.TypeDetailID";
	public static void main(String[] args) {
		SQLParserUtils parser = new SQLParserUtils();
		//parser.parse("select 'select * from C , D',' select * from E' from B");
		//if (true) return;
		List list = new ArrayList();
		list.add(new String[]{"select * from B","B"});
		list.add(new String[]{"select * from B,C","B,C"});
		list.add(new String[]{"select * from B ,C","B,C"});
		list.add(new String[]{"select * from B , C","B,C"});
		list.add(new String[]{"select * from B a","B"});
		list.add(new String[]{"select * from B a,C,D","B,C,D"});
		list.add(new String[]{"select * from B a,C e ,D","B,C,D"});
		list.add(new String[]{"select * from B a,C e ,D f","B,C,D"});
		list.add(new String[]{"select * from B,(select * from C),D","B,C,D"});
		list.add(new String[]{"select * from B, (select * from C),D","B,C,D"});
		list.add(new String[]{"select * from B, ( select * from C),D","B,C,D"});
		list.add(new String[]{"select * from B, ( select * from C),D,E","B,C,D,E"});
		list.add(new String[]{"select * from B,(select * from C ),D","B,C,D"});
		list.add(new String[]{"select * from B,(select * from C ) ,D","B,C,D"});
		list.add(new String[]{"select * from B,(select * from C), D","B,C,D"});
		list.add(new String[]{"select * from B,(select * from C ) , D","B,C,D"});
		list.add(new String[]{"select * from B,(select * from C ) , (select * from D )","B,C,D"});
		list.add(new String[]{"select * from B,(select * from C ) , (select * from D ),E","B,C,D,E"});
		list.add(new String[]{"select * from B,(select C.ID , D.ID from C ) , (select * from D ),E","B,C,D,E"});

		list.add(new String[]{"select (select C.ID,D.ID from C ) from B, D","B,C,D"});
		list.add(new String[]{"select (select C.ID,D.ID from C ) , E from B, D","B,C,D"});
		list.add(new String[]{"select (select C.ID,D.ID from C ), E from B, D","B,C,D"});
		list.add(new String[]{"select (select C.ID,D.ID from C ),E from B, D","B,C,D"});
		list.add(new String[]{"select a from t1 union select b from t2","t1,t2"});
		

		list.add(new String[]{"select * from B where C =1","B"});
		list.add(new String[]{"select * from B where C = (select 1 from D)","B,D"});
		list.add(new String[]{"select * from B where C = (select 1 from D) AND E = (select 2 from F)","B,D,F"});
		
		
		list.add(new String[]{"select * from B INNER JOIN C ON C.ID = D.ID","B,C"});
		list.add(new String[]{"select * from B INNER JOIN C ON C.ID = D.ID INNER JOIN E ON 1=1","B,C,E"});
		list.add(new String[]{"select * from B INNER JOIN C ON C.ID = (select G,H FROM I) INNER JOIN E ON 1=1","B,C,E,I"});
		list.add(new String[]{"select * from B INNER JOIN C ON C.ID = (select G,H FROM I ) INNER JOIN E ON 1=1","B,C,E,I"});
		list.add(new String[]{"select * from B INNER JOIN C ON C.ID = ( select G,H FROM I ) INNER JOIN E ON 1=1","B,C,E,I"});
		
		

		list.add(new String[]{"select 'select * from C' from B","B"});
		list.add(new String[]{"select 'select * from C,D' from B","B"});
		list.add(new String[]{"select 'select * from C , D',E from B","B"});
		list.add(new String[]{"select 'select * from C , D',' select * from E' from B","B"});
		list.add(new String[]{"select 'select * from C , D','F',' select * from E' from B","B"});
		list.add(new String[]{"select 'select * from C , D',' F',' select * from E' from B","B"});
		list.add(new String[]{"select 'select * from C , D',' F ',' select * from E' from B","B"});
		

		list.add(new String[]{"select * from 'B'","'B'"});
		list.add(new String[]{"select * from 'B','C'","'B','C'"});
		
		list.add(new String[]{sql1,"dbo.D_Materiel,dbo.D_Routing,dbo.D_MoldClass,dbo.D_Type_Detail,dbo.D_User"});
		//String sql  = "select ' form \"' * from \"B\",C where a='c'";
		//String sql  = "select ' form \"' * from \"B\",C";
		for (String[] tmp :list){
			
			Assert.assertTrue(tmp[0],parser.test(tmp[0],tmp[1].split(",")));
			{
				System.out.println(tmp[0]+"--->"+tmp[1]);
				Map tables = parser.parse(tmp[0]);
				System.out.print("表名：");
				for (String key :tables.keySet()) {
					System.out.println(key);
				}
			}
		}
		// TODO Auto-generated method stub

	}

}


================================================
FILE: src/main/java/io/mycat/route/parser/util/WildcardUtil.java
================================================
package io.mycat.route.parser.util;

public class WildcardUtil {
  
  public static String wildcard(String name) {
    if (name.startsWith("`")) {
      name = name.replaceAll("`", "");
    } else if (name.startsWith("\"")) {
      name = name.replaceAll("\"", "");
    } else if (name.startsWith("'")) {
      name = name.replaceAll("'", "");
    }
    return name;
  }

  public static void wildcards(String[] names) {
    for (int i = 0; i < names.length; i++) {
      names[i] = wildcard(names[i]);
    }
  }
}


================================================
FILE: src/main/java/io/mycat/route/sequence/BatchInsertSequence.java
================================================
package io.mycat.route.sequence;

import io.mycat.route.sequence.handler.*;
import org.slf4j.Logger; import org.slf4j.LoggerFactory;

import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr;
import com.alibaba.druid.sql.ast.statement.SQLInsertStatement.ValuesClause;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlInsertStatement;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;

import io.mycat.MycatServer;
import io.mycat.cache.LayerCachePool;
import io.mycat.catlets.Catlet;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.SystemConfig;
import io.mycat.config.model.TableConfig;
import io.mycat.route.RouteResultset;
import io.mycat.route.factory.RouteStrategyFactory;
import io.mycat.server.ServerConnection;
import io.mycat.server.parser.ServerParse;
import io.mycat.sqlengine.EngineCtx;
import io.mycat.util.StringUtil;

/**
 * 执行批量插入sequence Id
 * @author 兵临城下
 * @date 2015/03/20
 */
public class BatchInsertSequence implements Catlet {
	private static final Logger LOGGER = LoggerFactory.getLogger(BatchInsertSequence.class);
	
	private RouteResultset rrs;//路由结果集
	private String executeSql;//接收执行处理任务的sql
	private SequenceHandler sequenceHandler;//sequence处理对象
	
	//重新路由使用
	private SystemConfig sysConfig;
	private SchemaConfig schema;
	private int sqltype; 
	private String charset; 
	private ServerConnection sc;
	private LayerCachePool cachePool;

	@Override
	public void processSQL(String sql, EngineCtx ctx) {
		throw new UnsupportedOperationException("batch insert is not supported because the mycat 1.6 does not support multi statements.");
//		try {
//
//			getRoute(executeSql);
//			RouteResultsetNode[] nodes = rrs.getNodes();
//			if (nodes == null || nodes.length == 0 || nodes[0].getName() == null
//					|| nodes[0].getName().equals("")) {
//				ctx.getSession().getSource().writeErrMessage(ErrorCode.ER_NO_DB_ERROR,
//						"No dataNode found ,please check tables defined in schema:"
//								+ ctx.getSession().getSource().getSchema());
//				return;
//			}
//
//			sc.getSession2().execute(rrs, sqltype);//将路由好的数据执行入库
//
//		} catch (Exception e) {
//			LOGGER.error("BatchInsertSequence.processSQL(String sql, EngineCtx ctx)",e);
//		}
	}

	@Override
	public void route(SystemConfig sysConfig, SchemaConfig schema, int sqlType,
			String realSQL, String charset, ServerConnection sc,
			LayerCachePool cachePool) {
		int rs = ServerParse.parse(realSQL);
		this.sqltype = rs & 0xff;
		this.sysConfig=sysConfig; 
		this.schema=schema;
		this.charset=charset; 
		this.sc=sc;	
		this.cachePool=cachePool;	
		
		try {
			MySqlStatementParser parser = new MySqlStatementParser(realSQL);	 
			SQLStatement statement = parser.parseStatement();
			MySqlInsertStatement insert = (MySqlInsertStatement)statement;
			if(insert.getValuesList()!=null){
				String tableName = StringUtil.getTableName(realSQL).toUpperCase();
				TableConfig tableConfig = schema.getTables().get(tableName);
				String primaryKey = tableConfig.getPrimaryKey();//获得表的主键字段
				
				SQLIdentifierExpr sqlIdentifierExpr = new SQLIdentifierExpr();
				sqlIdentifierExpr.setName(primaryKey);
				insert.getColumns().add(sqlIdentifierExpr);
				
				if(sequenceHandler == null){
					int seqHandlerType = MycatServer.getInstance().getConfig().getSystem().getSequenceHandlerType();
					switch(seqHandlerType){
						case SystemConfig.SEQUENCEHANDLER_MYSQLDB:
							sequenceHandler = IncrSequenceMySQLHandler.getInstance();
							break;
						case SystemConfig.SEQUENCEHANDLER_LOCALFILE:
							sequenceHandler = IncrSequencePropHandler.getInstance();
							break;
						case SystemConfig.SEQUENCEHANDLER_LOCAL_TIME:
							sequenceHandler = IncrSequenceTimeHandler.getInstance();
							break;
						case SystemConfig.SEQUENCEHANDLER_ZK_DISTRIBUTED:
							sequenceHandler = DistributedSequenceHandler.getInstance(MycatServer.getInstance().getConfig().getSystem());
							break;
						case SystemConfig.SEQUENCEHANDLER_ZK_GLOBAL_INCREMENT:
							sequenceHandler = IncrSequenceZKHandler.getInstance();
							break;
						default:
							throw new java.lang.IllegalArgumentException("Invalid sequnce handler type "+seqHandlerType);
					}
				}
				
				for(ValuesClause vc : insert.getValuesList()){
					SQLIntegerExpr sqlIntegerExpr = new SQLIntegerExpr();
					long value = sequenceHandler.nextId(tableName.toUpperCase());
					sqlIntegerExpr.setNumber(value);//插入生成的sequence值
					vc.addValue(sqlIntegerExpr);
				}
				
				String insertSql = insert.toString();
				this.executeSql = insertSql;
			}
			
		} catch (Exception e) {
			LOGGER.error("BatchInsertSequence.route(......)",e);
		}
	}
	
	/**
	 * 根据sql获得路由执行结果
	 * @param sql
	 */
	private void getRoute(String sql){
		try {
			rrs =RouteStrategyFactory.getRouteStrategy().route(sysConfig, schema, sqltype,sql,charset, sc, cachePool);
		} catch (Exception e) {
			LOGGER.error("BatchInsertSequence.getRoute(String sql)",e);
		}
	}

}


================================================
FILE: src/main/java/io/mycat/route/sequence/handler/DistributedSequenceHandler.java
================================================
package io.mycat.route.sequence.handler;


import io.mycat.config.loader.console.ZookeeperPath;
import io.mycat.config.loader.zkprocess.comm.ZkConfig;
import io.mycat.config.loader.zkprocess.comm.ZkParamCfg;
import io.mycat.config.model.SystemConfig;
import io.mycat.route.util.PropertiesUtil;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.recipes.leader.CancelLeadershipException;
import org.apache.curator.framework.recipes.leader.LeaderSelector;
import org.apache.curator.framework.recipes.leader.LeaderSelectorListenerAdapter;
import org.apache.curator.framework.state.ConnectionState;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.curator.utils.CloseableUtils;
import org.apache.zookeeper.CreateMode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.Closeable;
import java.io.IOException;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;

/**
 * 基于ZK与本地配置的分布式ID生成器(可以通过ZK获取集群（机房）唯一InstanceID，也可以通过配置文件配置InstanceID)
 * ID结构：long 64位，ID最大可占63位
 * |current time millis(微秒时间戳38位,可以使用17年)|clusterId（机房或者ZKid，通过配置文件配置5位）|instanceId（实例ID，可以通过ZK或者配置文件获取，5位）|threadId（线程ID，9位）|increment(自增,6位)
 * 一共63位，可以承受单机房单机器单线程1000*(2^6)=640000的并发。
 * 无悲观锁，无强竞争，吞吐量更高
 * 

 * 配置文件：sequence_distributed_conf.properties
 * 只要配置里面：INSTANCEID=ZK就是从ZK上获取InstanceID
 *
 * @author Hash Zhang
 * @version 1.0
 * @time 00:08:03 2016/5/3
 */
public class DistributedSequenceHandler extends LeaderSelectorListenerAdapter implements Closeable, SequenceHandler {
    protected static final Logger LOGGER = LoggerFactory.getLogger(DistributedSequenceHandler.class);
    private static final String SEQUENCE_DB_PROPS = "sequence_distributed_conf.properties";
    private static DistributedSequenceHandler instance;

    private final long timestampBits = 38L;
    private final long clusterIdBits = 5L;
    private final long instanceIdBits = 5L;
    private final long threadIdBits = 9L;
    private final long incrementBits = 6L;

    private final long timestampMask = (1L << timestampBits) - 1L;

    private final long incrementShift = 0L;
    private final long threadIdShift = incrementShift + incrementBits;
    private final long instanceIdShift = threadIdShift + threadIdBits;
    private final long clusterIdShift = instanceIdShift + instanceIdBits;
    private final long timestampShift = clusterIdShift + clusterIdBits;

    private final long maxIncrement = 1L << incrementBits;
    private final long maxThreadId = 1L << threadIdBits;
    private final long maxinstanceId = 1L << instanceIdBits;
    private final long maxclusterId = 1L << instanceIdBits;

    private volatile long instanceId;
    private long clusterId;

    private ThreadLocal threadInc = new ThreadLocal<>();
    private ThreadLocal threadLastTime = new ThreadLocal<>();
    private ThreadLocal threadID = new ThreadLocal<>();
    private long nextID = 0L;

    private final static String PATH = ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_BASE.getKey()
            + io.mycat.config.loader.zkprocess.comm.ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_CLUSTERID)
            + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SEQUENCE.getKey();
    // private final static String PATH = "/mycat/sequence";
    private final static String INSTANCE_PATH = ZookeeperPath.ZK_SEPARATOR.getKey()
            + ZookeeperPath.FLOW_ZK_PATH_SEQUENCE_INSTANCE.getKey();
    private final static String LEADER_PATH = ZookeeperPath.ZK_SEPARATOR.getKey()
            + ZookeeperPath.FLOW_ZK_PATH_SEQUENCE_LEADER.getKey();
    private SystemConfig mycatConfig;
    private String ID;

    private int mark[];
    private volatile boolean isLeader = false;
    private volatile String slavePath;
    // 配置是否载入好
    private volatile boolean ready = false;

    private CuratorFramework client;

    private LeaderSelector leaderSelector;

    private final ScheduledExecutorService timerExecutor = Executors.newSingleThreadScheduledExecutor();
    private ScheduledExecutorService leaderExecutor;
    private final long SELF_CHECK_PERIOD = 10L;

    public static DistributedSequenceHandler getInstance(SystemConfig systemConfig) {
        if (instance == null) {
            instance = new DistributedSequenceHandler(systemConfig);
        }
        return instance;
    }

    public long getClusterId() {
        return clusterId;
    }

    public void setClusterId(long clusterId) {
        this.clusterId = clusterId;
    }

    public LeaderSelector getLeaderSelector() {
        return leaderSelector;
    }

    public long getInstanceId() {
        return instanceId;
    }

    public void setInstanceId(long instanceId) {
        this.instanceId = instanceId;
    }

    public CuratorFramework getClient() {
        return client;
    }

    public void setClient(CuratorFramework client) {
        this.client = client;
    }

    public DistributedSequenceHandler(SystemConfig mycatConfig) {
        this.mycatConfig = mycatConfig;
        ID = mycatConfig.getBindIp() + mycatConfig.getServerPort();
    }

    public void load() {
        // load sequnce properties
        Properties props = PropertiesUtil.loadProps(SEQUENCE_DB_PROPS);
        if ("ZK".equals(props.getProperty("INSTANCEID"))) {
            initializeZK(ZkConfig.getInstance().getZkURL());
        } else {
            this.instanceId = Long.parseLong(props.getProperty("INSTANCEID"));
            this.ready = true;
        }
        this.clusterId = Long.valueOf(props.getProperty("CLUSTERID"));

    }

    public void initializeZK(String zkAddress) {
        this.client = CuratorFrameworkFactory.newClient(zkAddress, new ExponentialBackoffRetry(1000, 3));
        this.client.start();
        try {
            if (client.checkExists().forPath(PATH.concat(INSTANCE_PATH)) == null) {
                client.create().creatingParentContainersIfNeeded().forPath(PATH.concat(INSTANCE_PATH));
            }
        } catch (Exception e) {
            // do nothing
        }
        this.leaderSelector = new LeaderSelector(client, PATH.concat(LEADER_PATH), this);
        this.leaderSelector.autoRequeue();
        this.leaderSelector.start();
        Runnable runnable = new Runnable() {
            @Override
            public void run() {
                try {
                    while (leaderSelector.getLeader() == null) {
                        Thread.currentThread().yield();
                    }
                    if (!leaderSelector.hasLeadership()) {
                        isLeader = false;
                        if (slavePath != null && client.checkExists().forPath(slavePath) != null) {
                            return;
                        }
                        slavePath = client.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL_SEQUENTIAL)
                                .forPath(PATH.concat("/instance/node"), "ready".getBytes());
                        while ("ready".equals(new String(client.getData().forPath(slavePath)))) {
                            Thread.currentThread().yield();
                        }
                        instanceId = Long.parseLong(new String(client.getData().forPath(slavePath)));
                        ready = true;
                    }
                } catch (Exception e) {
                    LOGGER.warn("Caught exception while handling zk!", e);
                }
            }
        };
        timerExecutor.scheduleAtFixedRate(runnable, 1L, 10L, TimeUnit.SECONDS);
    }

    @Override
    public long nextId(String prefixName) {
        // System.out.println(instanceId);
        while (!ready) {
            try {
                Thread.sleep(50);
            } catch (InterruptedException e) {
                LOGGER.warn("Unexpected thread interruption!");
                Thread.currentThread().interrupt();
            }
        }
        long time = System.currentTimeMillis();
        if (threadLastTime.get() == null) {
            threadLastTime.set(time);
        }
        if (threadInc.get() == null) {
            threadInc.set(0L);
        }
        if (threadID.get() == null) {
            threadID.set(getNextThreadID());
        }
        long a = threadInc.get();
        if ((a + 1L) >= maxIncrement) {
            if (threadLastTime.get() == time) {
                time = blockUntilNextMillis(time);
            }
            threadInc.set(0L);
        } else {
            threadInc.set(a + 1L);
        }
        threadLastTime.set(time);
        return ((time & timestampMask) << timestampShift) | (((threadID.get() % maxThreadId) << threadIdShift))
                | (instanceId << instanceIdShift) | (clusterId << clusterIdShift) | a;
    }

    private synchronized Long getNextThreadID() {
        long i = nextID;
        nextID++;
        return i;
    }

    private long blockUntilNextMillis(long time) {
        while (System.currentTimeMillis() == time) {
        }
        return System.currentTimeMillis();
    }

    @Override
    public void stateChanged(CuratorFramework client, ConnectionState newState) {
        if (newState == ConnectionState.SUSPENDED || newState == ConnectionState.LOST) {
            this.isLeader = false;
            leaderExecutor.shutdownNow();
            throw new CancelLeadershipException();
        }
    }

    @Override
    public void takeLeadership(final CuratorFramework curatorFramework) {
        this.isLeader = true;
        this.instanceId = 1;
        this.ready = true;
        this.mark = new int[(int) maxinstanceId];
        List children = null;
        try {
            if (this.slavePath != null) {
                client.delete().forPath(slavePath);
            }
            if (client.checkExists().forPath(PATH.concat(INSTANCE_PATH)) != null) {
                children = client.getChildren().forPath(PATH.concat(INSTANCE_PATH));
            }
            if (children != null) {
                for (String child : children) {
                    String data = new String(
                            client.getData().forPath(PATH.concat(INSTANCE_PATH.concat("/").concat(child))));
                    if (!"ready".equals(data)) {
                        mark[Integer.parseInt(data)] = 1;
                    }
                }
            }
        } catch (Exception e) {
            LOGGER.warn("Caught exception while handling zk!", e);
        }

        leaderExecutor = Executors.newSingleThreadScheduledExecutor();
        leaderExecutor.scheduleAtFixedRate(new Runnable() {
            @Override
            public void run() {
                try {
                    while (!client.isStarted()) {
                        Thread.currentThread().yield();
                    }
                    List children = client.getChildren().forPath(PATH.concat(INSTANCE_PATH));
                    int mark2[] = new int[(int) maxinstanceId];
                    for (String child : children) {
                        String data = new String(client.getData().forPath(PATH.concat("/instance/" + child)));
                        if ("ready".equals(data)) {
                            int i = nextFree();
                            client.setData().forPath(PATH.concat(INSTANCE_PATH.concat("/").concat(child)),
                                    ("" + i).getBytes());
                            mark2[i] = 1;
                        } else {
                            mark2[Integer.parseInt(data)] = 1;
                        }
                    }
                    mark = mark2;
                } catch (Exception e) {
                    LOGGER.warn("Caught exception while handling zk!", e);
                }
            }
        }, 0L, 3L, TimeUnit.SECONDS);
        while (true) {
            Thread.currentThread().yield();
        }
    }

    private int nextFree() {
        for (int i = 0; i < mark.length; i++) {
            if (i == 1) {
                continue;
            }
            if (mark[i] != 1) {
                mark[i] = 1;
                return i;
            }
        }
        return -1;
    }

    @Override
    public void close() throws IOException {
        CloseableUtils.closeQuietly(this.leaderSelector);
        CloseableUtils.closeQuietly(this.client);
    }
}


================================================
FILE: src/main/java/io/mycat/route/sequence/handler/HttpIncrSequenceHandler.java
================================================
package io.mycat.route.sequence.handler;

import io.mycat.route.util.PropertiesUtil;
import okhttp3.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.Properties;

public class HttpIncrSequenceHandler implements SequenceHandler {
    protected static final Logger LOGGER = LoggerFactory
            .getLogger(HttpIncrSequenceHandler.class);
    private final String url;
    private long count = 0;
    private long limit = -1;

    public HttpIncrSequenceHandler() {
        Properties props = PropertiesUtil.loadProps("sequence_http_conf.properties");
        this.url = (String) props.get("url");
    }

    @Override
    public synchronized long nextId(String prefixName) {
        if (count > limit) {
            try {
                OkHttpClient client = new OkHttpClient.Builder()
                        .build();
                RequestBody body = new FormBody.Builder()
                        .add("prefixName", prefixName)
                        .add("count",Long.toString(count))
                        .add("limit", Long.toString(limit))
                        .build();
                Request request = new Request.Builder()
                        .url(url)
                        .post(body)
                        .build();
                final Call call = client.newCall(request);
                Response response = call.execute();
                ResponseBody body1 = response.body();
                String s = new String(body1.bytes());
                String[] split = s.split(",");
                this.count = Long.parseLong(split[0]);
                this.limit = Long.parseLong(split[1]);
            } catch (Throwable e) {
                LOGGER.error("", e);
                throw new RuntimeException(e);
            }
        }
        return count++;
    }
}

================================================
FILE: src/main/java/io/mycat/route/sequence/handler/IncrSequenceBDBHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.sequence.handler;

import java.util.Map;

/**
 * BDB 数据库实现递增序列号
 * 
 * @author Michael
 * @time Create on 2013-12-29 下午11:05:44
 * @version 1.0
 */
public class IncrSequenceBDBHandler extends IncrSequenceHandler {

	private static class IncrSequenceBDBHandlerHolder {
		private static final IncrSequenceBDBHandler instance = new IncrSequenceBDBHandler();
	}

	public static IncrSequenceBDBHandler getInstance() {
		return IncrSequenceBDBHandlerHolder.instance;
	}

	private IncrSequenceBDBHandler() {
	}

	@Override
	public Map getParaValMap(String prefixName) {
		
		return null;
	}

	@Override
	public Boolean fetchNextPeriod(String prefixName) {
		
		return null;
	}

	@Override
	public Boolean updateCURIDVal(String prefixName, Long val) {
		
		return null;
	}

}

================================================
FILE: src/main/java/io/mycat/route/sequence/handler/IncrSequenceHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.sequence.handler;

import java.util.Map;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * 递增序列号处理器
 * 
 * @author Michael
 * @time Create on 2013-12-29 下午10:42:39
 * @version 1.0
 */
public abstract class IncrSequenceHandler implements SequenceHandler {

	public static final Logger logger = LoggerFactory
			.getLogger(IncrSequenceHandler.class);

	public static final String FILE_NAME = "sequence_conf.properties";

	public static final String KEY_HIS_NAME = ".HISIDS";// 1-10000,50001-60000
	public static final String KEY_MIN_NAME = ".MINID";// 1
	public static final String KEY_MAX_NAME = ".MAXID";// 10000
	public static final String KEY_CUR_NAME = ".CURID";// 888

	public abstract Map getParaValMap(String prefixName);

	public abstract Boolean updateCURIDVal(String prefixName, Long val);

	public abstract Boolean fetchNextPeriod(String prefixName);

	@Override
	public long nextId(String prefixName) {
		Map paraMap = this.getParaValMap(prefixName);
		if (null == paraMap) {
			throw new RuntimeException("fetch Param Values error.");
		}
		Long nextId = Long.parseLong(paraMap.get(prefixName + KEY_CUR_NAME)) + 1;
		Long maxId = Long.parseLong(paraMap.get(prefixName + KEY_MAX_NAME));
		if (nextId > maxId) {
			fetchNextPeriod(prefixName);
			return nextId(prefixName);
		}
		updateCURIDVal(prefixName, nextId);
		return nextId.longValue();

	}
}

================================================
FILE: src/main/java/io/mycat/route/sequence/handler/IncrSequenceMySQLHandler.java
================================================
package io.mycat.route.sequence.handler;

import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import io.mycat.MycatServer;
import io.mycat.backend.BackendConnection;
import io.mycat.backend.datasource.PhysicalDBNode;
import io.mycat.backend.mysql.nio.handler.ResponseHandler;
import io.mycat.config.MycatConfig;
import io.mycat.config.util.ConfigException;
import io.mycat.net.mysql.ErrorPacket;
import io.mycat.net.mysql.RowDataPacket;
import io.mycat.route.RouteResultsetNode;
import io.mycat.route.util.PropertiesUtil;
import io.mycat.server.parser.ServerParse;

public class IncrSequenceMySQLHandler implements SequenceHandler {

	protected static final Logger LOGGER = LoggerFactory
			.getLogger(IncrSequenceMySQLHandler.class);

	private static final String SEQUENCE_DB_PROPS = "sequence_db_conf.properties";
	protected static final String errSeqResult = "-999999,null";
	protected static Map latestErrors = new ConcurrentHashMap();
	private final FetchMySQLSequnceHandler mysqlSeqFetcher = new FetchMySQLSequnceHandler();

	private static class IncrSequenceMySQLHandlerHolder {
		private static final IncrSequenceMySQLHandler instance = new IncrSequenceMySQLHandler();
	}

	public static IncrSequenceMySQLHandler getInstance() {
		return IncrSequenceMySQLHandlerHolder.instance;
	}

	private IncrSequenceMySQLHandler() {

		load();
	}
	//加载配置文件
	public void load() {
		// load sequnce properties
		Properties props = PropertiesUtil.loadProps(SEQUENCE_DB_PROPS);
		removeDesertedSequenceVals(props);
		putNewSequenceVals(props);
	}

	//移除旧的配置
	private void removeDesertedSequenceVals(Properties props) {
		Iterator> i = seqValueMap.entrySet()
				.iterator();
		while (i.hasNext()) {
			Map.Entry entry = i.next();
			if (!props.containsKey(entry.getKey())) {
				i.remove();
			}
		}
	}

	private void putNewSequenceVals(Properties props) {
		for (Map.Entry entry : props.entrySet()) {
			String seqName = (String) entry.getKey();
			String dataNode = (String) entry.getValue();
			if (!seqValueMap.containsKey(seqName)) {
				seqValueMap.put(seqName, new SequenceVal(seqName, dataNode));
			} else {
				seqValueMap.get(seqName).dataNode = dataNode;
			}
		}
	}

	/**
	 * save sequnce -> curval
	 */
	private ConcurrentHashMap seqValueMap = new ConcurrentHashMap();

	@Override
	public long nextId(String seqName) {
		SequenceVal seqVal = seqValueMap.get(seqName);
		if (seqVal == null) {
			throw new ConfigException("can't find definition for sequence :"
					+ seqName);
		}
		if (!seqVal.isSuccessFetched()) {
			//从数据库获取
			return getSeqValueFromDB(seqVal);
		} else {
			//已经设置 获取下一个有效id
			return getNextValidSeqVal(seqVal);
		}

	}
	//获取有效的sequence
	private Long getNextValidSeqVal(SequenceVal seqVal) {
		Long nexVal = seqVal.nextValue();
		//当前id有效返回 无效则从数据库获取
		if (seqVal.isNexValValid(nexVal)) {
			return nexVal;
		} else {
//			seqVal.fetching.compareAndSet(true, false);
			return getSeqValueFromDB(seqVal);
		}
	}

	//在这边用锁。
	/*
	 * 1.尝试获取fetching锁， 
	 *    成功： 
	 *       再次判断下一个值是否有效，有效直接返回
	 *       否则 调用waitFinish请求后端的请求
	 *     失败：
	 *        进行等待，等待一定次数 尝试从数据库获取
	 *             别的后端请求已经成功获取，调用getNextValidSeqVal获取下一个可用值
	 *  
	 * 
	 * */
	private long getSeqValueFromDB(SequenceVal seqVal) {
		if (LOGGER.isDebugEnabled()) {
			LOGGER.debug("get next segement of sequence from db for sequnce:"
					+ seqVal.seqName + " curVal " + seqVal.curVal);
		}
		//设置正在获取
		boolean isLock = seqVal.fetching.compareAndSet(false, true);
		if(isLock) {
			//判断当前的是否有效。
			if(seqVal.successFetched == true) {
				Long nexVal = seqVal.nextValue();
				if (seqVal.isNexValValid(nexVal)) {
					seqVal.fetching.compareAndSet(true, false);
					return nexVal;
				}
			}
						
			//发起请求sql 等待到返回  或者进行
			Long[] values = seqVal.fetchSequenceFromDB( mysqlSeqFetcher, 1, true); //只有一个线程可以进 并且有重试机制。
			if (values == null) {
				throw new RuntimeException("can't fetch sequnce in db,sequnce :"
						+ seqVal.seqName + " detail:"
						+ mysqlSeqFetcher.getLastestError(seqVal.seqName));
			} else {
					seqVal.setCurValue(values[0]); 
					seqVal.maxSegValue = values[1];
					seqVal.successFetched = true; //设置successFetched
					return values[0];
			
			}
		} else {
			long count = 0 ;
			//正在获取 ，或者还未返回
			while(seqVal.fetching.get() || seqVal.successFetched == false){
				try {										
					Thread.sleep(10);
					if(++count > 10000L) {
						return this.getSeqValueFromDB(seqVal);
					}
				} catch (InterruptedException e) {
					e.printStackTrace();
					return this.getSeqValueFromDB(seqVal);
				}
			}
			return this.getNextValidSeqVal(seqVal);
		}	
	}

}

class FetchMySQLSequnceHandler implements ResponseHandler {
	private static final Logger LOGGER = LoggerFactory
			.getLogger(FetchMySQLSequnceHandler.class);

	public void execute(SequenceVal seqVal) {
		MycatConfig conf = MycatServer.getInstance().getConfig();
		PhysicalDBNode mysqlDN = conf.getDataNodes().get(seqVal.dataNode);
		try {
			if (LOGGER.isDebugEnabled()) {
				LOGGER.debug("execute in datanode " + seqVal.dataNode
						+ " for fetch sequnce sql " + seqVal.sql);
			}
			// 修正获取seq的逻辑，在读写分离的情况下只能走写节点。修改Select模式为Update模式。
			mysqlDN.getConnection(mysqlDN.getDatabase(), true,
					new RouteResultsetNode(seqVal.dataNode, ServerParse.UPDATE,
							seqVal.sql), this, seqVal);
		} catch (Exception e) {
			seqVal.dbfinished = true;

			LOGGER.warn("get connection err " + e);
		}

	}

	public String getLastestError(String seqName) {
		return IncrSequenceMySQLHandler.latestErrors.get(seqName);
	}

	@Override
	public void connectionAcquired(BackendConnection conn) {

		conn.setResponseHandler(this);
		try {
			//发起sql请求
			SequenceVal sequenceVal = ((SequenceVal) conn.getAttachment()) ;
			conn.query(sequenceVal.sql);
		} catch (Exception e) {
			executeException(conn, e);
		}
	}
	//连接错误
	@Override
	public void connectionError(Throwable e, BackendConnection conn) {
		((SequenceVal) conn.getAttachment()).dbfinished = true;
		LOGGER.warn("connectionError " + e);

	}
	//返回错误结果处理
	@Override
	public void errorResponse(byte[] data, BackendConnection conn) {
		SequenceVal seqVal = ((SequenceVal) conn.getAttachment());
//		seqVal.dbfinished = true;
		seqVal.setDbfinished();
		ErrorPacket err = new ErrorPacket();
		err.read(data);
		String errMsg = new String(err.message);
		LOGGER.warn("errorResponse " + err.errno + " " + errMsg);
		IncrSequenceMySQLHandler.latestErrors.put(seqVal.seqName, errMsg);
		conn.release();

	}

	@Override
	public void okResponse(byte[] ok, BackendConnection conn) {
		boolean executeResponse = conn.syncAndExcute();
		if (executeResponse) {
			((SequenceVal) conn.getAttachment()).setDbfinished();
//			((SequenceVal) conn.getAttachment()).dbfinished = true;
			conn.release();
		}

	}
	//获取一行的数据 
	@Override
	public void rowResponse(byte[] row, BackendConnection conn) {
		RowDataPacket rowDataPkg = new RowDataPacket(1);
		rowDataPkg.read(row);
		byte[] columnData = rowDataPkg.fieldValues.get(0);
		String columnVal = new String(columnData);
		SequenceVal seqVal = (SequenceVal) conn.getAttachment();
		if (IncrSequenceMySQLHandler.errSeqResult.equals(columnVal)) {
			seqVal.dbretVal = IncrSequenceMySQLHandler.errSeqResult;
			LOGGER.warn(" sequnce sql returned err value ,sequence:"
					+ seqVal.seqName + " " + columnVal + " sql:" + seqVal.sql);
		}else {
			seqVal.dbretVal = columnVal;
		}
	}
	//结果集合接受完毕
	@Override
	public void rowEofResponse(byte[] eof, BackendConnection conn) {
		SequenceVal sequenceVal = ((SequenceVal) conn.getAttachment());
		conn.release();
//		sequenceVal.dbfinished = true;
		sequenceVal.setDbfinished();
	}
	//错误返回异常处理
	private void executeException(BackendConnection c, Throwable e) {
		SequenceVal seqVal = ((SequenceVal) c.getAttachment());
//		seqVal.dbfinished = true;
		seqVal.setDbfinished();
		String errMgs=e.toString();
		LOGGER.error("{}",e);
		IncrSequenceMySQLHandler.latestErrors.put(seqVal.seqName, errMgs);
		c.close("exception:" +errMgs);

	}

	@Override
	public void writeQueueAvailable() {

	}
	//连接关闭异常处理
	@Override
	public void connectionClose(BackendConnection conn, String reason) {
		//((SequenceVal) conn.getAttachment()).dbfinished = true;
		((SequenceVal) conn.getAttachment()).setDbfinished();
		LOGGER.warn("connection closed " + conn + " reason:" + reason);
	}

	@Override
	public void fieldEofResponse(byte[] header, List fields,
			byte[] eof, BackendConnection conn) {

	}

}



================================================
FILE: src/main/java/io/mycat/route/sequence/handler/IncrSequencePropHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.sequence.handler;

import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.OutputStream;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;

/**
 * 本地prop文件实现递增序列号
 * 
 * @author Michael
 * @time Create on 2013-12-29 下午11:00:05
 * @version 1.0
 */
public class IncrSequencePropHandler extends IncrSequenceHandler {
	private String filePath;

	private static class IncrSequencePropHandlerHolder {
		private static final IncrSequencePropHandler instance = new IncrSequencePropHandler();
	}

	public static IncrSequencePropHandler getInstance() {
		return IncrSequencePropHandlerHolder.instance;
	}

	private IncrSequencePropHandler() {
		filePath = Thread.currentThread().getContextClassLoader()
				.getResource("").getPath().replaceAll("%20", " ")
				+ FILE_NAME;
		// filePath = SystemConfig.getHomePath() + "/conf/seq_gloal.properties";
	}

	@Override
	public Map getParaValMap(String prefixName) {

		Map valMap = new HashMap();
		Properties prop = new Properties();
		try {
			prop.load(new FileInputStream(filePath));
			valMap.put(prefixName + KEY_HIS_NAME,
					prop.getProperty(prefixName + KEY_HIS_NAME));
			valMap.put(prefixName + KEY_MIN_NAME,
					prop.getProperty(prefixName + KEY_MIN_NAME));
			valMap.put(prefixName + KEY_MAX_NAME,
					prop.getProperty(prefixName + KEY_MAX_NAME));
			valMap.put(prefixName + KEY_CUR_NAME,
					prop.getProperty(prefixName + KEY_CUR_NAME));
		} catch (Exception e) {
			logger.error(e.getMessage());
			return null;
		}

		return valMap;
	}

	@Override
	public Boolean fetchNextPeriod(String prefixName) {
		Properties props = new Properties();
		try {
			props.load(new FileInputStream(filePath));
			String minStr = props.getProperty(prefixName + KEY_MIN_NAME);
			String maxStr = props.getProperty(prefixName + KEY_MAX_NAME);
			String hisIDS = props.getProperty(prefixName + KEY_HIS_NAME);
			props.setProperty(prefixName + KEY_HIS_NAME,
					"".equals(hisIDS) ? minStr + "-" + maxStr : "," + minStr
							+ "-" + maxStr);
			long minId = Long.parseLong(minStr);
			long maxId = Long.parseLong(maxStr);
			props.setProperty(prefixName + KEY_MIN_NAME, (maxId + 1) + "");
			props.setProperty(prefixName + KEY_MAX_NAME,
					(maxId - minId + maxId + 1) + "");
			props.setProperty(prefixName + KEY_CUR_NAME, maxStr);
			OutputStream fos = new FileOutputStream(filePath);
			props.store(fos, "");
		} catch (Exception e) {
			logger.error(e.getLocalizedMessage());
			return false;
		}
		return true;
	}

	@Override
	public synchronized Boolean  updateCURIDVal(String prefixName, Long val) {
		Properties props = new Properties();
		try {
			props.load(new FileInputStream(filePath));
			props.setProperty(prefixName + KEY_CUR_NAME, val.longValue() + "");
			OutputStream fos = new FileOutputStream(filePath);
			props.store(fos, "");
		} catch (Exception e) {
			logger.error(e.getLocalizedMessage());
			return false;
		}
		return true;
	}

}

================================================
FILE: src/main/java/io/mycat/route/sequence/handler/IncrSequenceTimeHandler.java
================================================
package io.mycat.route.sequence.handler;

import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;

import org.slf4j.Logger; import org.slf4j.LoggerFactory;

public class IncrSequenceTimeHandler implements SequenceHandler {
    protected static final Logger LOGGER = LoggerFactory.getLogger(IncrSequenceTimeHandler.class);

	private static final String SEQUENCE_DB_PROPS = "sequence_time_conf.properties";
	private static final IncrSequenceTimeHandler instance = new IncrSequenceTimeHandler();
	private static IdWorker workey = new IdWorker(1,1);


	public static IncrSequenceTimeHandler getInstance() {
		return IncrSequenceTimeHandler.instance;
	}

	public IncrSequenceTimeHandler() {
		load();
	}


	public void load(){
		// load sequnce properties
		Properties props = loadProps(SEQUENCE_DB_PROPS);

		long workid = Long.parseLong(props.getProperty("WORKID"));
		long dataCenterId = Long.parseLong(props.getProperty("DATAACENTERID"));

		workey = new IdWorker(workid,dataCenterId);
	}
	private Properties loadProps(String propsFile){
		Properties props = new Properties();
		InputStream inp = Thread.currentThread().getContextClassLoader().getResourceAsStream(propsFile);

		if (inp == null) {
			throw new java.lang.RuntimeException("time sequnce properties not found " + propsFile);
		}
		try {
			props.load(inp);
		} catch (IOException e) {
			throw new java.lang.RuntimeException(e);
		}
		return props;
	}
	@Override
	public long nextId(String prefixName) {
		return workey.nextId();
	}


	/**
	* 64位ID (42(毫秒)+5(机器ID)+5(业务编码)+12(重复累加))
	* @author sw
	*/
	static class IdWorker {
		private final static long twepoch = 1288834974657L;
		// 机器标识位数
		private final static long workerIdBits = 5L;
		// 数据中心标识位数
		private final static long datacenterIdBits = 5L;
		// 机器ID最大值 31
		private final static long maxWorkerId = -1L ^ (-1L << workerIdBits);
		// 数据中心ID最大值 31
		private final static long maxDatacenterId = -1L ^ (-1L << datacenterIdBits);
		// 毫秒内自增位
		private final static long sequenceBits = 12L;
		// 机器ID偏左移12位
		private final static long workerIdShift = sequenceBits;
		private final static long datacenterIdShift = sequenceBits + workerIdBits;
		// 时间毫秒左移22位
		private final static long timestampLeftShift = sequenceBits + workerIdBits + datacenterIdBits;

		private final static long sequenceMask = -1L ^ (-1L << sequenceBits);

		private static long lastTimestamp = -1L;

		private long sequence = 0L;
		private final long workerId;
		private final long datacenterId;

		public IdWorker(long workerId, long datacenterId) {
			if (workerId > maxWorkerId || workerId < 0) {
				throw new IllegalArgumentException("worker Id can't be greater than %d or less than 0");
			}
			if (datacenterId > maxDatacenterId || datacenterId < 0) {
				throw new IllegalArgumentException("datacenter Id can't be greater than %d or less than 0");
			}
			this.workerId = workerId;
			this.datacenterId = datacenterId;
		}

		public synchronized long nextId() {
			long timestamp = timeGen();
			if (timestamp < lastTimestamp) {
			try {
				throw new Exception("Clock moved backwards.  Refusing to generate id for "+ (lastTimestamp - timestamp) + " milliseconds");
			} catch (Exception e) {
				LOGGER.error("error",e);
			}
			}

			if (lastTimestamp == timestamp) {
				// 当前毫秒内，则+1
				sequence = (sequence + 1) & sequenceMask;
				if (sequence == 0) {
					// 当前毫秒内计数满了，则等待下一秒
					timestamp = tilNextMillis(lastTimestamp);
				}
			} else {
				sequence = timestamp & 1;
			}
			lastTimestamp = timestamp;
			// ID偏移组合生成最终的ID，并返回ID
			long nextId = ((timestamp - twepoch) << timestampLeftShift)
					| (datacenterId << datacenterIdShift)
					| (workerId << workerIdShift) | sequence;

			return nextId;
		}

		private long tilNextMillis(final long lastTimestamp) {
			long timestamp = this.timeGen();
			while (timestamp <= lastTimestamp) {
				timestamp = this.timeGen();
			}
			return timestamp;
		}

		private long timeGen() {
			return System.currentTimeMillis();
		}



	}




}


================================================
FILE: src/main/java/io/mycat/route/sequence/handler/IncrSequenceZKHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.sequence.handler;


import io.mycat.config.loader.console.ZookeeperPath;
import io.mycat.config.loader.zkprocess.comm.ZkConfig;
import io.mycat.config.loader.zkprocess.comm.ZkParamCfg;
import io.mycat.route.util.PropertiesUtil;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreMutex;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.curator.utils.ZKPaths;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.UnsupportedEncodingException;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;

/**
 * zookeeper 实现递增序列号
 * 配置文件：sequence_conf.properties
 * 只要配置好ZK地址和表名的如下属性
 * TABLE.MINID 某线程当前区间内最小值
 * TABLE.MAXID 某线程当前区间内最大值
 * TABLE.CURID 某线程当前区间内当前值
 * 文件配置的MAXID以及MINID决定每次取得区间，这个对于每个线程或者进程都有效
 * 文件中的这三个属性配置只对第一个进程的第一个线程有效，其他线程和进程会动态读取ZK
 *
 * @author Hash Zhang
 * @version 1.0
 * @time 23:35 2016/5/6
 */
public class IncrSequenceZKHandler extends IncrSequenceHandler {
    protected static final Logger LOGGER = LoggerFactory.getLogger(IncrSequenceHandler.class);
    private final static String PATH = ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_BASE.getKey()
            + ZookeeperPath.ZK_SEPARATOR.getKey()
            + io.mycat.config.loader.zkprocess.comm.ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_CLUSTERID)
            + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SEQUENCE.getKey()
            + ZookeeperPath.ZK_SEPARATOR.getKey() + ZookeeperPath.FLOW_ZK_PATH_SEQUENCE_INCREMENT_SEQ.getKey();
    private final static String LOCK = "/lock";
    private final static String SEQ = "/seq";
    private final static IncrSequenceZKHandler instance = new IncrSequenceZKHandler();

    public static IncrSequenceZKHandler getInstance() {
        return instance;
    }

    private ThreadLocal>> tableParaValMapThreadLocal = new ThreadLocal<>();

    private CuratorFramework client;
    private ThreadLocal interProcessSemaphoreMutexThreadLocal = new ThreadLocal<>();
    private Properties props;

    public void load() {
        props = PropertiesUtil.loadProps(FILE_NAME);
        String zkAddress = ZkConfig.getInstance().getZkURL();
        try {
            initializeZK(props, zkAddress);
        } catch (Exception e) {
            LOGGER.error("Error caught while initializing ZK:" + e.getCause());
        }
    }

    public void threadLocalLoad() throws Exception {
        Enumeration enu = props.propertyNames();
        while (enu.hasMoreElements()) {
            String key = (String) enu.nextElement();
            if (key.endsWith(KEY_MIN_NAME)) {
                handle(key);
            }
        }
    }

    public void initializeZK(Properties props, String zkAddress) throws Exception {
        this.client = CuratorFrameworkFactory.newClient(zkAddress, new ExponentialBackoffRetry(1000, 3));
        this.client.start();
        this.props = props;
        Enumeration enu = props.propertyNames();
        while (enu.hasMoreElements()) {
            String key = (String) enu.nextElement();
            if (key.endsWith(KEY_MIN_NAME)) {
                handle(key);
            }
        }
    }

    private void handle(String key) throws Exception {
        String table = key.substring(0, key.indexOf(KEY_MIN_NAME));
        InterProcessSemaphoreMutex interProcessSemaphoreMutex = interProcessSemaphoreMutexThreadLocal.get();
        if (interProcessSemaphoreMutex == null) {
            interProcessSemaphoreMutex = new InterProcessSemaphoreMutex(client, PATH + "/" + table + SEQ + LOCK);
            interProcessSemaphoreMutexThreadLocal.set(interProcessSemaphoreMutex);
        }
        Map> tableParaValMap = tableParaValMapThreadLocal.get();
        if (tableParaValMap == null) {
            tableParaValMap = new HashMap<>();
            tableParaValMapThreadLocal.set(tableParaValMap);
        }
        Map paraValMap = tableParaValMap.get(table);
        if (paraValMap == null) {
            paraValMap = new ConcurrentHashMap<>();
            tableParaValMap.put(table, paraValMap);

            String seqPath = PATH + ZookeeperPath.ZK_SEPARATOR.getKey() + table + SEQ;

            Stat stat = this.client.checkExists().forPath(seqPath);

            if (stat == null || (stat.getDataLength() == 0)) {
                paraValMap.put(table + KEY_MIN_NAME, props.getProperty(key));
                paraValMap.put(table + KEY_MAX_NAME, props.getProperty(table + KEY_MAX_NAME));
                paraValMap.put(table + KEY_CUR_NAME, props.getProperty(table + KEY_CUR_NAME));
                try {
                    String val = props.getProperty(table + KEY_MIN_NAME);
                    client.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT)
                            .forPath(PATH + "/" + table + SEQ, val.getBytes());
                } catch (Exception e) {
                    LOGGER.debug("Node exists! Maybe other instance is initializing!");
                }
            }
            fetchNextPeriod(table);
        }
    }

    @Override
    public Map getParaValMap(String prefixName) {
        Map> tableParaValMap = tableParaValMapThreadLocal.get();
        if (tableParaValMap == null) {
            try {
                threadLocalLoad();
            } catch (Exception e) {
                LOGGER.error("Error caught while loding configuration within current thread:" + e.getCause());
            }
            tableParaValMap = tableParaValMapThreadLocal.get();
        }
        Map paraValMap = tableParaValMap.get(prefixName);
        return paraValMap;
    }

    @Override
    public Boolean fetchNextPeriod(String prefixName) {
        InterProcessSemaphoreMutex interProcessSemaphoreMutex = interProcessSemaphoreMutexThreadLocal.get();
        try {
            if (interProcessSemaphoreMutex == null) {
                throw new IllegalStateException("IncrSequenceZKHandler should be loaded first!");
            }
            interProcessSemaphoreMutex.acquire();
            Map> tableParaValMap = tableParaValMapThreadLocal.get();
            if (tableParaValMap == null) {
                throw new IllegalStateException("IncrSequenceZKHandler should be loaded first!");
            }
            Map paraValMap = tableParaValMap.get(prefixName);
            if (paraValMap == null) {
                throw new IllegalStateException("IncrSequenceZKHandler should be loaded first!");
            }
            if (paraValMap.get(prefixName + KEY_MAX_NAME) == null) {
                paraValMap.put(prefixName + KEY_MAX_NAME, props.getProperty(prefixName + KEY_MAX_NAME));
            }
            if (paraValMap.get(prefixName + KEY_MIN_NAME) == null) {
                paraValMap.put(prefixName + KEY_MIN_NAME, props.getProperty(prefixName + KEY_MIN_NAME));
            }
            if (paraValMap.get(prefixName + KEY_CUR_NAME) == null) {
                paraValMap.put(prefixName + KEY_CUR_NAME, props.getProperty(prefixName + KEY_CUR_NAME));
            }
            long period = Long.parseLong(paraValMap.get(prefixName + KEY_MAX_NAME))
                    - Long.parseLong(paraValMap.get(prefixName + KEY_MIN_NAME));
            long now = Long.parseLong(new String(client.getData().forPath(PATH + "/" + prefixName + SEQ)));
            client.setData().forPath(PATH + "/" + prefixName + SEQ, ((now + period + 1) + "").getBytes());

            paraValMap.put(prefixName + KEY_MAX_NAME, (now + period + 1) + "");
            paraValMap.put(prefixName + KEY_MIN_NAME, (now + 1) + "");
            paraValMap.put(prefixName + KEY_CUR_NAME, (now) + "");

        } catch (Exception e) {
            LOGGER.error("Error caught while updating period from ZK:" + e.getCause());
        } finally {
            try {
                interProcessSemaphoreMutex.release();
            } catch (Exception e) {
                LOGGER.error("Error caught while realeasing distributed lock" + e.getCause());
            }
        }
        return true;
    }

    @Override
    public Boolean updateCURIDVal(String prefixName, Long val) {
        Map> tableParaValMap = tableParaValMapThreadLocal.get();
        if (tableParaValMap == null) {
            throw new IllegalStateException("IncrSequenceZKHandler should be loaded first!");
        }
        Map paraValMap = tableParaValMap.get(prefixName);
        if (paraValMap == null) {
            throw new IllegalStateException("IncrSequenceZKHandler should be loaded first!");
        }
        paraValMap.put(prefixName + KEY_CUR_NAME, val + "");
        return true;
    }

    public static void main(String[] args) throws UnsupportedEncodingException {
        IncrSequenceZKHandler incrSequenceZKHandler = new IncrSequenceZKHandler();
        incrSequenceZKHandler.load();
        System.out.println(incrSequenceZKHandler.nextId("TRAVELRECORD"));
        System.out.println(incrSequenceZKHandler.nextId("TRAVELRECORD"));
        System.out.println(incrSequenceZKHandler.nextId("TRAVELRECORD"));
        System.out.println(incrSequenceZKHandler.nextId("TRAVELRECORD"));
    }
}


================================================
FILE: src/main/java/io/mycat/route/sequence/handler/SequenceHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.sequence.handler;

/**
 * 
 * @author Michael
 * @time Create on 2013-12-20 下午3:35:53
 * @version 1.0
 */
public interface SequenceHandler {

	public long nextId(String prefixName);

}

================================================
FILE: src/main/java/io/mycat/route/sequence/handler/SequenceVal.java
================================================
package io.mycat.route.sequence.handler;

import io.mycat.MycatServer;
import io.mycat.backend.datasource.PhysicalDBNode;
import io.mycat.backend.datasource.PhysicalDatasource;
import io.mycat.backend.jdbc.JDBCDatasource;
import io.mycat.config.MycatConfig;
import io.mycat.config.model.DataHostConfig;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;

class SequenceVal {
	private static final Logger LOGGER = LoggerFactory
			.getLogger(SequenceVal.class);

	private static final String errSeqResult = "-999999,null";
	public AtomicLong curVal = new AtomicLong(0); //当前可用的id
	public volatile long maxSegValue; //最大的可用id

	public volatile String dbretVal = null; //数据库的返回结果
	public volatile boolean dbfinished ;//请求是否成功返回
	public AtomicBoolean fetching = new AtomicBoolean(false); //是否正在获取id序列当中。
	public volatile boolean successFetched; //是否已经成功的获取并且设置值了。
	public volatile String dataNode;//请求的分片节点
	public final String seqName;//那个表
	public final String sql;//请求分片的sql SELECT mycat_seq_nextval('tableName');
	//初始化
	public void reset(){

		dbretVal = null;
		dbfinished = false;
		successFetched = false; //添加代码

	}
	//设置成功返回
	public void setDbfinished() {
		dbfinished = true;
	}
	//
	public SequenceVal(String seqName, String dataNode) {
		this.seqName = seqName;
		this.dataNode = dataNode;
		sql = "SELECT mycat_seq_nextval('" + seqName + "')";
	}
	//nexVal是否可以用的判断
	public boolean isNexValValid(Long nexVal) {
		if (nexVal < this.maxSegValue) {
			return true;
		} else {
			return false;
		}
	}

	//设置当前的可用值。
	public void setCurValue(long newValue) {
		curVal.set(newValue);
	}
	// 重试次数 保证最大可能性的获取到。
	/*
	 *1.后端获取序列
	 *  如果返回的结果为超时，异常，尝试数据库重试获取
	 *     返回的为0，0   尝试数据库重试获取
	 *     结果还未返回，调用函数继续等待
	 *  成功返回则正常返回
	 * */

	public Long[] fetchSequenceFromDB(FetchMySQLSequnceHandler mysqlSeqFetcher, int retryCount, boolean canSendFetch) {
		DataHostConfig dataHostConfig = getDBHostConfig();

		if (dataHostConfig.isJDBCDriver()) {
			return getSequenceValueByJDBC(retryCount);
		}

		if (dataHostConfig.isNativeDriver()) {
			return getSequenceByNaitve(mysqlSeqFetcher, retryCount, canSendFetch);
		}

		throw new RuntimeException("fetching sequence can not support the db driver:" + dataHostConfig.getDbDriver());
	}

	private Long[] getSequenceByNaitve(FetchMySQLSequnceHandler mysqlSeqFetcher, int retryCount, boolean canSendFetch) {
		final int systemRetryCount = MycatServer.getInstance().getConfig().getSystem().getSequnceMySqlRetryCount();
		//进入waitFinish小于4次，或者可以后端获取数据
		if(retryCount <= systemRetryCount && canSendFetch) {

			this.reset();
			mysqlSeqFetcher.execute(this);
		} else if(retryCount > systemRetryCount){
			fetching.compareAndSet(true, false); //
			return null;
		}
		long start = System.currentTimeMillis();
		//直接等待
		long mysqlWaitTime = MycatServer.getInstance().getConfig().getSystem().getSequnceMySqlWaitTime();

		long end = start + mysqlWaitTime;
		while (System.currentTimeMillis() < end) {
			if(dbfinished){
				if (dbretVal == IncrSequenceMySQLHandler.errSeqResult) {
					fetching.compareAndSet(true, false); //修改
					throw new RuntimeException(
							"sequnce not found in db table ");
				}
				//进行处理 还有可能是链接错误等。
				if(dbretVal == null ){
					LOGGER.warn("can't fetch sequnce in db,sequnce :"
							+ seqName + " detail:"
							+ mysqlSeqFetcher.getLastestError(seqName) + "\n"
									+ ", and retry " + (retryCount) +" time");
					//数据库之类的连接错误，休息一下在重试。
					sleep(10);
					return getSequenceByNaitve(mysqlSeqFetcher, ++retryCount , true);
				}
				String[] items = dbretVal.split(",");

				Long curVal = Long.parseLong(items[0]);
				int span = Integer.parseInt(items[1]);
				//处理返回0，0
				if(0 == curVal) {
					LOGGER.warn("can't fetch sequnce in db,sequnce :"
							+ seqName + " detail:"
							+  " fetch return 0,0 , and retry " + (retryCount) +" time");
					//数据库之类的连接错误，休息一下在重试。
					sleep(100);
					return getSequenceByNaitve(mysqlSeqFetcher, ++retryCount, true);
				}
				fetching.compareAndSet(true, false); //修改s
				return new Long[] { curVal, curVal + span };
			} else{
				sleep(10);
			}
		}
		//等待超时 重试
		LOGGER.warn("wait sequnce in db sequnce  :"
				+ seqName + " detail:"
				+ " wait timeout " + " retry time:" + retryCount);
		return getSequenceByNaitve(mysqlSeqFetcher, ++retryCount, false);
	}


	private Long[] getSequenceValueByJDBC(int retryCount) {
		MycatConfig conf = MycatServer.getInstance().getConfig();
		PhysicalDBNode mysqlDN = conf.getDataNodes().get(this.dataNode);
		final int systemRetryCount = MycatServer.getInstance().getConfig().getSystem().getSequnceMySqlRetryCount();
		long mysqlWaitTime = MycatServer.getInstance().getConfig().getSystem().getSequnceMySqlWaitTime();

		if (retryCount > systemRetryCount){
			this.fetching.compareAndSet(true, false);
			return null;
		}

		if (LOGGER.isDebugEnabled()) {
			LOGGER.debug("execute in datanode " + this.dataNode
					+ " for fetch sequence sql " + this.sql + " with retry count:" + retryCount);
		}

		long start = System.currentTimeMillis();
		//直接等待
		long end = start + mysqlWaitTime;
		while (System.currentTimeMillis() < end) {
			PhysicalDatasource physicalDatasource = mysqlDN.getDbPool().getSource();
			if(physicalDatasource instanceof JDBCDatasource) {
				JDBCDatasource jdbcDatasource = (JDBCDatasource) physicalDatasource;

				Connection con = null;
				PreparedStatement pstmt = null;
				ResultSet rs = null;
				try {
					con = jdbcDatasource.getDruidConnection();
					String useDB = "use `" + mysqlDN.getDatabase() + "`;";
					pstmt = con.prepareStatement(useDB);
					pstmt.execute();
					pstmt = con.prepareStatement(this.sql);
					rs = pstmt.executeQuery();
					String returnedValue = "";
					if (rs.next())
						returnedValue = rs.getString(1);

					if (StringUtils.isEmpty(returnedValue) || errSeqResult.equals(returnedValue)) {
						LOGGER.warn("can't fetch sequnce in db,sequnce :"
								+ this.seqName + " detail:"
								+  " fetch return -999999,null , and retry " + (retryCount) +" time");
						//数据库之类的连接错误，休息一下在重试。
						sleep(100);
						return getSequenceValueByJDBC(++retryCount);
					}
					Long curVal = Long.parseLong(returnedValue.split(",")[0]);
					Long increment = Long.parseLong(returnedValue.split(",")[1]);

					if (curVal == 0) {
						LOGGER.warn("can't fetch sequnce in db,sequnce :"
								+ this.seqName + " detail:"
								+  " fetch return 0,0 , and retry " + (retryCount) +" time");
						//数据库之类的连接错误，休息一下在重试。
						sleep(100);
						return getSequenceValueByJDBC(++retryCount);
					} else {
						this.fetching.compareAndSet(true, false);
						return new Long[]{curVal, curVal + increment};
					}
				} catch (Exception e) {
					LOGGER.warn("get sequence value err ", e);
				} finally {
					try {
						con.close();
						pstmt.close();
						rs.close();
					} catch (SQLException e) {
						e.printStackTrace();
					}
				}
			}
		}
		LOGGER.error("get sequence:" + this.seqName + " value failure, please check the sequence config");
		this.fetching.compareAndSet(true, false);
		return null;
	}

	private DataHostConfig getDBHostConfig() {
		MycatConfig conf = MycatServer.getInstance().getConfig();
		PhysicalDBNode mysqlDN = conf.getDataNodes().get(this.dataNode);
		if (mysqlDN == null) {
			throw new RuntimeException("can not find the data node with name:" + this.dataNode);
		}
		PhysicalDatasource physicalDatasource = mysqlDN.getDbPool().getSource();
		return physicalDatasource.getHostConfig();
	}

	public void sleep(long time) {
		try {
			Thread.sleep(time);
		} catch (InterruptedException e) {
			IncrSequenceMySQLHandler.LOGGER
					.warn("wait db fetch sequnce err " + e);
		}
	}
	//是否成功返回。
	public boolean isSuccessFetched() {
		return successFetched;
	}
	//下一个可用的id
	public long nextValue() {
		if (successFetched == false) {
			throw new RuntimeException(
					"sequnce fetched failed  from db ");
		}
		return curVal.incrementAndGet();
	}
}


================================================
FILE: src/main/java/io/mycat/route/sequence/handler/SnowflakeIdSequenceHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.sequence.handler;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * 本地默认获取的全局ID（用于单机或者测试） 

 * java for base on https://github.com/twitter/snowflake
 * 
 * @author Michael
 * @time Create on 2013-12-22 下午4:52:25
 * @version 1.0
 */
public class SnowflakeIdSequenceHandler implements SequenceHandler {

	private final static Logger logger = LoggerFactory
			.getLogger(SequenceHandler.class);

	private final long workerId;
	private final long datacenterId;
	private static final long twepoch = 1355285532520L;

	private static final long workerIdBits = 5L;
	private static final long datacenterIdBits = 5L;
	private static final long maxWorkerId = -1L ^ -1L << workerIdBits;
	private static final long maxDatacenterId = -1L ^ -1L << datacenterIdBits;
	private static final long sequenceBits = 12L;
	private static final long workerIdShift = sequenceBits;
	private static final long datacenterIdShift = sequenceBits + workerIdBits;

	private static final long timestampLeftShift = sequenceBits
			+ workerIdBits;
	private static final long sequenceMask = -1L ^ -1L << sequenceBits;

	private long sequence = 0L;
	private long lastTimestamp = -1L;

	public SnowflakeIdSequenceHandler(long workerId, long datacenterId) {
		super();
		System.out.println("maxWorkerId = " + maxWorkerId);
		System.out.println("maxDatacenterId = " + maxDatacenterId);
		if (workerId > this.maxWorkerId || workerId < 0) {
			throw new IllegalArgumentException(String.format(
					"worker Id can't be greater than %d or less than 0",
					this.maxWorkerId));
		}

		this.workerId = workerId;
		if (datacenterId > maxDatacenterId || datacenterId < 0) {
			throw new IllegalArgumentException(String.format(
					"datacenter Id can't be greater than %d or less than 0",
					maxDatacenterId));

		}
		this.datacenterId = datacenterId;
		logger.info(String
				.format("worker starting. timestamp left shift %d, datacenter id bits %d, worker id bits %d, sequence bits %d, workerid %d",
						timestampLeftShift, datacenterIdBits, workerIdBits,
						sequenceBits, workerId));

	}

	public SnowflakeIdSequenceHandler(long workerId) {
		this(workerId, 13);
	}

	// 默认
	public SnowflakeIdSequenceHandler() {
		this(23, 13);
	}

	@Override
	public synchronized long nextId(String prefixName) {
		long timestamp = this.timeGen();
		if (timestamp < this.lastTimestamp) {
			logger.error(
					"clock is moving backwards.  Rejecting requests until %d.",
					lastTimestamp);
			throw new RuntimeException(
					String.format(
							"Clock moved backwards.  Refusing to generate id for %d milliseconds",
							(this.lastTimestamp - timestamp)));
		}
		if (this.lastTimestamp == timestamp) {
			this.sequence = this.sequence + 1 & this.sequenceMask;
			if (this.sequence == 0) {
				timestamp = this.tilNextMillis(this.lastTimestamp);
			}
		} else {
			this.sequence = 0;
		}

		this.lastTimestamp = timestamp;
		return timestamp - this.twepoch << this.timestampLeftShift
				| this.datacenterId << this.datacenterIdShift
				| this.workerId << this.workerIdShift | this.sequence;
	}

	private synchronized long tilNextMillis(long lastTimestamp) {
		long timestamp = this.timeGen();
		while (timestamp <= lastTimestamp) {
			timestamp = this.timeGen();
		}
		return timestamp;
	}

	private long timeGen() {
		return System.currentTimeMillis();
	}

	public static void main(String[] args) {
		SnowflakeIdSequenceHandler gen = new SnowflakeIdSequenceHandler(16);
		System.out.println("nextId = " + gen.nextId(null));

	}
}

================================================
FILE: src/main/java/io/mycat/route/sequence/handler/ThirftClientSequenceHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.sequence.handler;


/**
 * 通过Thirft客户端获取集群中心分配的全局ID
 * 
 * @author Michael
 * @time Create on 2013-12-25 上午12:15:48
 * @version 1.0
 */
public class ThirftClientSequenceHandler implements SequenceHandler {

	@Override
	public long nextId(String prefixName) {
		
		return 0;
	}

}

================================================
FILE: src/main/java/io/mycat/route/util/CacheUtil.java
================================================
package io.mycat.route.util;

/**
 * Created by 862911 on 2016/5/4.
 */
public class CacheUtil {
        private static final int RUNS = 10;
        private static final int DIMENSION_1 = 1024 * 1024;
        private static final int DIMENSION_2 = 6;

        private static long[][] longs;

        public static void main(String[] args) throws Exception {
            Thread.sleep(10000);
            longs = new long[DIMENSION_1][];
            for (int i = 0; i < DIMENSION_1; i++) {
                longs[i] = new long[DIMENSION_2];
                for (int j = 0; j < DIMENSION_2; j++) {
                    longs[i][j] = 0L;
                }
            }
            System.out.println("starting....");

            long sum = 0L;
            for (int r = 0; r < RUNS; r++) {

                final long start = System.nanoTime();

//                slow
            for (int j = 0; j < DIMENSION_2; j++) {
                for (int i = 0; i < DIMENSION_1; i++) {
                    sum += longs[i][j];
                }
            }

                //fast
//                for (int i = 0; i < DIMENSION_1; i++) {
//                    for (int j = 0; j < DIMENSION_2; j++) {
//                        sum += longs[i][j];
//                    }
//                }

                System.out.println((System.nanoTime() - start));
            }

        }

}


================================================
FILE: src/main/java/io/mycat/route/util/PartitionUtil.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.route.util;

import io.mycat.util.StringUtil;

/**
 * 数据分区工具
 * 
 * @author mycat
 */
public final class PartitionUtil {

    // 分区长度:数据段分布定义，其中取模的数一定要是2^n， 因为这里使用x % 2^n == x & (2^n - 1)等式，来优化性能。
    private static final int PARTITION_LENGTH = 1024;

    // %转换为&操作的换算数值
    private static final long AND_VALUE = PARTITION_LENGTH - 1;

    // 分区线段
    private final int[] segment = new int[PARTITION_LENGTH];

    /**
     * 
     * @param count 表示定义的分区数
     * @param length 表示对应每个分区的取值长度
     * 注意：其中count,length两个数组的长度必须是一致的。
     * 约束：1024 = sum((count[i]*length[i])). count和length两个向量的点积恒等于1024
     * 

     */
    public PartitionUtil(int[] count, int[] length) {
        if (count == null || length == null || (count.length != length.length)) {
            throw new RuntimeException("error,check your scope & scopeLength definition.");
        }
        int segmentLength = 0;
        for (int i = 0; i < count.length; i++) {
            segmentLength += count[i];
        }
        int[] ai = new int[segmentLength + 1];

        int index = 0;
        for (int i = 0; i < count.length; i++) {
            for (int j = 0; j < count[i]; j++) {
                ai[++index] = ai[index - 1] + length[i];
            }
        }
        if (ai[ai.length - 1] != PARTITION_LENGTH) {
            throw new RuntimeException("error,check your partitionScope definition.");
        }

        // 数据映射操作
        for (int i = 1; i < ai.length; i++) {
            for (int j = ai[i - 1]; j < ai[i]; j++) {
                segment[j] = (i - 1);
            }
        }
    }

    public int partition(long hash) {
        return segment[(int) (hash & AND_VALUE)];
    }

    public int partition(String key, int start, int end) {
        return partition(StringUtil.hash(key, start, end));
    }

}

================================================
FILE: src/main/java/io/mycat/route/util/PropertiesUtil.java
================================================
package io.mycat.route.util;

import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;

/**
 * Property文件加载器
 *
 * @author Hash Zhang
 * @time 00:08:03 2016/5/3
 * @version 1.0
 */
public class PropertiesUtil {
    public static Properties loadProps(String propsFile){
        Properties props = new Properties();
        InputStream inp = Thread.currentThread().getContextClassLoader().getResourceAsStream(propsFile);

        if (inp == null) {
            throw new java.lang.RuntimeException("time sequnce properties not found " + propsFile);
        }
        try {
            props.load(inp);
        } catch (IOException e) {
            throw new java.lang.RuntimeException(e);
        }
        return props;
    }
}


================================================
FILE: src/main/java/io/mycat/route/util/RouterUtil.java
================================================
package io.mycat.route.util;

import java.sql.SQLNonTransientException;
import java.sql.SQLSyntaxErrorException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;

import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLCharExpr;
import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
import com.alibaba.druid.sql.ast.statement.SQLCharacterDataType;
import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition;
import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlCreateTableStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlInsertStatement;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.druid.wall.spi.WallVisitorUtils;
import com.google.common.base.Strings;
import com.google.common.collect.Maps;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;

import io.mycat.MycatServer;
import io.mycat.backend.datasource.PhysicalDBNode;
import io.mycat.backend.datasource.PhysicalDBPool;
import io.mycat.backend.datasource.PhysicalDatasource;
import io.mycat.backend.mysql.nio.handler.FetchStoreNodeOfChildTableHandler;
import io.mycat.backend.mysql.nio.handler.JDBCFetchStoreNodeOfChildTableHandler;
import io.mycat.cache.LayerCachePool;
import io.mycat.config.ErrorCode;
import io.mycat.config.MycatConfig;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.TableConfig;
import io.mycat.config.model.rule.RuleConfig;
import io.mycat.route.RouteResultset;
import io.mycat.route.RouteResultsetNode;
import io.mycat.route.SessionSQLPair;
import io.mycat.route.function.AbstractPartitionAlgorithm;
import io.mycat.route.function.SlotFunction;
import io.mycat.route.parser.druid.DruidShardingParseInfo;
import io.mycat.route.parser.druid.RouteCalculateUnit;
import io.mycat.server.ServerConnection;
import io.mycat.server.parser.ServerParse;
import io.mycat.sqlengine.mpp.ColumnRoutePair;
import io.mycat.sqlengine.mpp.LoadData;
import io.mycat.util.StringUtil;

/**
 * 从ServerRouterUtil中抽取的一些公用方法，路由解析工具类
 * @author wang.dw
 *
 */
public class RouterUtil {

	private static final Logger LOGGER = LoggerFactory.getLogger(RouterUtil.class);

	/**
	 * 移除执行语句中的数据库名
	 *
	 * @param stmt		 执行语句
	 * @param schema  	数据库名
	 * @return 			执行语句
	 * @author mycat
	 *
	 * @modification 修正移除schema的方法
	 * @date 2016/12/29
	 * @modifiedBy Hash Zhang
	 *
	 */
	public static String removeSchema(String stmt, String schema) {
		final String upStmt = stmt.toUpperCase();
		final String upSchema = schema.toUpperCase() + ".";
		final String upSchema2 = new StringBuilder("`").append(schema.toUpperCase()).append("`.").toString();
		int strtPos = 0;
		int indx = 0;

		int indx1 = upStmt.indexOf(upSchema, strtPos);
		int indx2 = upStmt.indexOf(upSchema2, strtPos);
		boolean flag = indx1 < indx2 ? indx1 == -1 : indx2 != -1;
		indx = !flag ? indx1 > 0 ? indx1 : indx2 : indx2 > 0 ? indx2 : indx1;
		if (indx < 0) {
			return stmt;
		}

		int firstE = upStmt.indexOf("'");
		int endE = upStmt.lastIndexOf("'");

		StringBuilder sb = new StringBuilder();
		while (indx > 0) {
			sb.append(stmt.substring(strtPos, indx));

			if (flag) {
				strtPos = indx + upSchema2.length();
			} else {
				strtPos = indx + upSchema.length();
			}
			if (indx > firstE && indx < endE && countChar(stmt, indx) % 2 == 1) {
				sb.append(stmt.substring(indx, indx + schema.length() + 1));
			}
			indx1 = upStmt.indexOf(upSchema, strtPos);
			indx2 = upStmt.indexOf(upSchema2, strtPos);
			flag = indx1 < indx2 ? indx1 == -1 : indx2 != -1;
			indx = !flag ? indx1 > 0 ? indx1 : indx2 : indx2 > 0 ? indx2 : indx1;
		}
		sb.append(stmt.substring(strtPos));
		return sb.toString();
	}

	private static int countChar(String sql,int end)
	{
		int count=0;
		boolean skipChar = false;
		for (int i = 0; i < end; i++) {
			if(sql.charAt(i)=='\'' && !skipChar) {
				count++;
				skipChar = false;
			}else if( sql.charAt(i)=='\\'){
				skipChar = true;
			}else{
				skipChar = false;
			}
		}
		return count;
	}

	/**
	 * 获取第一个节点作为路由
	 *
	 * @param rrs		          数据路由集合
	 * @param dataNode  	数据库所在节点
	 * @param stmt   		执行语句
	 * @return 				数据路由集合
	 *
	 * @author mycat
	 */
	public static RouteResultset routeToSingleNode(RouteResultset rrs,
			String dataNode, String stmt) {
		if (dataNode == null) {
			return rrs;
		}
		RouteResultsetNode[] nodes = new RouteResultsetNode[1];
		nodes[0] = new RouteResultsetNode(dataNode, rrs.getSqlType(), stmt);//rrs.getStatement()
		nodes[0].setSource(rrs);
		rrs.setNodes(nodes);
		rrs.setFinishedRoute(true);
		if(rrs.getDataNodeSlotMap().containsKey(dataNode)){
			nodes[0].setSlot(rrs.getDataNodeSlotMap().get(dataNode));
		}
		if (rrs.getCanRunInReadDB() != null) {
			nodes[0].setCanRunInReadDB(rrs.getCanRunInReadDB());
		}
		if(rrs.getRunOnSlave() != null){
			nodes[0].setRunOnSlave(rrs.getRunOnSlave());
		}

		return rrs;
	}



	/**
	 * 修复DDL路由
	 *
	 * @return RouteResultset
	 * @author aStoneGod
	 */
	public static RouteResultset routeToDDLNode(RouteResultset rrs, int sqlType, String stmt,SchemaConfig schema) throws SQLSyntaxErrorException {
		stmt = getFixedSql(stmt);
		String tablename = "";
		//去除sql前面的注册,如/* ApplicationName=DBeaver 6.0.1 - Main */，这个注册会导致create table出错
		stmt = stmt.replaceFirst("\\/\\*.*\\*\\/\\s*", "");
		final String upStmt = stmt.toUpperCase();
		String rrsStmt = new String(upStmt);
		if(upStmt.startsWith("CREATE")){
			if (upStmt.contains("CREATE INDEX ") || upStmt.contains("CREATE UNIQUE INDEX ")){
				tablename = RouterUtil.getTableName(stmt, RouterUtil.getCreateIndexPos(upStmt, 0));
				/**
				 * Date：2017年11月2日
				 * @author SvenAugustus
				修复oracle 语法不支持 drop index i_t_f on t_test，只有drop index i_t_f;
				 */
				if(!"oracle".equalsIgnoreCase(schema.getDefaultDataNodeDbType())){
					int onInd = upStmt.indexOf("ON", 0);
					rrsStmt= upStmt.substring(0, onInd);
				}
			}else {
				tablename = RouterUtil.getTableName(stmt, RouterUtil.getCreateTablePos(upStmt, 0));
			}
		}else if(upStmt.startsWith("DROP")){
			if (upStmt.contains("DROP INDEX ")){
				tablename = RouterUtil.getTableName(stmt, RouterUtil.getDropIndexPos(upStmt, 0));
			}else {
				tablename = RouterUtil.getTableName(stmt, RouterUtil.getDropTablePos(upStmt, 0));
			}
		}else if(upStmt.startsWith("ALTER")){
			tablename = RouterUtil.getTableName(stmt, RouterUtil.getAlterTablePos(upStmt, 0));
		}else if (upStmt.startsWith("TRUNCATE")){
			tablename = RouterUtil.getTableName(stmt, RouterUtil.getTruncateTablePos(upStmt, 0));
		}
		tablename = tablename.toUpperCase();

		if (schema.getTables().containsKey(tablename)){
			if(ServerParse.DDL==sqlType){
				List dataNodes = new ArrayList<>();
				Map tables = schema.getTables();
				TableConfig tc=tables.get(tablename);
				if (tables != null && (tc  != null)) {
					dataNodes = tc.getDataNodes();
				}
				boolean isSlotFunction= tc.getRule() != null && tc.getRule().getRuleAlgorithm() instanceof SlotFunction;
				Iterator iterator1 = dataNodes.iterator();
				int nodeSize = dataNodes.size();
				RouteResultsetNode[] nodes = new RouteResultsetNode[nodeSize];
				if(isSlotFunction){
					stmt=changeCreateTable(schema,tablename,stmt);
				}
				for(int i=0;i 0) {
			tableName = tableName.substring(ind2 + 1);
		}
		return tableName;
	}


	/**
	 * 获取show语句table名字
	 *
	 * @param stmt	        执行语句
	 * @param repPos   开始位置和位数
	 * @return 表名
	 * @author AStoneGod
	 */
	public static String getShowTableName(SchemaConfig schema, String stmt, int[] repPos) {
		int startPos = repPos[0];
        startPos = getStartPos(stmt, startPos);
		int secInd = stmt.indexOf(' ', startPos + 1);
		if (secInd < 0) {
			secInd = stmt.length();
		}

		repPos[1] = secInd;
		String tableName = stmt.substring(startPos, secInd).trim();
        if (!schema.isCheckSQLSchema()) {
            return tableName;
        }

        int ind2 = tableName.indexOf('.');
		if (ind2 > 0) {
			tableName = tableName.substring(ind2 + 1);
		}
		return tableName;
	}

	/**
	 * 获取语句中前关键字位置和占位个数表名位置
	 *
	 * @param upStmt     执行语句
	 * @param start      开始位置
	 * @return int[]	  关键字位置和占位个数
	 *
	 * @author mycat
	 *
	 * @modification 修改支持语句中包含“IF NOT EXISTS”的情况
	 * @date 2016/12/8
	 * @modifiedBy Hash Zhang
	 */
	public static int[] getCreateTablePos(String upStmt, int start) {
		String token1 = "CREATE ";
		String token2 = " TABLE ";
		String token3 = " EXISTS ";
		int createInd = upStmt.indexOf(token1, start);
		int tabInd1 = upStmt.indexOf(token2, start);
		int tabInd2 = upStmt.indexOf(token3, tabInd1);
		// 既包含CREATE又包含TABLE，且CREATE关键字在TABLE关键字之前
		if (createInd >= 0 && tabInd2 > 0 && tabInd2 > createInd) {
			return new int[] { tabInd2, token3.length() };
		} else if(createInd >= 0 && tabInd1 > 0 && tabInd1 > createInd) {
			return new int[] { tabInd1, token2.length() };
		} else {
			return new int[] { -1, token2.length() };// 不满足条件时，只关注第一个返回值为-1，第二个任意
		}
	}

	/**
	 * 获取语句中前关键字位置和占位个数表名位置
	 *
	 * @param upStmt
	 *            执行语句
	 * @param start
	 *            开始位置
	 * @return int[]关键字位置和占位个数
	 * @author aStoneGod
	 */
	public static int[] getCreateIndexPos(String upStmt, int start) {
		String token1 = "CREATE ";
		String token2 = " INDEX ";
		String token3 = " ON ";
		int createInd = upStmt.indexOf(token1, start);
		int idxInd = upStmt.indexOf(token2, start);
		int onInd = upStmt.indexOf(token3, start);
		// 既包含CREATE又包含INDEX，且CREATE关键字在INDEX关键字之前, 且包含ON...
		if (createInd >= 0 && idxInd > 0 && idxInd > createInd && onInd > 0 && onInd > idxInd) {
			return new int[] {onInd , token3.length() };
		} else {
			return new int[] { -1, token2.length() };// 不满足条件时，只关注第一个返回值为-1，第二个任意
		}
	}

	/**
	 * 获取ALTER语句中前关键字位置和占位个数表名位置
	 *
	 * @param upStmt   执行语句
	 * @param start    开始位置
	 * @return int[]   关键字位置和占位个数
	 * @author aStoneGod
	 */
	public static int[] getAlterTablePos(String upStmt, int start) {
		String token1 = "ALTER ";
		String token2 = " TABLE ";
		int createInd = upStmt.indexOf(token1, start);
		int tabInd = upStmt.indexOf(token2, start);
		// 既包含CREATE又包含TABLE，且CREATE关键字在TABLE关键字之前
		if (createInd >= 0 && tabInd > 0 && tabInd > createInd) {
			return new int[] { tabInd, token2.length() };
		} else {
			return new int[] { -1, token2.length() };// 不满足条件时，只关注第一个返回值为-1，第二个任意
		}
	}

	/**
	 * 获取DROP语句中前关键字位置和占位个数表名位置
	 *
	 * @param upStmt 	执行语句
	 * @param start  	开始位置
	 * @return int[]	关键字位置和占位个数
	 * @author aStoneGod
	 */
	public static int[] getDropTablePos(String upStmt, int start) {
		//增加 if exists判断
		if(upStmt.contains("EXISTS")){
			String token1 = "IF ";
			String token2 = " EXISTS ";
			int ifInd = upStmt.indexOf(token1, start);
			int tabInd = upStmt.indexOf(token2, start);
			if (ifInd >= 0 && tabInd > 0 && tabInd > ifInd) {
				return new int[] { tabInd, token2.length() };
			} else {
				return new int[] { -1, token2.length() };// 不满足条件时，只关注第一个返回值为-1，第二个任意
			}
		}else {
			String token1 = "DROP ";
			String token2 = " TABLE ";
			int createInd = upStmt.indexOf(token1, start);
			int tabInd = upStmt.indexOf(token2, start);

			if (createInd >= 0 && tabInd > 0 && tabInd > createInd) {
				return new int[] { tabInd, token2.length() };
			} else {
				return new int[] { -1, token2.length() };// 不满足条件时，只关注第一个返回值为-1，第二个任意
			}
		}
	}


	/**
	 * 获取DROP语句中前关键字位置和占位个数表名位置
	 *
	 * @param upStmt
	 *            执行语句
	 * @param start
	 *            开始位置
	 * @return int[]关键字位置和占位个数
	 * @author aStoneGod
	 */

	public static int[] getDropIndexPos(String upStmt, int start) {
		String token1 = "DROP ";
		String token2 = " INDEX ";
		String token3 = " ON ";
		int createInd = upStmt.indexOf(token1, start);
		int idxInd = upStmt.indexOf(token2, start);
		int onInd = upStmt.indexOf(token3, start);
		// 既包含CREATE又包含INDEX，且CREATE关键字在INDEX关键字之前, 且包含ON...
		if (createInd >= 0 && idxInd > 0 && idxInd > createInd && onInd > 0 && onInd > idxInd) {
			return new int[] {onInd , token3.length() };
		} else {
			return new int[] { -1, token2.length() };// 不满足条件时，只关注第一个返回值为-1，第二个任意
		}
	}

	/**
	 * 获取TRUNCATE语句中前关键字位置和占位个数表名位置
	 *
	 * @param upStmt    执行语句
	 * @param start     开始位置
	 * @return int[]	关键字位置和占位个数
	 * @author aStoneGod
	 */
	public static int[] getTruncateTablePos(String upStmt, int start) {
		String token1 = "TRUNCATE ";
		String token2 = " TABLE ";
		int createInd = upStmt.indexOf(token1, start);
		int tabInd = upStmt.indexOf(token2, start);
		// 既包含CREATE又包含TABLE，且CREATE关键字在TABLE关键字之前
		if (createInd >= 0 && tabInd > 0 && tabInd > createInd) {
			return new int[] { tabInd, token2.length() };
		} else {
			return new int[] { -1, token2.length() };// 不满足条件时，只关注第一个返回值为-1，第二个任意
		}
	}

	/**
	 * 获取语句中前关键字位置和占位个数表名位置
	 *
	 * @param upStmt   执行语句
	 * @param start    开始位置
	 * @return int[]   关键字位置和占位个数
	 * @author mycat
	 */
	public static int[] getSpecPos(String upStmt, int start) {
		String token1 = " FROM ";
		String token2 = " IN ";
		int tabInd1 = upStmt.indexOf(token1, start);
		int tabInd2 = upStmt.indexOf(token2, start);
		if (tabInd1 > 0) {
			if (tabInd2 < 0) {
				return new int[] { tabInd1, token1.length() };
			}
			return (tabInd1 < tabInd2) ? new int[] { tabInd1, token1.length() }
					: new int[] { tabInd2, token2.length() };
		} else {
			return new int[] { tabInd2, token2.length() };
		}
	}

	/**
	 * 获取开始位置后的 LIKE、WHERE 位置 如果不含 LIKE、WHERE 则返回执行语句的长度
	 *
	 * @param upStmt   执行sql
	 * @param start    开始位置
	 * @return int
	 * @author mycat
	 */
	public static int getSpecEndPos(String upStmt, int start) {
		int tabInd = upStmt.toUpperCase().indexOf(" LIKE ", start);
		if (tabInd < 0) {
			tabInd = upStmt.toUpperCase().indexOf(" WHERE ", start);
		}
		if (tabInd < 0) {
			return upStmt.length();
		}
		return tabInd;
	}

	public static boolean processWithMycatSeq(SchemaConfig schema, int sqlType,
			String origSQL, ServerConnection sc) {
		// check if origSQL is with global sequence
		// @micmiu it is just a simple judgement
		//对应本地文件配置方式：insert into table1(id,name) values(next value for MYCATSEQ_GLOBAL,‘test’);
		// edit by dingw,增加mycatseq_ 兼容，因为ServerConnection的373行，进行路由计算时，将原始语句全部转换为小写
		if (origSQL.indexOf(" MYCATSEQ_") != -1 || origSQL.indexOf("mycatseq_") != -1) {
			processSQL(sc,schema,origSQL,sqlType);
			return true;
		}
		return false;
	}

	public static void processSQL(ServerConnection sc,SchemaConfig schema,String sql,int sqlType){
//		int sequenceHandlerType = MycatServer.getInstance().getConfig().getSystem().getSequenceHandlerType();
		final SessionSQLPair sessionSQLPair = new SessionSQLPair(sc.getSession2(), schema, sql, sqlType);
//      modify by yanjunli  序列获取修改为多线程方式。使用分段锁方式,一个序列一把锁。  begin
//		MycatServer.getInstance().getSequnceProcessor().addNewSql(sessionSQLPair);
		MycatServer.getInstance().getSequenceExecutor().execute(new Runnable() {
			@Override
			public void run() {
				MycatServer.getInstance().getSequnceProcessor().executeSeq(sessionSQLPair);
			}
		});
//      modify   序列获取修改为多线程方式。使用分段锁方式,一个序列一把锁。  end
//		}
	}

	public static boolean processInsert(SchemaConfig schema, int sqlType,
			String origSQL, ServerConnection sc) throws SQLNonTransientException {
		String tableName = StringUtil.getTableName(origSQL).toUpperCase();
		TableConfig tableConfig = schema.getTables().get(tableName);
		boolean processedInsert=false;
		//判断是有自增字段
		if (null != tableConfig && tableConfig.isAutoIncrement()) {
			String primaryKey = tableConfig.getPrimaryKey();
			processedInsert=processInsert(sc,schema,sqlType,origSQL,tableName,primaryKey);
		}
		return processedInsert;
	}
	/*
	 *  找到返回主键的的位置
	 *  找不到返回 -1
	 * */
	private static int isPKInFields(String origSQL,String primaryKey,int firstLeftBracketIndex,int firstRightBracketIndex){

		if (primaryKey == null) {
			throw new RuntimeException("please make sure the primaryKey's config is not null in schemal.xml");
		}

		boolean isPrimaryKeyInFields = false;
		int  pkStart = 0;
		String upperSQL = origSQL.substring(firstLeftBracketIndex, firstRightBracketIndex + 1).toUpperCase();
		for (int pkOffset = 0, primaryKeyLength = primaryKey.length();;) {
			pkStart = upperSQL.indexOf(primaryKey, pkOffset);
			if (pkStart >= 0 && pkStart < firstRightBracketIndex) {
				char pkSide = upperSQL.charAt(pkStart - 1);
				if (pkSide <= ' ' || pkSide == '`' || pkSide == ',' || pkSide == '(') {
					pkSide = upperSQL.charAt(pkStart + primaryKey.length());
					isPrimaryKeyInFields = pkSide <= ' ' || pkSide == '`' || pkSide == ',' || pkSide == ')';
				}
				if (isPrimaryKeyInFields) {
					break;
				}
				pkOffset = pkStart + primaryKeyLength;
			} else {
				break;
			}
		}
		if (isPrimaryKeyInFields) {
			return firstLeftBracketIndex + pkStart;
		} else {
			return  -1;
		}

	}

	public static boolean processInsert(ServerConnection sc,SchemaConfig schema,
			int sqlType,String origSQL,String tableName,String primaryKey) throws SQLNonTransientException {

		int firstLeftBracketIndex = origSQL.indexOf("(");
		int firstRightBracketIndex = origSQL.indexOf(")");
		String upperSql = origSQL.toUpperCase();
		int valuesIndex = upperSql.indexOf("VALUES");
		int selectIndex = upperSql.indexOf("SELECT");
		int fromIndex = upperSql.indexOf("FROM");
		//屏蔽insert into table1 select * from table2语句
		if(firstLeftBracketIndex < 0) {
			String msg = "invalid sql:" + origSQL;
			LOGGER.warn(msg);
			throw new SQLNonTransientException(msg);
		}
		//屏蔽批量插入
		if(selectIndex > 0 &&fromIndex>0&&selectIndex>firstRightBracketIndex&&valuesIndex<0) {
			String msg = "multi insert not provided" ;
			LOGGER.warn(msg);
			throw new SQLNonTransientException(msg);
		}
		//插入语句必须提供列结构，因为MyCat默认对于表结构无感知
		if(valuesIndex + "VALUES".length() <= firstLeftBracketIndex) {
			throw new SQLSyntaxErrorException("insert must provide ColumnList");
		}
		Object[] vauleArrayAndSuffixStr = parseSqlValueArrayAndSuffixStr(origSQL , valuesIndex);
		List> vauleArray = (List>) vauleArrayAndSuffixStr[0];
		String suffixStr = null;
		if (vauleArrayAndSuffixStr.length > 1) {
			suffixStr = (String) vauleArrayAndSuffixStr[1];
		}
		//两种情况处理 1 有主键的 id ,但是值为null 进行改下
		//            2 没有主键的 需要插入 进行改写

		//如果主键不在插入语句的fields中，则需要进一步处理
		boolean processedInsert= false;
		int pkStart = isPKInFields(origSQL,primaryKey,firstLeftBracketIndex,firstRightBracketIndex);


		if(pkStart == -1){
			processedInsert = true;
			handleBatchInsert(sc, schema, sqlType,origSQL, valuesIndex, tableName, primaryKey, vauleArray, suffixStr);
		} else {
			//判断 主键id的值是否为null
			if(pkStart != -1) {
				String subPrefix = origSQL.substring(0, pkStart);
				char c;
				int pkIndex = 0;
				for(int index = 0, len = subPrefix.length(); index < len; index++) {
					c = subPrefix.charAt(index);
					if(c == ',') {
						pkIndex ++;
					}
				}
				processedInsert  = handleBatchInsertWithPK(sc, schema, sqlType,origSQL, valuesIndex, tableName, primaryKey, vauleArray, suffixStr, pkIndex);
			}
		}
		return processedInsert;
	}

	private static boolean handleBatchInsertWithPK(ServerConnection sc, SchemaConfig schema, int sqlType,
			String origSQL, int valuesIndex, String tableName, String primaryKey, List> vauleList,
			String suffixStr, int pkIndex) {
		boolean processedInsert = false;
//	  	final String pk = "\\("+primaryKey+",";
		final String mycatSeqPrefix = "next value for MYCATSEQ_"+tableName.toUpperCase() ;

		/*"VALUES".length() ==6 */
		String prefix = origSQL.substring(0, valuesIndex + 6);
//

		StringBuilder sb = new StringBuilder("");
		for(List list : vauleList) {
			sb.append("(");
			String pkValue = list.get(pkIndex).trim().toLowerCase();
			//null值替换为 next value for MYCATSEQ_tableName
			if("null".equals(pkValue.trim())) {
				list.set(pkIndex, mycatSeqPrefix);
				processedInsert = true;
			}
			for(String val : list) {
				sb.append(val).append(",");
			}
			sb.setCharAt(sb.length() - 1, ')');
			sb.append(",");
		}
		sb.setCharAt(sb.length() - 1, ' ');
		if (suffixStr != null) {
			sb.append(suffixStr);
		}
		if(processedInsert) {
			processSQL(sc, schema,prefix+sb.toString(), sqlType);
		}
		return processedInsert;
	}

	public static List handleBatchInsert(String origSQL, int valuesIndex) {
		List handledSQLs = new LinkedList<>();
		String prefix = origSQL.substring(0, valuesIndex + "VALUES".length());
		String values = origSQL.substring(valuesIndex + "VALUES".length());
		int flag = 0;
		StringBuilder currentValue = new StringBuilder();
		currentValue.append(prefix);
		for (int i = 0; i < values.length(); i++) {
			char j = values.charAt(i);
			if (j == '(' && flag == 0) {
				flag = 1;
				currentValue.append(j);
			} else if (j == '\"' && flag == 1) {
				flag = 2;
				currentValue.append(j);
			} else if (j == '\'' && flag == 1) {
				flag = 3;
				currentValue.append(j);
			} else if (j == '\\' && flag == 2) {
				flag = 4;
				currentValue.append(j);
			} else if (j == '\\' && flag == 3) {
				flag = 5;
				currentValue.append(j);
			} else if (flag == 4) {
				flag = 2;
				currentValue.append(j);
			} else if (flag == 5) {
				flag = 3;
				currentValue.append(j);
			} else if (j == '\"' && flag == 2) {
				flag = 1;
				currentValue.append(j);
			} else if (j == '\'' && flag == 3) {
				flag = 1;
				currentValue.append(j);
			} else if (j == ')' && flag == 1) {
				flag = 0;
				currentValue.append(j);
				handledSQLs.add(currentValue.toString());
				currentValue = new StringBuilder();
				currentValue.append(prefix);
			} else if (j == ',' && flag == 0) {
				continue;
			} else {
				currentValue.append(j);
			}
		}
		return handledSQLs;
	}
	/**
	 * 对于插入的sql : "insert into hotnews(title,name) values('test1',\"name\"),('(test)',\"(test)\"),('\\\"',\"\\'\"),(\")\",\"\\\"\\')\")"：
	 *  需要返回结果：
	 *[[ 'test1', "name"],
	 *	['(test)', "(test)"],
	 *	['\"', "\'"],
	 *	[")", "\"\')"],
	 *	[ 1,  null]
	 * 值结果的解析
	 */
	public static Object[] parseSqlValueArrayAndSuffixStr(String origSQL, int valuesIndex) {
		List> valueArray = new ArrayList<>();
		String valuesAndSuffixStr = origSQL.substring(valuesIndex + 6);// 6 values 长度为6
		int pos = 0 ;
		int flag  = 4;
		int len = valuesAndSuffixStr.length();
		StringBuilder currentValue = new StringBuilder();
//        int colNum = 2; //
		char c ;
		List curList = new ArrayList<>();
		int parenCount = 0;
		for( ;pos < len; pos ++) {
			c = valuesAndSuffixStr.charAt(pos);
			if (flag == 1  || flag == 2) {
				currentValue.append(c);
				if (c == '\\') {
					char nextCode = valuesAndSuffixStr.charAt(pos + 1);
					if (nextCode == '\'' || nextCode == '\"') {
						currentValue.append(nextCode);
						pos++;
						continue;
					}
				}
				if (c == '\"' && flag == 1) {
					flag = 0;
					continue;
				}
				if (c == '\'' && flag == 2) {
					flag = 0;
					continue;
				}
			} else if (flag == 5) {
				currentValue.append(c);
				if (c == '(') {
					parenCount++;
				} else if (c == ')') {
					parenCount--;
				}
				if (parenCount == 0) {
					flag = 0;
				}
			} else if (c == '\"'){
				currentValue.append(c);
				flag = 1;
			} else if (c == '\'') {
				currentValue.append(c);
				flag = 2;
			} else if (c == '(') {
				if (flag == 4) {
					curList = new ArrayList<>();
					flag = 0;
				} else {
					currentValue.append(c);
					flag = 5;
					parenCount++;
				}
			} else if (flag == 4) {
				if (c == 'o' || c == 'O') {
					String suffixStr = valuesAndSuffixStr.substring(pos);
					return new Object[]{valueArray, suffixStr};
				}
				continue;
			} else if (c == ',') {
//                System.out.println(currentValue);
				curList.add(currentValue.toString());
				currentValue.delete(0, currentValue.length());
			} else if (c == ')'){
				flag = 4;
//                System.out.println(currentValue);
				curList.add(currentValue.toString());
				currentValue.delete(0, currentValue.length());
				valueArray.add(curList);
			}  else {
				currentValue.append(c);
			}
		}
		return new Object[]{valueArray};
	}
	/**
	 * 对于主键不在插入语句的fields中的SQL，需要改写。比如hotnews主键为id，插入语句为：
	 * insert into hotnews(title) values('aaa');
	 * 需要改写成：
	 * insert into hotnews(id, title) values(next value for MYCATSEQ_hotnews,'aaa');
	 */
	public static void handleBatchInsert(ServerConnection sc, SchemaConfig schema,
			int sqlType,String origSQL, int valuesIndex,String tableName, String primaryKey , List> vauleList, String suffixStr) {

		final String pk = "\\("+primaryKey+",";
		final String mycatSeqPrefix = "(next value for MYCATSEQ_"+tableName.toUpperCase()+"";

		/*"VALUES".length() ==6 */
		String prefix = origSQL.substring(0, valuesIndex + 6);
//
		prefix = prefix.replaceFirst("\\(", pk);

		StringBuilder sb = new StringBuilder("");
		for(List list : vauleList) {
			sb.append(mycatSeqPrefix);
			for(String val : list) {
				sb.append(",").append(val);
			}
			sb.append("),");
		}
		sb.setCharAt(sb.length() - 1, ' ');
		if (suffixStr != null) {
			sb.append(suffixStr);
		}
		processSQL(sc, schema,prefix+sb.toString(), sqlType);
	}
//	  /**
//	  * 对于主键不在插入语句的fields中的SQL，需要改写。比如hotnews主键为id，插入语句为：
//	  * insert into hotnews(title) values('aaa');
//	  * 需要改写成：
//	  * insert into hotnews(id, title) values(next value for MYCATSEQ_hotnews,'aaa');
//	  */
//    public static void handleBatchInsert(ServerConnection sc, SchemaConfig schema,
//            int sqlType,String origSQL, int valuesIndex,String tableName, String primaryKey) {
//
//    	final String pk = "\\("+primaryKey+",";
//        final String mycatSeqPrefix = "(next value for MYCATSEQ_"+tableName.toUpperCase()+",";
//
//    	/*"VALUES".length() ==6 */
//        String prefix = origSQL.substring(0, valuesIndex + 6);
//        String values = origSQL.substring(valuesIndex + 6);
//
//        prefix = prefix.replaceFirst("\\(", pk);
//        values = values.replaceFirst("\\(", mycatSeqPrefix);
//        values =Pattern.compile(",\\s*\\(").matcher(values).replaceAll(","+mycatSeqPrefix);
//        processSQL(sc, schema,prefix+values, sqlType);
//    }


	public static RouteResultset routeToMultiNode(boolean cache,RouteResultset rrs, Collection dataNodes, String stmt) {
		RouteResultsetNode[] nodes = new RouteResultsetNode[dataNodes.size()];
		int i = 0;
		RouteResultsetNode node;
		for (String dataNode : dataNodes) {
			node = new RouteResultsetNode(dataNode, rrs.getSqlType(), stmt);
			node.setSource(rrs);
			if(rrs.getDataNodeSlotMap().containsKey(dataNode)){
				node.setSlot(rrs.getDataNodeSlotMap().get(dataNode));
			}
			if (rrs.getCanRunInReadDB() != null) {
				node.setCanRunInReadDB(rrs.getCanRunInReadDB());
			}
			if(rrs.getRunOnSlave() != null){
				nodes[0].setRunOnSlave(rrs.getRunOnSlave());
			}
			nodes[i++] = node;
		}
		rrs.setCacheAble(cache);
		rrs.setNodes(nodes);
		return rrs;
	}

	public static RouteResultset routeToMultiNode(boolean cache, RouteResultset rrs, Collection dataNodes,
			String stmt, boolean isGlobalTable) {

		rrs = routeToMultiNode(cache, rrs, dataNodes, stmt);
		rrs.setGlobalTable(isGlobalTable);
		return rrs;
	}

	public static void routeForTableMeta(RouteResultset rrs,
			SchemaConfig schema, String tableName, String sql) {
		String dataNode = null;
		if (isNoSharding(schema,tableName)) {//不分库的直接从schema中获取dataNode
			dataNode = schema.getDataNode();
		} else {
			dataNode = getMetaReadDataNode(schema, tableName);
		}

		RouteResultsetNode[] nodes = new RouteResultsetNode[1];
		nodes[0] = new RouteResultsetNode(dataNode, rrs.getSqlType(), sql);
		nodes[0].setSource(rrs);
		if(rrs.getDataNodeSlotMap().containsKey(dataNode)){
			nodes[0].setSlot(rrs.getDataNodeSlotMap().get(dataNode));
		}
		if (rrs.getCanRunInReadDB() != null) {
			nodes[0].setCanRunInReadDB(rrs.getCanRunInReadDB());
		}
		if(rrs.getRunOnSlave() != null){
			nodes[0].setRunOnSlave(rrs.getRunOnSlave());
		}
		rrs.setNodes(nodes);
	}

	/**
	 * 根据表名随机获取一个节点
	 *
	 * @param schema     数据库名
	 * @param table      表名
	 * @return 			  数据节点
	 * @author mycat
	 */
	private static String getMetaReadDataNode(SchemaConfig schema,
			String table) {
		// Table名字被转化为大写的，存储在schema
		table = table.toUpperCase();
		String dataNode = null;
		Map tables = schema.getTables();
		TableConfig tc;
		if (tables != null && (tc = tables.get(table)) != null) {
			dataNode = getAliveRandomDataNode(tc);
		}
		return dataNode;
	}

	/**
	 * 解决getRandomDataNode方法获取错误节点的问题.
	 * @param tc
	 * @return
	 */
	public static String getAliveRandomDataNode(TableConfig tc) {
		List randomDns = (List)tc.getDataNodes().clone();

		MycatConfig mycatConfig = MycatServer.getInstance().getConfig();
		if (mycatConfig != null) {
			Collections.shuffle(randomDns);
			for (String randomDn : randomDns) {
				PhysicalDBNode physicalDBNode = mycatConfig.getDataNodes().get(randomDn);
				if (physicalDBNode != null) {
					if (physicalDBNode.getDbPool().getSource().isAlive()) {
						for (PhysicalDBPool pool : MycatServer.getInstance().getConfig().getDataHosts().values()) {
							PhysicalDatasource source = pool.getSource();
							if (source.getHostConfig().containDataNode(randomDn) && pool.getSource().isAlive()) {
								return randomDn;
							}
						}
					}
				}
			}
		}

		// all fail return default
		return tc.getRandomDataNode();
	}

	@Deprecated
	private static String getRandomDataNode(TableConfig tc) {
		//写节点不可用，意味着读节点也不可用。
		//直接使用下一个 dataHost
		String randomDn = tc.getRandomDataNode();
		MycatConfig mycatConfig = MycatServer.getInstance().getConfig();
		if (mycatConfig != null) {
			PhysicalDBNode physicalDBNode = mycatConfig.getDataNodes().get(randomDn);
			if (physicalDBNode != null) {
				if (physicalDBNode.getDbPool().getSource().isAlive()) {
					for (PhysicalDBPool pool : MycatServer.getInstance()
							.getConfig()
							.getDataHosts()
							.values()) {
						if (pool.getSource().getHostConfig().containDataNode(randomDn)) {
							continue;
						}

						if (pool.getSource().isAlive()) {
							return pool.getSource().getHostConfig().getRandomDataNode();
						}
					}
				}
			}
		}

		//all fail return default
		return randomDn;
	}

	/**
	 * 根据 ER分片规则获取路由集合
	 *
	 * @param stmt            执行的语句
	 * @param rrs      		     数据路由集合
	 * @param tc	      	     表实体
	 * @param joinKeyVal      连接属性
	 * @return RouteResultset(数据路由集合)	 *
	 * @throws SQLNonTransientException，IllegalShardingColumnValueException
	 * @author mycat
	 */

	public static RouteResultset routeByERParentKey(ServerConnection sc,SchemaConfig schema,
			int sqlType,String stmt,
			RouteResultset rrs, TableConfig tc, String joinKeyVal)
			throws SQLNonTransientException {

		// only has one parent level and ER parent key is parent
		// table's partition key
		if (tc.isSecondLevel()
				//判断是否为二级子表（父表不再有父表）
				&& tc.getParentTC().getPartitionColumn()
				.equals(tc.getParentKey())) { // using
			// parent
			// rule to
			// find
			// datanode
			Set parentColVal = new HashSet(1);
			ColumnRoutePair pair = new ColumnRoutePair(joinKeyVal);
			parentColVal.add(pair);
			Set dataNodeSet = ruleCalculate(tc.getParentTC(), parentColVal,rrs.getDataNodeSlotMap());
			if (dataNodeSet.isEmpty() || dataNodeSet.size() > 1) {
				throw new SQLNonTransientException(
						"parent key can't find  valid datanode ,expect 1 but found: "
								+ dataNodeSet.size());
			}
			String dn = dataNodeSet.iterator().next();
			if (LOGGER.isDebugEnabled()) {
				LOGGER.debug("found partion node (using parent partion rule directly) for child table to insert  "
						+ dn + " sql :" + stmt);
			}
			return RouterUtil.routeToSingleNode(rrs, dn, stmt);
		}
		return null;
	}

	/**
	 * @return dataNodeIndex -> [partitionKeysValueTuple+]
	 */
	public static Set ruleByJoinValueCalculate(RouteResultset rrs, TableConfig tc,
			Set colRoutePairSet) throws SQLNonTransientException {

		String joinValue = "";

		if(colRoutePairSet.size() > 1) {
			LOGGER.warn("joinKey can't have multi Value");
		} else {
			Iterator it = colRoutePairSet.iterator();
			ColumnRoutePair joinCol = it.next();
			joinValue = joinCol.colValue;
		}

		Set retNodeSet = new LinkedHashSet();

		Set nodeSet;
		if (tc.isSecondLevel()
				&& tc.getParentTC().getPartitionColumn()
				.equals(tc.getParentKey())) { // using
			// parent
			// rule to
			// find
			// datanode

			nodeSet = ruleCalculate(tc.getParentTC(),colRoutePairSet,rrs.getDataNodeSlotMap());
			if (nodeSet.isEmpty()) {
				throw new SQLNonTransientException(
						"parent key can't find  valid datanode ,expect 1 but found: "
								+ nodeSet.size());
			}
			if (LOGGER.isDebugEnabled()) {
				LOGGER.debug("found partion node (using parent partion rule directly) for child table to insert  "
						+ nodeSet + " sql :" + rrs.getStatement());
			}
			retNodeSet.addAll(nodeSet);

//			for(ColumnRoutePair pair : colRoutePairSet) {
//				nodeSet = ruleCalculate(tc.getParentTC(),colRoutePairSet);
//				if (nodeSet.isEmpty() || nodeSet.size() > 1) {//an exception would be thrown, if sql was executed on more than on sharding
//					throw new SQLNonTransientException(
//							"parent key can't find  valid datanode ,expect 1 but found: "
//									+ nodeSet.size());
//				}
//				String dn = nodeSet.iterator().next();
//				if (LOGGER.isDebugEnabled()) {
//					LOGGER.debug("found partion node (using parent partion rule directly) for child table to insert  "
//							+ dn + " sql :" + rrs.getStatement());
//				}
//				retNodeSet.addAll(nodeSet);
//			}
			return retNodeSet;
		} else {
			retNodeSet.addAll(tc.getParentTC().getDataNodes());
		}

		return retNodeSet;
	}


	/**
	 * @return dataNodeIndex -> [partitionKeysValueTuple+]
	 */
	public static Set ruleCalculate(TableConfig tc,
			Set colRoutePairSet,Map   dataNodeSlotMap)  {
		Set routeNodeSet = new LinkedHashSet();
		String col = tc.getRule().getColumn();
		RuleConfig rule = tc.getRule();
		AbstractPartitionAlgorithm algorithm = rule.getRuleAlgorithm();
		for (ColumnRoutePair colPair : colRoutePairSet) {
			if (colPair.colValue != null) {
				Integer nodeIndx = algorithm.calculate(StringUtil.removeBackquote(colPair.colValue));
				if (nodeIndx == null) {
					throw new IllegalArgumentException(
							"can't find datanode for sharding column:" + col
									+ " val:" + colPair.colValue);
				} else {
					String dataNode = tc.getDataNodes().get(nodeIndx);
					routeNodeSet.add(dataNode);
					if(algorithm instanceof SlotFunction) {
						dataNodeSlotMap.put(dataNode,((SlotFunction) algorithm).slotValue());
					}
					colPair.setNodeId(nodeIndx);
				}
			} else if (colPair.rangeValue != null) {
				Integer[] nodeRange = algorithm.calculateRange(
						String.valueOf(colPair.rangeValue.beginValue),
						String.valueOf(colPair.rangeValue.endValue));
				if (nodeRange != null) {
					/**
					 * 不能确认 colPair的 nodeid是否会有其它影响
					 */
					if (nodeRange.length == 0) {
						routeNodeSet.addAll(tc.getDataNodes());
					} else {
						ArrayList dataNodes = tc.getDataNodes();
						String dataNode = null;
						for (Integer nodeId : nodeRange) {
							dataNode = dataNodes.get(nodeId);
							if(algorithm instanceof SlotFunction) {
								dataNodeSlotMap.put(dataNode,((SlotFunction) algorithm).slotValue());
							}
							routeNodeSet.add(dataNode);
						}
					}
				}
			}

		}
		return routeNodeSet;
	}

	/**
	 * 多表路由
	 */
	public static RouteResultset tryRouteForTables(SchemaConfig schema, DruidShardingParseInfo ctx,
			RouteCalculateUnit routeUnit, RouteResultset rrs, boolean isSelect, LayerCachePool cachePool)
			throws SQLNonTransientException {

		List tables = ctx.getTables();


		if(schema.isNoSharding()||(tables.size() >= 1&&isNoSharding(schema,tables.get(0)))) {
			return routeToSingleNode(rrs, schema.getDataNode(), ctx.getSql());
		}
		// 系统表，不会记录到DruidShardingParseInfo里面，此时(tables.size()为0，直接路由到schema默认节点
		if (tables.size() == 0) {
			return routeToSingleNode(rrs, schema.getDataNode(), ctx.getSql());
		}
		
		//每个表对应的路由映射
		Map> tablesRouteMap = new HashMap>();

		//为全局表和单库表找路由
		for(String tableName : tables) {

			TableConfig tableConfig = schema.getTables().get(tableName.toUpperCase());

			if(tableConfig == null) {
				//add 如果表读取不到则先将表名从别名中读取转化后再读取
				String alias = ctx.getTableAliasMap().get(tableName);
				if(!StringUtil.isEmpty(alias)){
					tableConfig = schema.getTables().get(alias.toUpperCase());
				}

				if(tableConfig == null){
					if (StringUtils.isNotEmpty(schema.getDataNode())) {
						return routeToSingleNode(rrs, schema.getDataNode(), ctx.getSql());
					} else {
						String msg = "can't find table define in schema " + tableName + " alias：" + alias + ", schema:"
								+ schema.getName();
						LOGGER.warn(msg);
						throw new SQLNonTransientException(msg);
					}
				}

			}
			if(tableConfig.isGlobalTable()) {//全局表
				if(tablesRouteMap.get(tableName) == null) {
					tablesRouteMap.put(tableName, new HashSet());
				}
				tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
			} else if(tablesRouteMap.get(tableName) == null) { //余下的表都是单库表
				tablesRouteMap.put(tableName, new HashSet());
				tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
			}

			if(tableConfig.getDistTables().size() > 0) {
				Map> subTablesmap = rrs.getSubTableMaps();
				if (subTablesmap == null) {
					subTablesmap = Maps.newHashMap();
					rrs.setSubTableMaps(subTablesmap);
				}

				subTablesmap.put(tableName.toUpperCase(), tableConfig.getDistTables());
			}
		}


		//只有一个表的
		if(tables.size() == 1) {
			return RouterUtil.tryRouteForOneTable(schema, ctx, routeUnit, tables.get(0), rrs, isSelect, cachePool);
		}

		Set retNodesSet = new HashSet();

		//分库解析信息不为空
		Map>> tablesAndConditions = routeUnit.getTablesAndConditions();
		if(tablesAndConditions != null && tablesAndConditions.size() > 0) {
			//为分库表找路由
			RouterUtil.findRouteWithcConditionsForTables(schema, rrs, tablesAndConditions, tablesRouteMap, ctx.getSql(), cachePool, isSelect);
			if(rrs.isFinishedRoute()) {
				return rrs;
			}
		}


		boolean isFirstAdd = true;
		for(Map.Entry> entry : tablesRouteMap.entrySet()) {
			if(entry.getValue() == null || entry.getValue().size() == 0) {
				throw new SQLNonTransientException("parent key can't find any valid datanode ");
			} else {
				if(isFirstAdd) {
					retNodesSet.addAll(entry.getValue());
					isFirstAdd = false;
				} else {
					retNodesSet.retainAll(entry.getValue());
					if(retNodesSet.size() == 0) {//两个表的路由无交集
						String errMsg = "invalid route in sql, multi tables found but datanode has no intersection "
								+ " sql:" + ctx.getSql();
						LOGGER.warn(errMsg);
						throw new SQLNonTransientException(errMsg);
					}
				}
			}
		}

		if(retNodesSet != null && retNodesSet.size() > 0) {
			String tableName = tables.get(0);
			TableConfig tableConfig = schema.getTables().get(tableName.toUpperCase());
			if(tableConfig.isDistTable()){
			    routeToDistTableNode(schema, rrs, ctx.getSql(), tablesAndConditions, cachePool, isSelect, null);
				return rrs;
			}

			if(retNodesSet.size() > 1 && isAllGlobalTable(ctx, schema)) {
				// mulit routes ,not cache route result
				if (isSelect) {
					rrs.setCacheAble(false);
					ArrayList retNodeList = new ArrayList(retNodesSet);
					Collections.shuffle(retNodeList);//by kaiz : add shuffle
					routeToSingleNode(rrs, retNodeList.get(0), ctx.getSql());
				}
				else {//delete 删除全局表的记录
					routeToMultiNode(isSelect, rrs, retNodesSet, ctx.getSql(),true);
				}

			} else {
				routeToMultiNode(isSelect, rrs, retNodesSet, ctx.getSql());
			}

		}
		return rrs;

	}


	/**
	 *
	 * 单表路由
	 */
	public static RouteResultset tryRouteForOneTable(SchemaConfig schema, DruidShardingParseInfo ctx,
			RouteCalculateUnit routeUnit, String tableName, RouteResultset rrs, boolean isSelect,
			LayerCachePool cachePool) throws SQLNonTransientException {

		if (isNoSharding(schema, tableName)) {
			return routeToSingleNode(rrs, schema.getDataNode(), ctx.getSql());
		}

		TableConfig tc = schema.getTables().get(tableName);
		if(tc == null) {
			String msg = "can't find table define in schema " + tableName + " schema:" + schema.getName();
			LOGGER.warn(msg);
			throw new SQLNonTransientException(msg);
		}

		Map>> tablesAndConditions = routeUnit.getTablesAndConditions();
        if(tc.isDistTable()){
            Set keySet = tablesAndConditions.keySet();
            //Map.Entry>> entry = (Entry>>) tablesAndConditions.get(keySet.toArray()[0]);
            return routeToDistTableNode(schema,rrs,ctx.getSql(), tablesAndConditions, cachePool,isSelect, null);
        }

		if(tc.isGlobalTable()) {//全局表
			if(isSelect) {
				// global select ,not cache route result
				rrs.setCacheAble(false);
				return routeToSingleNode(rrs, getAliveRandomDataNode(tc)/*getRandomDataNode(tc)*/, ctx.getSql());
			} else {//insert into 全局表的记录
				return routeToMultiNode(false, rrs, tc.getDataNodes(), ctx.getSql(),true);
			}
		} else {//单表或者分库表
			if (!checkRuleRequired(schema, ctx, routeUnit, tc)) {
				throw new IllegalArgumentException("route rule for table "
						+ tc.getName() + " is required: " + ctx.getSql());

			}
			if(tc.getPartitionColumn() == null && !tc.isSecondLevel()) {//单表且不是childTable
//				return RouterUtil.routeToSingleNode(rrs, tc.getDataNodes().get(0),ctx.getSql());
				return routeToMultiNode(rrs.isCacheAble(), rrs, tc.getDataNodes(), ctx.getSql());
			} else {
				//每个表对应的路由映射
				Map> tablesRouteMap = new HashMap>();
				if(routeUnit.getTablesAndConditions() != null && routeUnit.getTablesAndConditions().size() > 0) {
					RouterUtil.findRouteWithcConditionsForTables(schema, rrs, routeUnit.getTablesAndConditions(), tablesRouteMap, ctx.getSql(), cachePool, isSelect);
					if(rrs.isFinishedRoute()) {
						return rrs;
					}
				}

				if(tablesRouteMap.get(tableName) == null) {
					return routeToMultiNode(rrs.isCacheAble(), rrs, tc.getDataNodes(), ctx.getSql());
				} else {
					return routeToMultiNode(rrs.isCacheAble(), rrs, tablesRouteMap.get(tableName), ctx.getSql());
				}
			}
		}
	}

	private static RouteResultset routeToDistTableNode(SchemaConfig schema, RouteResultset rrs,
        String orgSql, Map>> tablesAndConditions,
        LayerCachePool cachePool, boolean isSelect, Map.Entry>> entry) throws SQLNonTransientException {


        String tableName = null;
        if (entry != null) {
            tableName = entry.getKey().toUpperCase();
        } else {
            List tables = rrs.getTables();
            tableName = tables.get(0);
        }
        TableConfig tableConfig = schema.getTables().get(tableName);
        if(tableConfig == null) {
            String msg = "can't find table define in schema " + tableName + " schema:" + schema.getName();
            LOGGER.warn(msg);
            throw new SQLNonTransientException(msg);
        }
        if(tableConfig.isGlobalTable()){
            String msg = "can't suport district table  " + tableName + " schema:" + schema.getName() + " for global table ";
            LOGGER.warn(msg);
            throw new SQLNonTransientException(msg);
        }
        String partionCol = tableConfig.getPartitionColumn();
    //  String primaryKey = tableConfig.getPrimaryKey();
        //boolean isLoadData=false;
    
        Set tablesRouteSet = new HashSet();
    
        List dataNodes = tableConfig.getDataNodes();
        if(dataNodes.size()>1){
            String msg = "can't suport district table  " + tableName + " schema:" + schema.getName() + " for mutiple dataNode " + dataNodes;
            LOGGER.warn(msg);
            throw new SQLNonTransientException(msg);
        }
        String dataNode = dataNodes.get(0);
    
        RouteResultsetNode[] nodes = null;
        //主键查找缓存暂时不实现
        if(tablesAndConditions.isEmpty()){
            List subTables = tableConfig.getDistTables();
            tablesRouteSet.addAll(subTables);
    
            nodes = getNode(rrs, orgSql, tablesRouteSet, dataNode, false, tableName);
        } else {
            if (entry == null) {
                for(Map.Entry>> entry1 : tablesAndConditions.entrySet()) {
                    //boolean isFoundPartitionValue = partionCol != null && entry.getValue().get(partionCol) != null;
                    setNodes(rrs, tableConfig, partionCol, tablesRouteSet, entry1);
                }
            } else {
                setNodes(rrs, tableConfig, partionCol, tablesRouteSet, entry);
            }
    
    
            nodes = getNode(rrs, orgSql, tablesRouteSet, dataNode, true, tableName);
        }
        rrs.setNodes(nodes);
        rrs.setSubTables(tablesRouteSet);
        rrs.setFinishedRoute(true);
    
        return rrs;
    }
	
	private static void setNodes(RouteResultset rrs, TableConfig tableConfig, String partionCol,
        Set tablesRouteSet, Map.Entry>> entry1)
        throws SQLNonTransientException {
        Map> columnsMap = entry1.getValue();
    
        Set partitionValue = columnsMap.get(partionCol);
        if(partitionValue == null || partitionValue.size() == 0) {
            tablesRouteSet.addAll(tableConfig.getDistTables());
        } else {
            for(ColumnRoutePair pair : partitionValue) {
                AbstractPartitionAlgorithm algorithm = tableConfig.getRule().getRuleAlgorithm();
                if(pair.colValue != null) {
                    Integer tableIndex = algorithm.calculate(pair.colValue);
                    if(tableIndex == null) {
                        String msg = "can't find any valid datanode :" + tableConfig.getName()
                                + " -> " + tableConfig.getPartitionColumn() + " -> " + pair.colValue;
                        LOGGER.warn(msg);
                        throw new SQLNonTransientException(msg);
                    }
                    String subTable = tableConfig.getDistTables().get(tableIndex);
                    if(subTable != null) {
                        tablesRouteSet.add(subTable);
                        if(algorithm instanceof SlotFunction){
                            rrs.getDataNodeSlotMap().put(subTable,((SlotFunction) algorithm).slotValue());
                        }
                    }
                }
                if(pair.rangeValue != null) {
                    Integer[] tableIndexs = algorithm
                            .calculateRange(pair.rangeValue.beginValue.toString(), pair.rangeValue.endValue.toString());
                    for(Integer idx : tableIndexs) {
                        String subTable = tableConfig.getDistTables().get(idx);
                        if(subTable != null) {
                            tablesRouteSet.add(subTable);
                            if(algorithm instanceof SlotFunction){
                                rrs.getDataNodeSlotMap().put(subTable,((SlotFunction) algorithm).slotValue());
                            }
                        }
                    }
                }
            }
        }
    }

	private static RouteResultsetNode[] getNode(RouteResultset rrs, String orgSql, Set tablesRouteSet,
			String dataNode, boolean is, String tableName) {
		Object[] subTables =  tablesRouteSet.toArray();
		RouteResultsetNode[] nodes = new RouteResultsetNode[subTables.length];
		Map dataNodeSlotMap= rrs.getDataNodeSlotMap();
		for(int i=0;i> subTableMaps = rrs.getSubTableMaps();
				if(subTableMaps != null) {
					List list = subTableMaps.get(tableName);
					int index = 0;
					for (String subTable : list) {
						if (table.equals(subTable)) {
							break;
						}
						index++;
					}
					for (String tableSource : subTableMaps.keySet()) {
						Map subTableNames = nodes[i].getSubTableNames();
						if (subTableNames == null) {
							subTableNames = Maps.newHashMap();
							nodes[i].setSubTableNames(subTableNames);
						}
						if (tableSource.equals(tableName)) {
							subTableNames.put(tableSource, table);
						} else {
							subTableNames.put(tableSource, subTableMaps.get(tableSource).get(index));
						}

					}
				}
			} else {
				Map> subTableMaps = rrs.getSubTableMaps();
				if(subTableMaps != null) {
					for (String tableSource : subTableMaps.keySet()) {
						Map subTableNames = nodes[i].getSubTableNames();
						if (subTableNames == null) {
							subTableNames = Maps.newHashMap();
							nodes[i].setSubTableNames(subTableNames);
						}
						subTableNames.put(tableSource, subTableMaps.get(tableSource).get(i));
					}
				}
			}

			nodes[i].setSource(rrs);
			if(rrs.getDataNodeSlotMap().containsKey(dataNode)){
				nodes[i].setSlot(rrs.getDataNodeSlotMap().get(dataNode));
			}
			if (rrs.getCanRunInReadDB() != null) {
				nodes[i].setCanRunInReadDB(rrs.getCanRunInReadDB());
			}
			if(dataNodeSlotMap.containsKey(table))  {
				nodes[i].setSlot(dataNodeSlotMap.get(table));
			}
			if(rrs.getRunOnSlave() != null){
				nodes[0].setRunOnSlave(rrs.getRunOnSlave());
			}
		}
		return nodes;
	}

	/**
	 * 处理分库表路由
	 */
	public static void findRouteWithcConditionsForTables(SchemaConfig schema, RouteResultset rrs,
			Map>> tablesAndConditions,
			Map> tablesRouteMap, String sql, LayerCachePool cachePool, boolean isSelect)
			throws SQLNonTransientException {

		//为分库表找路由
		for(Map.Entry>> entry : tablesAndConditions.entrySet()) {
			String tableName = entry.getKey().toUpperCase();
			TableConfig tableConfig = schema.getTables().get(tableName);
			if(tableConfig == null) {
				String msg = "can't find table define in schema "
						+ tableName + " schema:" + schema.getName();
				LOGGER.warn(msg);
				throw new SQLNonTransientException(msg);
			}
			if(tableConfig.getDistTables()!=null && tableConfig.getDistTables().size()>0){
			    routeToDistTableNode(schema,rrs,sql, tablesAndConditions, cachePool,isSelect, entry);
			}
			//全局表或者不分库的表略过（全局表后面再计算）
			if(tableConfig.isGlobalTable() || schema.getTables().get(tableName).getDataNodes().size() == 1) {
				continue;
			} else {//非全局表：分库表、childTable、其他
				Map> columnsMap = entry.getValue();
				String joinKey = tableConfig.getJoinKey();
				String partionCol = tableConfig.getPartitionColumn();
				String primaryKey = tableConfig.getPrimaryKey();
				boolean isFoundPartitionValue = partionCol != null && entry.getValue().get(partionCol) != null;
				boolean isLoadData=false;
				if (LOGGER.isDebugEnabled()
						&& sql.startsWith(LoadData.loadDataHint)||rrs.isLoadData()) {
					//由于load data一次会计算很多路由数据，如果输出此日志会极大降低load data的性能
					isLoadData=true;
				}
				if(entry.getValue().get(primaryKey) != null && entry.getValue().size() == 1&&!isLoadData)
				{//主键查找
					// try by primary key if found in cache
					Set primaryKeyPairs = entry.getValue().get(primaryKey);
					if (primaryKeyPairs != null) {
						if (LOGGER.isDebugEnabled()) {
							LOGGER.debug("try to find cache by primary key ");
						}
						String tableKey = schema.getName() + '_' + tableName;
						tableKey = tableKey.toUpperCase();
						boolean allFound = true;
						for (ColumnRoutePair pair : primaryKeyPairs) {//可能id in(1,2,3)多主键
							String cacheKey = pair.colValue;
							String dataNode = (String) cachePool.get(tableKey, cacheKey);
							if (dataNode == null) {
								allFound = false;
								continue;
							} else {
								if(tablesRouteMap.get(tableName) == null) {
									tablesRouteMap.put(tableName, new HashSet());
								}
								tablesRouteMap.get(tableName).add(dataNode);
								continue;
							}
						}
						if (!allFound) {
							// need cache primary key ->datanode relation
							if (isSelect && tableConfig.getPrimaryKey() != null) {
								rrs.setPrimaryKey(tableKey + '.' + tableConfig.getPrimaryKey());
							}
						} else {//主键缓存中找到了就执行循环的下一轮
							continue;
						}
					}
				}
				if (isFoundPartitionValue) {//分库表
                    tablesRouteMap.clear();
					Set partitionValue = columnsMap.get(partionCol);
					if(partitionValue == null || partitionValue.size() == 0) {
						if(tablesRouteMap.get(tableName) == null) {
							tablesRouteMap.put(tableName, new HashSet());
						}
						tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
					} else {
						for(ColumnRoutePair pair : partitionValue) {
							AbstractPartitionAlgorithm algorithm = tableConfig.getRule().getRuleAlgorithm();
							if(pair.colValue != null) {
								Integer nodeIndex = algorithm.calculate(StringUtil.removeBackquote(pair.colValue));
								if(nodeIndex == null) {
									String msg = "can't find any valid datanode :" + tableConfig.getName()
											+ " -> " + tableConfig.getPartitionColumn() + " -> " + pair.colValue;
									LOGGER.warn(msg);
									throw new SQLNonTransientException(msg);
								}

								ArrayList dataNodes = tableConfig.getDataNodes();
								String node;
								if (nodeIndex >=0 && nodeIndex < dataNodes.size()) {
									node = dataNodes.get(nodeIndex);

								} else {
									node = null;
									String msg = "Can't find a valid data node for specified node index :"
											+ tableConfig.getName() + " -> " + tableConfig.getPartitionColumn()
											+ " -> " + pair.colValue + " -> " + "Index : " + nodeIndex;
									LOGGER.warn(msg);
									throw new SQLNonTransientException(msg);
								}
								if(node != null) {
									if(tablesRouteMap.get(tableName) == null) {
										tablesRouteMap.put(tableName, new HashSet());
									}
									if(algorithm instanceof SlotFunction){
										rrs.getDataNodeSlotMap().put(node,((SlotFunction) algorithm).slotValue());
									}
									tablesRouteMap.get(tableName).add(node);
								}
							}
							if(pair.rangeValue != null) {
								Integer[] nodeIndexs = algorithm
										.calculateRange(pair.rangeValue.beginValue.toString(), pair.rangeValue.endValue.toString());
								ArrayList dataNodes = tableConfig.getDataNodes();
								String node;
								for(Integer idx : nodeIndexs) {
									if (idx >= 0 && idx < dataNodes.size()) {
										node = dataNodes.get(idx);
									} else {
										String msg = "Can't find valid data node(s) for some of specified node indexes :"
												+ tableConfig.getName() + " -> " + tableConfig.getPartitionColumn();
										LOGGER.warn(msg);
										throw new SQLNonTransientException(msg);
									}
									if(node != null) {
										if(tablesRouteMap.get(tableName) == null) {
											tablesRouteMap.put(tableName, new HashSet());
										}
										if(algorithm instanceof SlotFunction){
											rrs.getDataNodeSlotMap().put(node,((SlotFunction) algorithm).slotValue());
										}
										tablesRouteMap.get(tableName).add(node);

									}
								}
							}
						}
					}
				} else if(joinKey != null && columnsMap.get(joinKey) != null && columnsMap.get(joinKey).size() != 0) {//childTable  (如果是select 语句的父子表join)之前要找到root table,将childTable移除,只留下root table
					Set joinKeyValue = columnsMap.get(joinKey);

					Set dataNodeSet = ruleByJoinValueCalculate(rrs, tableConfig, joinKeyValue);

					if (dataNodeSet.isEmpty()) {
						throw new SQLNonTransientException(
								"parent key can't find any valid datanode ");
					}
					if (LOGGER.isDebugEnabled()) {
						LOGGER.debug("found partion nodes (using parent partion rule directly) for child table to update  "
								+ Arrays.toString(dataNodeSet.toArray()) + " sql :" + sql);
					}
					if (dataNodeSet.size() > 1) {
						routeToMultiNode(rrs.isCacheAble(), rrs, dataNodeSet, sql);
						rrs.setFinishedRoute(true);
						return;
					} else {
						rrs.setCacheAble(true);
						routeToSingleNode(rrs, dataNodeSet.iterator().next(), sql);
						return;
					}

				} else {
					//没找到拆分字段，该表的所有节点都路由
					if(tablesRouteMap.get(tableName) == null) {
						tablesRouteMap.put(tableName, new HashSet());
					}
					boolean isSlotFunction= tableConfig.getRule() != null && tableConfig.getRule().getRuleAlgorithm() instanceof SlotFunction;
					if(isSlotFunction){
						for (String dn : tableConfig.getDataNodes()) {
							rrs.getDataNodeSlotMap().put(dn,-1);
						}
					}
					tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
				}
			}
		}
	}

	public static boolean isAllGlobalTable(DruidShardingParseInfo ctx, SchemaConfig schema) {
		boolean isAllGlobal = false;
		for(String table : ctx.getTables()) {
			TableConfig tableConfig = schema.getTables().get(table);
			if(tableConfig!=null && tableConfig.isGlobalTable()) {
				isAllGlobal = true;
			} else {
				return false;
			}
		}
		return isAllGlobal;
	}

	/**
	 *
	 * @param schema
	 * @param ctx
	 * @param tc
	 * @return true表示校验通过，false表示检验不通过
	 */
	public static boolean checkRuleRequired(SchemaConfig schema, DruidShardingParseInfo ctx, RouteCalculateUnit routeUnit, TableConfig tc) {
		if(!tc.isRuleRequired()) {
			return true;
		}
		boolean hasRequiredValue = false;
		String tableName = tc.getName();
		if(routeUnit.getTablesAndConditions().get(tableName) == null || routeUnit.getTablesAndConditions().get(tableName).size() == 0) {
			hasRequiredValue = false;
		} else {
			for(Map.Entry> condition : routeUnit.getTablesAndConditions().get(tableName).entrySet()) {

				String colName = condition.getKey();
				//条件字段是拆分字段
				if(colName.equals(tc.getPartitionColumn())) {
					hasRequiredValue = true;
					break;
				}
			}
		}
		return hasRequiredValue;
	}


	/**
	 * 增加判断支持未配置分片的表走默认的dataNode
	 * @param schemaConfig
	 * @param tableName
	 * @return
	 */
	public static boolean isNoSharding(SchemaConfig schemaConfig, String tableName) {
		// Table名字被转化为大写的，存储在schema
		tableName = tableName.toUpperCase();
		if (schemaConfig.isNoSharding()) {
			return true;
		}

		if (schemaConfig.getDataNode() != null && !schemaConfig.getTables().containsKey(tableName)) {
			return true;
		}

		return false;
	}

	/**
	 * 系统表判断,某些sql语句会查询系统表或者跟系统表关联
	 * @author lian
	 * @date 2016年12月2日
	 * @param tableName
	 * @return
	 */
	public static boolean isSystemSchema(String tableName) {
		// 以information_schema， mysql开头的是系统表
		if (tableName.startsWith("INFORMATION_SCHEMA.")
				|| tableName.startsWith("MYSQL.")
				|| tableName.startsWith("PERFORMANCE_SCHEMA.")) {
			return true;
		}

		return false;
	}

	/**
	 * 判断条件是否永真
	 * @param expr
	 * @return
	 */
	public static boolean isConditionAlwaysTrue(SQLExpr expr) {
		Object o = WallVisitorUtils.getValue(expr);
		if(Boolean.TRUE.equals(o)) {
			return true;
		}
		return false;
	}

	/**
	 * 判断条件是否永假的
	 * @param expr
	 * @return
	 */
	public static boolean isConditionAlwaysFalse(SQLExpr expr) {
		Object o = WallVisitorUtils.getValue(expr);
		if(Boolean.FALSE.equals(o)) {
			return true;
		}
		return false;
	}


	/**
	 * 该方法，返回是否是ER子表
	 * @param schema
	 * @param origSQL
	 * @param sc
	 * @return
	 * @throws SQLNonTransientException
	 *
	 * 备注说明：
	 *     edit by ding.w at 2017.4.28, 主要处理 CLIENT_MULTI_STATEMENTS(insert into ; insert into)的情况
	 *     目前仅支持mysql,并COM_QUERY请求包中的所有insert语句要么全部是er表，要么全部不是
	 *
	 *
	 */
	public static boolean processERChildTable(final SchemaConfig schema, final String origSQL,
			final ServerConnection sc) throws SQLNonTransientException {

		MySqlStatementParser parser = new MySqlStatementParser(origSQL);
		List statements = parser.parseStatementList();

		if(statements == null || statements.isEmpty() ) {
			throw new SQLNonTransientException(String.format("无效的SQL语句:%s", origSQL));
		}


		boolean erFlag = false; //是否是er表
		for(SQLStatement stmt : statements ) {
			MySqlInsertStatement insertStmt = (MySqlInsertStatement) stmt;
			String tableName = insertStmt.getTableName().getSimpleName().toUpperCase();
			final TableConfig tc = schema.getTables().get(tableName);

			if (null != tc && tc.isChildTable()) {
				erFlag = true;

				String sql = insertStmt.toString();

				final RouteResultset rrs = new RouteResultset(sql, ServerParse.INSERT);
				String joinKey = tc.getJoinKey();
				//因为是Insert语句，用MySqlInsertStatement进行parse
//				MySqlInsertStatement insertStmt = (MySqlInsertStatement) (new MySqlStatementParser(origSQL)).parseInsert();
				//判断条件完整性，取得解析后语句列中的joinkey列的index
				int joinKeyIndex = getJoinKeyIndex(insertStmt.getColumns(), joinKey);
				if (joinKeyIndex == -1) {
					String inf = "joinKey not provided :" + tc.getJoinKey() + "," + insertStmt;
					LOGGER.warn(inf);
					throw new SQLNonTransientException(inf);
				}
				//子表不支持批量插入
				if (isMultiInsert(insertStmt)) {
					String msg = "ChildTable multi insert not provided";
					LOGGER.warn(msg);
					throw new SQLNonTransientException(msg);
				}
				//取得joinkey的值
				String joinKeyVal = insertStmt.getValues().getValues().get(joinKeyIndex).toString();
				//解决bug #938，当关联字段的值为char类型时，去掉前后"'"
				String realVal = joinKeyVal;
				if (joinKeyVal.startsWith("'") && joinKeyVal.endsWith("'") && joinKeyVal.length() > 2) {
					realVal = joinKeyVal.substring(1, joinKeyVal.length() - 1);
				}

				// try to route by ER parent partion key
				//如果是二级子表（父表不再有父表）,并且分片字段正好是joinkey字段，调用routeByERParentKey
				RouteResultset theRrs = RouterUtil.routeByERParentKey(sc, schema, ServerParse.INSERT, sql, rrs, tc, realVal);
				if (theRrs != null) {
					boolean processedInsert=false;
					//判断是否需要全局序列号
					if ( sc!=null && tc.isAutoIncrement()) {
						String primaryKey = tc.getPrimaryKey();
						processedInsert=processInsert(sc,schema,ServerParse.INSERT,sql,tc.getName(),primaryKey);
					}
					if(processedInsert==false){
						rrs.setFinishedRoute(true);
						sc.getSession2().execute(rrs, ServerParse.INSERT);
					}
					// return true;
					//继续处理下一条
					continue;
				}

				// route by sql query root parent's datanode
				//如果不是二级子表或者分片字段不是joinKey字段结果为空，则启动异步线程去后台分片查询出datanode
				//只要查询出上一级表的parentkey字段的对应值在哪个分片即可
				final String findRootTBSql = tc.getLocateRTableKeySql().toLowerCase() + joinKeyVal;
				if (LOGGER.isDebugEnabled()) {
					LOGGER.debug("find root parent's node sql " + findRootTBSql);
				}

				ListenableFuture listenableFuture = MycatServer.getInstance().
						getListeningExecutorService().submit(new Callable() {
					@Override
					public String call() throws Exception {
						if (tc.getRootParent().getFetchStoreNodeByJdbc()) {
							JDBCFetchStoreNodeOfChildTableHandler jdbcFetchStoreNodeOfChildTableHandler =
									new JDBCFetchStoreNodeOfChildTableHandler();
							return jdbcFetchStoreNodeOfChildTableHandler
									.execute(schema.getName(), findRootTBSql, tc.getRootParent().getDataNodes());
						} else {
							FetchStoreNodeOfChildTableHandler fetchHandler = new FetchStoreNodeOfChildTableHandler();
							return fetchHandler.execute(schema.getName(), findRootTBSql, tc.getRootParent().getDataNodes(), sc);
						}

					}
				});


				Futures.addCallback(listenableFuture, new FutureCallback() {
					@Override
					public void onSuccess(String result) {
						//结果为空，证明上一级表中不存在那条记录，失败
						if (Strings.isNullOrEmpty(result)) {
							StringBuilder s = new StringBuilder();
							LOGGER.warn(s.append(sc.getSession2()).append(origSQL).toString() +
									" err:" + "can't find (root) parent sharding node for sql:" + origSQL);
							if(!sc.isAutocommit()) { // 处于事务下失败, 必须回滚
								sc.setTxInterrupt("can't find (root) parent sharding node for sql:" + origSQL);
							}
							sc.writeErrMessage(ErrorCode.ER_PARSE_ERROR, "can't find (root) parent sharding node for sql:" + origSQL);
							return;
						}

						if (LOGGER.isDebugEnabled()) {
							LOGGER.debug("found partion node for child table to insert " + result + " sql :" + origSQL);
						}
						//找到分片，进行插入（和其他的一样，需要判断是否需要全局自增ID）
						boolean processedInsert=false;
						if ( sc!=null && tc.isAutoIncrement()) {
							try {
								String primaryKey = tc.getPrimaryKey();
								processedInsert=processInsert(sc,schema,ServerParse.INSERT,origSQL,tc.getName(),primaryKey);
							} catch (SQLNonTransientException e) {
								LOGGER.warn("sequence processInsert error,",e);
								sc.writeErrMessage(ErrorCode.ER_PARSE_ERROR , "sequence processInsert error," + e.getMessage());
							}
						}
						if(processedInsert==false){
							RouteResultset executeRrs = RouterUtil.routeToSingleNode(rrs, result, origSQL);
							sc.getSession2().execute(executeRrs, ServerParse.INSERT);
						}

					}

					@Override
					public void onFailure(Throwable t) {
						StringBuilder s = new StringBuilder();
						LOGGER.warn(s.append(sc.getSession2()).append(origSQL).toString() +
								" err:" + t.getMessage());
						sc.writeErrMessage(ErrorCode.ER_PARSE_ERROR, t.getMessage() + " " + s.toString());
					}
				}, MycatServer.getInstance().
						getListeningExecutorService());

			} else if(erFlag) {
				throw new SQLNonTransientException(String.format("%s包含不是ER分片的表", origSQL));
			}
		}


		return erFlag;
	}

	/**
	 * 寻找joinKey的索引
	 *
	 * @param columns
	 * @param joinKey
	 * @return -1表示没找到，>=0表示找到了
	 */
	private static int getJoinKeyIndex(List columns, String joinKey) {
		for (int i = 0; i < columns.size(); i++) {
			String col = StringUtil.removeBackquote(columns.get(i).toString()).toUpperCase();
			if (col.equals(joinKey)) {
				return i;
			}
		}
		return -1;
	}

	/**
	 * 是否为批量插入：insert into ...values (),()...或 insert into ...select.....
	 *
	 * @param insertStmt
	 * @return
	 */
	private static boolean isMultiInsert(MySqlInsertStatement insertStmt) {
		return (insertStmt.getValuesList() != null && insertStmt.getValuesList().size() > 1)
				|| insertStmt.getQuery() != null;
	}
    /**
     * escape white spaces and get the real start position.
     * @author kevin
     * @param stmt  The sql statement.
     * @param startPos  The initial start position.
     * @return  int  The real start position.
     */
    private static int getStartPos(String stmt, int startPos) {
        while (startPos < stmt.length()) {
            if (!Character.isWhitespace(stmt.charAt(startPos))) {
                break;
            }
            ++startPos;
        }
        return startPos;
    }
}


================================================
FILE: src/main/java/io/mycat/server/NonBlockingSession.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese
 * opensource volunteers. you can redistribute it and/or modify it under the
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Any questions about this component can be directed to it's project Web address
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server;

import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import io.mycat.MycatServer;
import io.mycat.backend.BackendConnection;
import io.mycat.backend.datasource.PhysicalDBNode;
import io.mycat.backend.mysql.listener.SqlExecuteStage;
import io.mycat.backend.mysql.nio.handler.CommitNodeHandler;
import io.mycat.backend.mysql.nio.handler.KillConnectionHandler;
import io.mycat.backend.mysql.nio.handler.LockTablesHandler;
import io.mycat.backend.mysql.nio.handler.MiddlerResultHandler;
import io.mycat.backend.mysql.nio.handler.MultiNodeCoordinator;
import io.mycat.backend.mysql.nio.handler.MultiNodeQueryHandler;
import io.mycat.backend.mysql.nio.handler.RollbackNodeHandler;
import io.mycat.backend.mysql.nio.handler.RollbackReleaseHandler;
import io.mycat.backend.mysql.nio.handler.SingleNodeHandler;
import io.mycat.backend.mysql.nio.handler.UnLockTablesHandler;
import io.mycat.config.ErrorCode;
import io.mycat.config.MycatConfig;
import io.mycat.net.FrontendConnection;
import io.mycat.net.mysql.OkPacket;
import io.mycat.route.RouteResultset;
import io.mycat.route.RouteResultsetNode;
import io.mycat.server.parser.ServerParse;
import io.mycat.server.sqlcmd.SQLCmdConstant;

/**
 * @author mycat
 * @author mycat
 */
public class NonBlockingSession implements Session {

    public static final Logger LOGGER = LoggerFactory.getLogger(NonBlockingSession.class);

    private final ServerConnection source;
    //huangyiming add 避免出现jdk版本冲突
    private final ConcurrentMap target;
    // life-cycle: each sql execution
    private volatile SingleNodeHandler singleNodeHandler;
    private volatile MultiNodeQueryHandler multiNodeHandler;
    private volatile RollbackNodeHandler rollbackHandler;
    private final MultiNodeCoordinator multiNodeCoordinator;
    private final CommitNodeHandler commitHandler;
    private volatile String xaTXID;

   //huangyiming
  	private  volatile boolean canClose = true;

  	private volatile MiddlerResultHandler  middlerResultHandler;
    private boolean prepared;
    private RouteResultset rrs;

    public NonBlockingSession(ServerConnection source) {
        this.source = source;
        this.target = new ConcurrentHashMap(2, 0.75f);
        multiNodeCoordinator = new MultiNodeCoordinator(this);
        commitHandler = new CommitNodeHandler(this);
    }

    @Override
    public ServerConnection getSource() {
        return source;
    }

    @Override
    public int getTargetCount() {
        return target.size();
    }

    public Set getTargetKeys() {
        return target.keySet();
    }

    public BackendConnection getTarget(RouteResultsetNode key) {
        return target.get(key);
    }

    public Map getTargetMap() {
        return this.target;
    }

    public BackendConnection removeTarget(RouteResultsetNode key) {
        return target.remove(key);
    }

    @Override
    public void execute(RouteResultset rrs, int type) {
        this.rrs = rrs;
        // clear prev execute resources
        clearHandlesResources();
        if (LOGGER.isDebugEnabled()) {
            StringBuilder s = new StringBuilder();
            LOGGER.debug(s.append(source).append(rrs).toString() + " rrs ");
        }

        // 检查路由结果是否为空
        RouteResultsetNode[] nodes = rrs.getNodes();
        if (nodes == null || nodes.length == 0 || nodes[0].getName() == null || nodes[0].getName().equals("")) {
            source.writeErrMessage(ErrorCode.ER_NO_DB_ERROR,
                    "No dataNode found ,please check tables defined in schema:" + source.getSchema());
            source.getListener().fireEvent(SqlExecuteStage.END);
            return;
        }
        boolean autocommit = source.isAutocommit();
        final int initCount = target.size();
        if (nodes.length == 1) {
            singleNodeHandler = new SingleNodeHandler(rrs, this);
            if (this.isPrepared()) {
                singleNodeHandler.setPrepared(true);
            }

            try {
                if(initCount > 1){
                    checkDistriTransaxAndExecute(rrs,1,autocommit);
                }else{
                    singleNodeHandler.execute();
                }
            } catch (Exception e) {
                LOGGER.warn(new StringBuilder().append(source).append(rrs).toString(), e);
                source.writeErrMessage(ErrorCode.ERR_HANDLE_DATA, e.toString());
                source.getListener().fireEvent(SqlExecuteStage.END);
            }

        } else {

            multiNodeHandler = new MultiNodeQueryHandler(type, rrs, autocommit, this);
            if (this.isPrepared()) {
                multiNodeHandler.setPrepared(true);
            }
            try {
                if(((type == ServerParse.DELETE || type == ServerParse.INSERT || type == ServerParse.UPDATE) && !rrs.isGlobalTable() && nodes.length > 1)||initCount > 1) {
                    checkDistriTransaxAndExecute(rrs,2,autocommit);
                } else {
                    multiNodeHandler.execute();
                }
            } catch (Exception e) {
                LOGGER.warn(new StringBuilder().append(source).append(rrs).toString(), e);
                source.writeErrMessage(ErrorCode.ERR_HANDLE_DATA, e.toString());
                source.getListener().fireEvent(SqlExecuteStage.END);
            }
        }

        if (this.isPrepared()) {
            this.setPrepared(false);
        }
    }

    private void checkDistriTransaxAndExecute(RouteResultset rrs, int type,boolean autocommit) throws Exception {
        switch(MycatServer.getInstance().getConfig().getSystem().getHandleDistributedTransactions()) {
            case 1:
                source.writeErrMessage(ErrorCode.ER_NOT_ALLOWED_COMMAND, "Distributed transaction is disabled!");
                if(!autocommit){
                    source.setTxInterrupt("Distributed transaction is disabled!");
                }
                break;
            case 2:
                LOGGER.warn("Distributed transaction detected! RRS:" + rrs);
                if(type == 1){
                    singleNodeHandler.execute();
                }
                else{
                    multiNodeHandler.execute();
                }
                break;
            default:
                if(type == 1){
                    singleNodeHandler.execute();
                }
                else{
                    multiNodeHandler.execute();
                }
        }
    }

    private void checkDistriTransaxAndExecute() {
        if(!isALLGlobal()){
            switch(MycatServer.getInstance().getConfig().getSystem().getHandleDistributedTransactions()) {
                case 1:
                    source.writeErrMessage(ErrorCode.ER_NOT_ALLOWED_COMMAND, "Distributed transaction is disabled!Please rollback!");
                    source.setTxInterrupt("Distributed transaction is disabled!");
                    break;
                case 2:
                    multiNodeCoordinator.executeBatchNodeCmd(SQLCmdConstant.COMMIT_CMD);
                    LOGGER.warn("Distributed transaction detected! Targets:" + target);
                    break;
                default:
                    multiNodeCoordinator.executeBatchNodeCmd(SQLCmdConstant.COMMIT_CMD);

            }
        } else {
            multiNodeCoordinator.executeBatchNodeCmd(SQLCmdConstant.COMMIT_CMD);
        }
    }

    public void commit() {
        final int initCount = target.size();
        if (initCount <= 0) {
            ByteBuffer buffer = source.allocate();
            buffer = source.writeToBuffer(OkPacket.OK, buffer);
            source.write(buffer);
            /* 1. 如果开启了 xa 事务 */
            if(getXaTXID()!=null){
				setXATXEnabled(false);
			}
            /* 2. preAcStates 为true,事务结束后,需要设置为true。preAcStates 为ac上一个状态    */
            if(source.isPreAcStates()&&!source.isAutocommit()){
            	source.setAutocommit(true);
            }
            return;
        } else if (initCount == 1) {
        	//huangyiming add 避免出现jdk版本冲突
            // BackendConnection con = target.values().iterator().next();
            commitHandler.commit();
        } else {

            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug("multi node commit to send ,total " + initCount);
            }
            checkDistriTransaxAndExecute();
        }

    }

    private boolean isALLGlobal(){
        for(RouteResultsetNode routeResultsetNode:target.keySet()){
            if(routeResultsetNode.getSource()==null){
                return false;
            }
            else if(!routeResultsetNode.getSource().isGlobalTable()){
                return false;
            }
        }
        return true;
    }

    public void rollback() {
        final int initCount = target.size();
        if (initCount <= 0) {
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug("no session bound connections found ,no need send rollback cmd ");
            }
            ByteBuffer buffer = source.allocate();
            buffer = source.writeToBuffer(OkPacket.OK, buffer);
            source.write(buffer);
            /* 1. 如果开启了 xa 事务 */
            if(getXaTXID()!=null){
				setXATXEnabled(false);
			}
            /* 2. preAcStates 为true,事务结束后,需要设置为true。preAcStates 为ac上一个状态    */
            if(source.isPreAcStates()&&!source.isAutocommit()){
            	source.setAutocommit(true);
            }
            return;
        }

        rollbackHandler = new RollbackNodeHandler(this);
        rollbackHandler.rollback();
    }

	/**
	 * 执行lock tables语句方法
	 * @author songdabin
	 * @date 2016-7-9
	 * @param rrs
	 */
	public void lockTable(RouteResultset rrs) {
		// 检查路由结果是否为空
		RouteResultsetNode[] nodes = rrs.getNodes();
		if (nodes == null || nodes.length == 0 || nodes[0].getName() == null
				|| nodes[0].getName().equals("")) {
			source.writeErrMessage(ErrorCode.ER_NO_DB_ERROR,
					"No dataNode found ,please check tables defined in schema:"
							+ source.getSchema());
			return;
		}
		LockTablesHandler handler = new LockTablesHandler(this, rrs);
		source.setLocked(true);
		try {
			handler.execute();
		} catch (Exception e) {
			LOGGER.warn(new StringBuilder().append(source).append(rrs).toString(), e);
			source.writeErrMessage(ErrorCode.ERR_HANDLE_DATA, e.toString());
		}
	}

	/**
	 * 执行unlock tables语句方法
	 * @author songdabin
	 * @date 2016-7-9
	 * @param sql
	 */
	public void unLockTable(String sql) {
		UnLockTablesHandler handler = new UnLockTablesHandler(this, this.source.isAutocommit(), sql);
		handler.execute();
	}

    @Override
    public void cancel(FrontendConnection sponsor) {

    }

    /**
     * {@link ServerConnection#isClosed()} must be true before invoking this
     */
    public void terminate() {
        for (BackendConnection node : target.values()) {
            node.close("client closed ");
        }
        target.clear();
        clearHandlesResources();
    }

    public void closeAndClearResources(String reason) {
        for (BackendConnection node : target.values()) {
            node.closeWithoutRsp(reason);
        }
        target.clear();
        clearHandlesResources();
    }

    public void closeConnection(BackendConnection con, String reason) {
        Iterator> itor = target.entrySet().iterator();
        while (itor.hasNext()) {
            BackendConnection theCon = itor.next().getValue();
            if (theCon == con) {
                itor.remove();
                con.close(reason);
                if (LOGGER.isDebugEnabled()) {
                    LOGGER.debug("realse connection " + con);
                }
                break;
            }
        }

    }

    public void releaseConnectionIfSafe(BackendConnection conn, boolean debug,
                                        boolean needRollback) {
        RouteResultsetNode node = (RouteResultsetNode) conn.getAttachment();

        if (node != null) {
        	/*  分表 在
        	 *    1. 没有开启事务
        	 *    2. 读取走的从节点
        	 *    3. 没有执行过更新sql
        	 *    也需要释放连接
        	 */
//            if (node.isDisctTable()) {
//                return;
//            }
            if (MycatServer.getInstance().getConfig().getSystem().isStrictTxIsolation()) {
                // 如果是严格隔离级别模式的话,不考虑是否已经执行了modifiedSql,直接不释放连接
                if ((!this.source.isAutocommit() && !conn.isFromSlaveDB()) || this.source.isLocked()) {
                    return;
                }
            }
            if ((this.source.isAutocommit() || conn.isFromSlaveDB()
                    || !conn.isModifiedSQLExecuted()) && !this.source.isLocked()) {
                releaseConnection((RouteResultsetNode) conn.getAttachment(), LOGGER.isDebugEnabled(),
                        needRollback);
            }
        }
    }

    public void releaseConnection(RouteResultsetNode rrn, boolean debug,
                                  final boolean needRollback) {

        BackendConnection c = target.remove(rrn);
        if (c != null) {
            if (debug) {
                //LOGGER.debug("release connection " + c);
                String sql =  rrn.getStatement();
                if(sql!=null){
                    sql = sql.replaceAll("[\r\n]+", "");
                }
                LOGGER.debug("releaseConnection Connection@{} [id={}] for node={}, sql={}",
                    new Object[]{c.hashCode(), c.getId(), rrn.getName(), sql});
            }
            if (c.getAttachment() != null) {
                c.setAttachment(null);
            }
            if (!c.isClosedOrQuit()) {
                if (c.isAutocommit()) {
                    c.release();
                } else
                //if (needRollback)
                {
                    c.setResponseHandler(new RollbackReleaseHandler());
                    c.rollback();
                }
                //else {
				//	c.release();
				//}
            }
        }
    }

    public void releaseConnections(final boolean needRollback) {
        boolean debug = LOGGER.isDebugEnabled();

        for (RouteResultsetNode rrn : target.keySet()) {
            releaseConnection(rrn, debug, needRollback);
        }
    }

    public void releaseConnection(BackendConnection con) {
        Iterator> itor = target
                .entrySet().iterator();
        while (itor.hasNext()) {
            BackendConnection theCon = itor.next().getValue();
            if (theCon == con) {
                itor.remove();
                con.release();
                if (LOGGER.isDebugEnabled()) {
                    LOGGER.debug("realse connection " + con);
                }
                break;
            }
        }

    }

    /**
     * @return previous bound connection
     */
    public BackendConnection bindConnection(RouteResultsetNode key,
                                            BackendConnection conn) {
        // System.out.println("bind connection "+conn+
        // " to key "+key.getName()+" on sesion "+this);
        if(LOGGER.isDebugEnabled()){
            String sql =  key.getStatement();
            if(sql!=null){
                sql = sql.replaceAll("[\r\n]+", "");
            }
            LOGGER.debug("bindConnection Connection@{} [id={}] for node={}, sql={}",
                new Object[]{conn.hashCode(), conn.getId(), key.getName(), sql});
        }

        return target.put(key, conn);
    }

    public boolean tryExistsCon(final BackendConnection conn, RouteResultsetNode node) {
        if (conn == null) {
            return false;
        }

        boolean canReUse = false;
        // conn 是 slave db 的，并且 路由结果显示，本次sql可以重用该 conn
        if (conn.isFromSlaveDB() && (node.canRunnINReadDB(getSource().isAutocommit())
                && (node.getRunOnSlave() == null || node.getRunOnSlave()))) {
            canReUse = true;
        }

        // conn 是 master db 的，并且路由结果显示，本次sql可以重用该conn
        if (!conn.isFromSlaveDB() && (node.getRunOnSlave() == null || !node.getRunOnSlave())) {
            canReUse = true;
        }

        if (canReUse) {
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug("found connections in session to use " + conn
                        + " for " + node);
            }
            conn.setAttachment(node);
            return true;
        } else {
            // Previous connection and can't use anymore ,release it
            if (LOGGER.isWarnEnabled()) {
                LOGGER.warn("Release previous connection,can't be used in trasaction  "
                        + conn + " for " + node);
            }
            releaseConnection(node, LOGGER.isDebugEnabled(), false);
        }
        return false;
    }

//	public boolean tryExistsCon(final BackendConnection conn,
//			RouteResultsetNode node) {
//
//		if (conn == null) {
//			return false;
//		}
//		if (!conn.isFromSlaveDB()
//				|| node.canRunnINReadDB(getSource().isAutocommit())) {
//			if (LOGGER.isDebugEnabled()) {
//				LOGGER.debug("found connections in session to use " + conn
//						+ " for " + node);
//			}
//			conn.setAttachment(node);
//			return true;
//		} else {
//			// slavedb connection and can't use anymore ,release it
//			if (LOGGER.isDebugEnabled()) {
//				LOGGER.debug("release slave connection,can't be used in trasaction  "
//						+ conn + " for " + node);
//			}
//			releaseConnection(node, LOGGER.isDebugEnabled(), false);
//		}
//		return false;
//	}

    protected void kill() {
        boolean hooked = false;
        AtomicInteger count = null;
        Map killees = null;
        for (RouteResultsetNode node : target.keySet()) {
            BackendConnection c = target.get(node);
            if (c != null) {
                if (!hooked) {
                    hooked = true;
                    killees = new HashMap();
                    count = new AtomicInteger(0);
                }
                killees.put(node, c);
                count.incrementAndGet();
            }
        }
        if (hooked) {
            for (Entry en : killees
                    .entrySet()) {
                KillConnectionHandler kill = new KillConnectionHandler(
                        en.getValue(), this);
                MycatConfig conf = MycatServer.getInstance().getConfig();
                PhysicalDBNode dn = conf.getDataNodes().get(
                        en.getKey().getName());
                try {
                    dn.getConnectionFromSameSource(null, true, en.getValue(),
                            kill, en.getKey());
                } catch (Exception e) {
                    LOGGER.error(
                            "get killer connection failed for " + en.getKey(),
                            e);
                    kill.connectionError(e, null);
                }
            }
        }
    }

    private void clearHandlesResources() {
        SingleNodeHandler singleHander = singleNodeHandler;
        if (singleHander != null) {
            singleHander.clearResources();
            singleNodeHandler = null;
        }
        MultiNodeQueryHandler multiHandler = multiNodeHandler;
        if (multiHandler != null) {
            multiHandler.clearResources();
            multiNodeHandler = null;
        }
    }

    public void clearResources(final boolean needRollback) {
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("clear session resources " + this);
        }
        this.releaseConnections(needRollback);
        clearHandlesResources();
    }

    public boolean closed() {
        return source.isClosed();
    }

    private String genXATXID() {
        return MycatServer.getInstance().getXATXIDGLOBAL();
    }

    public void setXATXEnabled(boolean xaTXEnabled) {

        if (xaTXEnabled) {
        	LOGGER.info("XA Transaction enabled ,con " + this.getSource());
        	if(this.xaTXID == null){
        		xaTXID = genXATXID();
        	}
        }else{
        	LOGGER.info("XA Transaction disabled ,con " + this.getSource());
        	this.xaTXID = null;
        }
    }

    public String getXaTXID() {
        return xaTXID;
    }

    public boolean isPrepared() {
        return prepared;
    }

    public void setPrepared(boolean prepared) {
        this.prepared = prepared;
    }


	public boolean isCanClose() {
		return canClose;
	}

	public void setCanClose(boolean canClose) {
		this.canClose = canClose;
	}

	public MiddlerResultHandler getMiddlerResultHandler() {
		return middlerResultHandler;
	}

	public void setMiddlerResultHandler(MiddlerResultHandler middlerResultHandler) {
		this.middlerResultHandler = middlerResultHandler;
	}

    public void setAutoCommitStatus() {
		/* 1.  事务结束后,xa事务结束    */
		if(this.getXaTXID()!=null){
			this.setXATXEnabled(false);
		}
		/* 2. preAcStates 为true,事务结束后,需要设置为true。preAcStates 为ac上一个状态    */
		if(this.getSource().isPreAcStates()&&!this.getSource().isAutocommit()){
			this.getSource().setAutocommit(true);
        }
		this.getSource().clearTxInterrupt();

    }
	@Override
	public String toString() {
		// TODO Auto-generated method stub
		StringBuilder sb = new StringBuilder();
		for (BackendConnection backCon : target.values()) {
			sb.append(backCon).append("\r\n");
		}
		return sb.toString();
	}

    public RouteResultset getRrs() {
        return rrs;
    }

}


================================================
FILE: src/main/java/io/mycat/server/ServerConnection.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server;

import java.io.IOException;
import java.nio.channels.NetworkChannel;
import java.util.Queue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import io.mycat.MycatServer;
import io.mycat.backend.mysql.listener.DefaultSqlExecuteStageListener;
import io.mycat.backend.mysql.listener.SqlExecuteStageListener;
import io.mycat.config.ErrorCode;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.SystemConfig;
import io.mycat.net.FrontendConnection;
import io.mycat.route.RouteResultset;
import io.mycat.server.handler.MysqlProcHandler;
import io.mycat.server.parser.ServerParse;
import io.mycat.server.response.Heartbeat;
import io.mycat.server.response.InformationSchemaProfiling;
import io.mycat.server.response.InformationSchemaProfilingSqlyog;
import io.mycat.server.response.Ping;
import io.mycat.server.util.SchemaUtil;
import io.mycat.util.SplitUtil;
import io.mycat.util.TimeUtil;

/**
 * @author mycat
 */
public class ServerConnection extends FrontendConnection {
	private static final Logger LOGGER = LoggerFactory
			.getLogger(ServerConnection.class);
	private long authTimeout = SystemConfig.DEFAULT_AUTH_TIMEOUT;

	/** 保存SET SQL_SELECT_LIMIT的值, default 解析为-1. */
	private volatile  int sqlSelectLimit = -1;
	private volatile  boolean txReadonly;
	private volatile int txIsolation;
	private volatile boolean autocommit;
	private volatile boolean preAcStates; //上一个ac状态,默认为true
	private volatile boolean txInterrupted;
	private volatile String txInterrputMsg = "";
	private long lastInsertId;
	private NonBlockingSession session;
	/**
	 * 标志是否执行了lock tables语句，并处于lock状态
	 */
	private volatile boolean isLocked = false;
    private Queue executeSqlQueue;
    private SqlExecuteStageListener listener;
	
	public ServerConnection(NetworkChannel channel)
			throws IOException {
		super(channel);
		this.txInterrupted = false;
		this.autocommit = true;
		this.preAcStates = true;
		this.txReadonly = false;
        this.executeSqlQueue = new LinkedBlockingQueue<>();
        this.listener = new DefaultSqlExecuteStageListener(this);
	}

	@Override
	public boolean isIdleTimeout() {
		if (isAuthenticated) {
			return super.isIdleTimeout();
		} else {
			return TimeUtil.currentTimeMillis() > Math.max(lastWriteTime, lastReadTime) + this.authTimeout;
		}
	}

	public long getAuthTimeout() {
		return authTimeout;
	}

	public void setAuthTimeout(long authTimeout) {
		this.authTimeout = authTimeout;
	}

	public int getTxIsolation() {
		return txIsolation;
	}

	public void setTxIsolation(int txIsolation) {
		this.txIsolation = txIsolation;
	}

	public boolean isAutocommit() {
		return autocommit;
	}

	public void setAutocommit(boolean autocommit) {
		this.autocommit = autocommit;
	}

	public boolean isTxReadonly() {
		return txReadonly;
	}

	public void setTxReadonly(boolean txReadonly) {
		this.txReadonly = txReadonly;
	}

	public int getSqlSelectLimit() {
		return sqlSelectLimit;
	}

	public void setSqlSelectLimit(int sqlSelectLimit) {
		this.sqlSelectLimit = sqlSelectLimit;
	}

	public long getLastInsertId() {
		return lastInsertId;
	}

	public void setLastInsertId(long lastInsertId) {
		this.lastInsertId = lastInsertId;
	}

	/**
	 * 设置是否需要中断当前事务
	 */
	public void setTxInterrupt(String txInterrputMsg) {
		if (!autocommit && !txInterrupted) {
			txInterrupted = true;
			this.txInterrputMsg = txInterrputMsg;
		}
	}
	
	/**
	 * 
	 * 清空食事务中断
	 * */
	public void clearTxInterrupt() {
		if (!autocommit && txInterrupted) {
			txInterrupted = false;
			this.txInterrputMsg = "";
		}
	}
	
	public boolean isTxInterrupted()
	{
		return txInterrupted;
	}
	public NonBlockingSession getSession2() {
		return session;
	}

	public void setSession2(NonBlockingSession session2) {
		this.session = session2;
	}
	
	public boolean isLocked() {
		return isLocked;
	}

	public void setLocked(boolean isLocked) {
		this.isLocked = isLocked;
	}

	@Override
	public void ping() {
		Ping.response(this);
	}

	@Override
	public void heartbeat(byte[] data) {
		Heartbeat.response(this, data);
	}

    public void execute(String sql, int type) {
		//连接状态检查
		if (this.isClosed()) {
			LOGGER.warn("ignore execute ,server connection is closed " + this);
			return;
		}
		// 事务状态检查
		if (txInterrupted) {
			writeErrMessage(ErrorCode.ER_YES,
					"Transaction error, need to rollback." + txInterrputMsg);
			return;
		}

		// 检查当前使用的DB
		String db = this.schema;
		boolean isDefault = true;
		if (db == null) {
			db = SchemaUtil.detectDefaultDb(sql, type);
			if (db == null) {
				db = MycatServer.getInstance().getConfig().getUsers().get(user).getDefaultSchema();
				if (db == null) {
					writeErrMessage(ErrorCode.ERR_BAD_LOGICDB,
							"No MyCAT Database selected");
					return ;
				}
			}
			isDefault = false;
		}
		
		// 兼容PhpAdmin's, 支持对MySQL元数据的模拟返回
		//// TODO: 2016/5/20 支持更多information_schema特性
//		if (ServerParse.SELECT == type
//				&& db.equalsIgnoreCase("information_schema") ) {
//			MysqlInformationSchemaHandler.handle(sql, this);
//			return;
//		}

		if (ServerParse.SELECT == type 
				&& sql.contains("mysql") 
				&& sql.contains("proc")) {
			
			SchemaUtil.SchemaInfo schemaInfo = SchemaUtil.parseSchema(sql);
			if (schemaInfo != null 
					&& "mysql".equalsIgnoreCase(schemaInfo.schema)
					&& "proc".equalsIgnoreCase(schemaInfo.table)) {
				
				// 兼容MySQLWorkbench
				MysqlProcHandler.handle(sql, this);
				return;
			}
		}
		
		SchemaConfig schema = MycatServer.getInstance().getConfig().getSchemas().get(db);
		if (schema == null) {
			writeErrMessage(ErrorCode.ERR_BAD_LOGICDB,
					"Unknown MyCAT Database '" + db + "'");
			return;
		}

		//fix navicat   SELECT STATE AS `State`, ROUND(SUM(DURATION),7) AS `Duration`, CONCAT(ROUND(SUM(DURATION)/*100,3), '%') AS `Percentage` FROM INFORMATION_SCHEMA.PROFILING WHERE QUERY_ID= GROUP BY STATE ORDER BY SEQ
		if(ServerParse.SELECT == type &&sql.contains(" INFORMATION_SCHEMA.PROFILING ")&&sql.contains("CONCAT(ROUND(SUM(DURATION)/"))
		{
			InformationSchemaProfiling.response(this);
			return;
		}

		//fix sqlyog select state, round(sum(duration),5) as `duration (summed) in sec` from information_schema.profiling where query_id = 0 group by state order by `duration (summed) in sec` desc
		if(ServerParse.SELECT == type &&sql.contains(" information_schema.profiling ")&&sql.contains("duration (summed) in sec"))
		{
			InformationSchemaProfilingSqlyog.response(this);
			return;
		}
		/* 当已经设置默认schema时，可以通过在sql中指定其它schema的方式执行
		 * 相关sql，已经在mysql客户端中验证。
		 * 所以在此处增加关于sql中指定Schema方式的支持。
		 */
		if (isDefault && schema.isCheckSQLSchema() && isNormalSql(type)) {
			SchemaUtil.SchemaInfo schemaInfo = SchemaUtil.parseSchema(sql);
			if (schemaInfo != null && schemaInfo.schema != null && !schemaInfo.schema.equals(db)) {
				SchemaConfig schemaConfig = MycatServer.getInstance().getConfig().getSchemas().get(schemaInfo.schema);
				if (schemaConfig != null)
					schema = schemaConfig;
			}
		}

		routeEndExecuteSQL(sql, type, schema);

	}
	
	private boolean isNormalSql(int type) {
		return ServerParse.SELECT==type||ServerParse.INSERT==type||ServerParse.UPDATE==type||ServerParse.DELETE==type||ServerParse.DDL==type;
	}

    public RouteResultset routeSQL(String sql, int type) {

		// 检查当前使用的DB
		String db = this.schema;
		if (db == null) {
			db = SchemaUtil.detectDefaultDb(sql, type);
			if (db == null){
				db = MycatServer.getInstance().getConfig().getUsers().get(user).getDefaultSchema();
				if (db == null) {
					writeErrMessage(ErrorCode.ERR_BAD_LOGICDB,
							"No MyCAT Database selected");
					return null;
				}
			}

		}
		SchemaConfig schema = MycatServer.getInstance().getConfig()
				.getSchemas().get(db);
		if (schema == null) {
			writeErrMessage(ErrorCode.ERR_BAD_LOGICDB,
					"Unknown MyCAT Database '" + db + "'");
			return null;
		}

		// 路由计算
		RouteResultset rrs = null;
		try {
			rrs = MycatServer
					.getInstance()
					.getRouterservice()
					.route(MycatServer.getInstance().getConfig().getSystem(),
							schema, type, sql, this.charset, this);

		} catch (Exception e) {
			StringBuilder s = new StringBuilder();
			LOGGER.warn(s.append(this).append(sql).toString() + " err:" + e.toString(),e);
			String msg = e.getMessage();
			writeErrMessage(ErrorCode.ER_PARSE_ERROR, msg == null ? e.getClass().getSimpleName() : msg);
			return null;
		}
		return rrs;
	}




	public void routeEndExecuteSQL(String sql, final int type, final SchemaConfig schema) {
		// 路由计算
		RouteResultset rrs = null;
		try {
			rrs = MycatServer
					.getInstance()
					.getRouterservice()
					.route(MycatServer.getInstance().getConfig().getSystem(),
							schema, type, sql, this.charset, this);

		} catch (Exception e) {
			StringBuilder s = new StringBuilder();
			LOGGER.warn(s.append(this).append(sql).toString() + " err:" + e.toString(),e);
			String msg = e.getMessage();
			writeErrMessage(ErrorCode.ER_PARSE_ERROR, msg == null ? e.getClass().getSimpleName() : msg);
			return;
		}
		if (rrs != null) {
            // #支持mariadb驱动useBatchMultiSend=true,连续接收到的sql先放入队列，等待前面处理完成后再继续处理。
            // 参考https://mariadb.com/kb/en/option-batchmultisend-description/
            boolean executeNow = false;
            synchronized (this.executeSqlQueue) {
                executeNow = this.executeSqlQueue.isEmpty();
                this.executeSqlQueue.add(new SqlEntry(sql, type, rrs));
                if (LOGGER.isDebugEnabled()) {
                    LOGGER.debug("add queue,executeSqlQueue size {}", executeSqlQueue.size());
                }
            }

            if (executeNow) {
                this.executeSqlId++;
                session.execute(rrs, rrs.isSelectForUpdate() ? ServerParse.UPDATE : type);
            }
        }
    }

	/**
	 * 提交事务
	 */
	public void commit() {
		if (txInterrupted) {
			LOGGER.warn("receive commit ,but found err message in Transaction {}",this);
			this.rollback();
//			writeErrMessage(ErrorCode.ER_YES,
//					"Transaction error, need to rollback.");
		} else {
			session.commit();
		}
	}

	/**
	 * 回滚事务
	 */
	public void rollback() {
		// 状态检查
		if (txInterrupted) {
			txInterrupted = false;
		}

		// 执行回滚
		session.rollback();
	}
	/**
	 * 执行lock tables语句方法
	 * @param sql
	 */
	public void lockTable(String sql) {
		// 事务中不允许执行lock table语句
		if (!autocommit) {
			writeErrMessage(ErrorCode.ER_YES, "can't lock table in transaction!");
			return;
		}
		// 已经执行了lock table且未执行unlock table之前的连接不能再次执行lock table命令
		if (isLocked) {
			writeErrMessage(ErrorCode.ER_YES, "can't lock multi-table");
			return;
		}
		RouteResultset rrs = routeSQL(sql, ServerParse.LOCK);
		if (rrs != null) {
			session.lockTable(rrs);
		}
	}
	
	/**
	 * 执行unlock tables语句方法
	 * @param sql
	 */
	public void unLockTable(String sql) {
		sql = sql.replaceAll("\n", " ").replaceAll("\t", " ");
		String[] words = SplitUtil.split(sql, ' ', true);
		if (words.length==2 && ("table".equalsIgnoreCase(words[1]) || "tables".equalsIgnoreCase(words[1]))) {
			isLocked = false;
			session.unLockTable(sql);
		} else {
			writeErrMessage(ErrorCode.ER_UNKNOWN_COM_ERROR, "Unknown command");
		}
		
	}

	/**
	 * 撤销执行中的语句
	 * 
	 * @param sponsor
	 *            发起者为null表示是自己
	 */
	public void cancel(final FrontendConnection sponsor) {
		processor.getExecutor().execute(new Runnable() {
			@Override
			public void run() {
				session.cancel(sponsor);
			}
		});
	}

	@Override
	public void close(String reason) {
		super.close(reason);
		session.terminate();
		if(getLoadDataInfileHandler()!=null)
		{
			getLoadDataInfileHandler().clear();
		}
	}

	/**
	 * add huangyiming 检测字符串中某字符串出现次数
	 * @param srcText
	 * @param findText
	 * @return
	 */
	public static int appearNumber(String srcText, String findText) {
	    int count = 0;
	    Pattern p = Pattern.compile(findText);
	    Matcher m = p.matcher(srcText);
	    while (m.find()) {
	        count++;
	    }
	    return count;
	}
	@Override
	public String toString() {
		
		return "ServerConnection [id=" + id + ", schema=" + schema + ", host="
				+ host + ", user=" + user + ",txIsolation=" + txIsolation
				+ ", autocommit=" + autocommit + ", schema=" + schema+ ", executeSql=" + executeSql + "]" +
				this.getSession2();
		
	}

	public boolean isPreAcStates() {
		return preAcStates;
	}

	public void setPreAcStates(boolean preAcStates) {
		this.preAcStates = preAcStates;
	}

    public SqlExecuteStageListener getListener() {
        return listener;
    }

    public void setListener(SqlExecuteStageListener listener) {
        this.listener = listener;
    }

    @Override
    public void checkQueueFlow() {
        RouteResultset rrs = session.getRrs();
        if (rrs != null && rrs.getNodes().length > 1 && session.getRrs().needMerge()) {
            // 多节点合并结果集语句需要拉取所有数据，无法流控
            return;
        } else {
            // 非合并结果集语句进行流量控制检查。
            flowController.check(session.getTargetMap());
        }
    }

    @Override
    public void resetConnection() {
        // 1 简单点直接关闭后端连接。若按照mysql官方的提交事务或回滚事务，mycat都会回包给应用，引发包乱序。
        session.closeAndClearResources("receive com_reset_connection");

        // 2 重置用户变量
        this.txInterrupted = false;
        this.autocommit = true;
        this.preAcStates = true;
        this.txReadonly = false;
        this.lastInsertId = 0;

        super.resetConnection();
    }
    /**
     * sql执行完成后回调函数
     */
    public void onEventSqlCompleted() {
        SqlEntry sqlEntry = null;
        synchronized (this.executeSqlQueue) {
            this.executeSqlQueue.poll();// 弹出已经执行成功的
            sqlEntry = this.executeSqlQueue.peek();
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug("poll queue,executeSqlQueue size {}", this.executeSqlQueue.size());
            }
        }
        if (sqlEntry != null) {
            this.executeSqlId++;
            session.execute(sqlEntry.rrs, sqlEntry.rrs.isSelectForUpdate() ? ServerParse.UPDATE : sqlEntry.type);
        }
    }

    private class SqlEntry {
        public String sql;
        public int type;
        public RouteResultset rrs;

        public SqlEntry(String sql, int type, RouteResultset rrs) {
            this.sql = sql;
            this.type = type;
            this.rrs = rrs;
        }
    }

}


================================================
FILE: src/main/java/io/mycat/server/ServerConnectionFactory.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server;

import java.io.IOException;
import java.nio.channels.NetworkChannel;

import io.mycat.MycatServer;
import io.mycat.config.MycatPrivileges;
import io.mycat.config.model.SystemConfig;
import io.mycat.net.FrontendConnection;
import io.mycat.net.factory.FrontendConnectionFactory;
import io.mycat.server.handler.ServerLoadDataInfileHandler;
import io.mycat.server.handler.ServerPrepareHandler;

/**
 * @author mycat
 */
public class ServerConnectionFactory extends FrontendConnectionFactory {

    @Override
    protected FrontendConnection getConnection(NetworkChannel channel) throws IOException {
        SystemConfig sys = MycatServer.getInstance().getConfig().getSystem();
        ServerConnection c = new ServerConnection(channel);
        MycatServer.getInstance().getConfig().setSocketParams(c, true);
        c.setAuthTimeout(sys.getAuthTimeout());
        c.setPrivileges(MycatPrivileges.instance());
        c.setQueryHandler(new ServerQueryHandler(c));
        c.setLoadDataInfileHandler(new ServerLoadDataInfileHandler(c));
        c.setPrepareHandler(new ServerPrepareHandler(c,sys.getMaxPreparedStmtCount()));
        c.setTxIsolation(sys.getTxIsolation());
        c.setSession2(new NonBlockingSession(c));
        return c;
    }

}

================================================
FILE: src/main/java/io/mycat/server/ServerQueryHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese
 * opensource volunteers. you can redistribute it and/or modify it under the
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Any questions about this component can be directed to it's project Web address
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import io.mycat.config.ErrorCode;
import io.mycat.net.handler.FrontendQueryHandler;
import io.mycat.net.mysql.OkPacket;
import io.mycat.route.RouteService;
import io.mycat.server.handler.BeginHandler;
import io.mycat.server.handler.CommandHandler;
import io.mycat.server.handler.Explain2Handler;
import io.mycat.server.handler.ExplainHandler;
import io.mycat.server.handler.KillHandler;
import io.mycat.server.handler.MigrateHandler;
import io.mycat.server.handler.SavepointHandler;
import io.mycat.server.handler.SelectHandler;
import io.mycat.server.handler.SetHandler;
import io.mycat.server.handler.ShowHandler;
import io.mycat.server.handler.StartHandler;
import io.mycat.server.handler.UseHandler;
import io.mycat.server.parser.ServerParse;

/**
 * @author mycat
 */
public class ServerQueryHandler implements FrontendQueryHandler {
	private static final Logger LOGGER = LoggerFactory
			.getLogger(ServerQueryHandler.class);

	private final ServerConnection source;
	protected Boolean readOnly;

	public void setReadOnly(Boolean readOnly) {
		this.readOnly = readOnly;
	}

	public ServerQueryHandler(ServerConnection source) {
		this.source = source;
	}

	@Override
	public void query(String sql) {

		ServerConnection c = this.source;
		if (LOGGER.isDebugEnabled()) {
			LOGGER.debug(new StringBuilder().append(c).append(sql).toString());
		}
		//
		int rs = ServerParse.parse(sql);
		int sqlType = rs & 0xff;

		switch (sqlType) {
		//explain sql
		case ServerParse.EXPLAIN:
			ExplainHandler.handle(sql, c, rs >>> 8);
			break;
		//explain2 datanode=? sql=?
		case ServerParse.EXPLAIN2:
			Explain2Handler.handle(sql, c, rs >>> 8);
			break;
		case ServerParse.COMMAND:
			CommandHandler.handle(sql, c, 16);
			break;
		case ServerParse.SET:
			SetHandler.handle(sql, c, rs >>> 8);
			break;
		case ServerParse.SHOW:
			ShowHandler.handle(sql, c, rs >>> 8);
			break;
		case ServerParse.SELECT:
			SelectHandler.handle(sql, c, rs >>> 8);
			break;
		case ServerParse.START:
			StartHandler.handle(sql, c, rs >>> 8);
			break;
		case ServerParse.BEGIN:
			BeginHandler.handle(sql, c);
			break;
		//不支持oracle的savepoint事务回退点
		case ServerParse.SAVEPOINT:
			SavepointHandler.handle(sql, c);
			break;
		case ServerParse.KILL:
			KillHandler.handle(sql, rs >>> 8, c);
			break;
		//不支持KILL_Query
		case ServerParse.KILL_QUERY:
			LOGGER.warn(new StringBuilder().append("Unsupported command:").append(sql).toString());
			c.writeErrMessage(ErrorCode.ER_UNKNOWN_COM_ERROR,"Unsupported command");
			break;
		case ServerParse.USE:
			UseHandler.handle(sql, c, rs >>> 8);
			break;
		case ServerParse.COMMIT:
			c.commit();
			break;
		case ServerParse.ROLLBACK:
			c.rollback();
			break;
		case ServerParse.HELP:
			LOGGER.warn(new StringBuilder().append("Unsupported command:").append(sql).toString());
			c.writeErrMessage(ErrorCode.ER_SYNTAX_ERROR, "Unsupported command");
			break;
		case ServerParse.MYSQL_CMD_COMMENT:
			c.write(c.writeToBuffer(OkPacket.OK, c.allocate()));
			break;
		case ServerParse.MYSQL_COMMENT:
			c.write(c.writeToBuffer(OkPacket.OK, c.allocate()));
			break;
        case ServerParse.LOAD_DATA_INFILE_SQL:
			if(RouteService.isHintSql(sql) > -1){ // 目前仅支持注解 datanode,原理为直接将导入sql发送到指定mysql节点
				c.execute(sql , ServerParse.LOAD_DATA_INFILE_SQL);
			}else{
				c.loadDataInfileStart(sql);
			}
			break;
		case ServerParse.MIGRATE: {
		    try {
                MigrateHandler.handle(sql, c);
            }catch (Throwable e){
		        //MigrateHandler中InterProcessMutex slaveIDsLock 会连接zk,zk连接不上会导致类加载失败,
                // 此后再调用此命令,将会出现类未定义,所以最终还是需要重启mycat
		        e.printStackTrace();
                String msg = "Mycat is not connected to zookeeper!!\n";
                msg += "Please start zookeeper and restart mycat so that this mycat can temporarily execute the migration command.If other mycat does not connect to this zookeeper, they will not be able to perceive changes in the migration task.\n";
                msg += "After starting zookeeper,you can command tas follow:\n\nmigrate -table=schema.test -add=dn2,dn3 -force=true\n\nto perform the migration.\n";
                LOGGER.error(e.getMessage());
                LOGGER.error(msg);
                c.writeErrMessage(ErrorCode.ER_UNKNOWN_ERROR, msg);
            }
			break;
		}
		case ServerParse.LOCK:
        	c.lockTable(sql);
        	break;
        case ServerParse.UNLOCK:
        	c.unLockTable(sql);
        	break;
		default:
			if(readOnly){
				LOGGER.warn(new StringBuilder().append("User readonly:").append(sql).toString());
				c.writeErrMessage(ErrorCode.ER_USER_READ_ONLY, "User readonly");
				break;
			}
			c.execute(sql, rs & 0xff);
		}

		switch (sqlType) {
			case ServerParse.SELECT:
			case ServerParse.DELETE:
			case ServerParse.UPDATE:
			case ServerParse.INSERT:
			case ServerParse.COMMAND:
				// curd 在后面会更新
				break;
			default:
				c.setExecuteSql(null);
		}
	}

}


================================================
FILE: src/main/java/io/mycat/server/Session.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server;

import io.mycat.net.FrontendConnection;
import io.mycat.route.RouteResultset;

/**
 * @author mycat
 */
public interface Session {

    /**
     * 取得源端连接
     */
    FrontendConnection getSource();

    /**
     * 取得当前目标端数量
     */
    int getTargetCount();

    /**
     * 开启一个会话执行
     */
    void execute(RouteResultset rrs, int type);

    /**
     * 提交一个会话执行
     */
    void commit();

    /**
     * 回滚一个会话执行
     */
    void rollback();

    /**
     * 取消一个正在执行中的会话
     * 
     * @param sponsor
     *            如果发起者为null，则表示由自己发起。
     */
    void cancel(FrontendConnection sponsor);

    /**
     * 终止会话，必须在关闭源端连接后执行该方法。
     */
    void terminate();
    

}

================================================
FILE: src/main/java/io/mycat/server/handler/BeginHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server.handler;

import io.mycat.server.ServerConnection;

/**
 * @author mycat
 */
public final class BeginHandler {
    private static final byte[] AC_OFF = new byte[] { 7, 0, 0, 1, 0, 0, 0, 0,
            0, 0, 0 };
    public static void handle(String stmt, ServerConnection c) {
        if (c.isAutocommit())
        {
            c.write(c.writeToBuffer(AC_OFF, c.allocate()));
        }else
        {
            c.getSession2().commit() ;
        }
        c.setAutocommit(false);
    }

}

================================================
FILE: src/main/java/io/mycat/server/handler/CommandHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server.handler;
import java.sql.SQLNonTransientException;
import java.util.List;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import io.mycat.MycatServer;
import io.mycat.backend.BackendConnection;
import io.mycat.backend.mysql.nio.handler.SimpleLogHandler;
import io.mycat.backend.mysql.nio.handler.SingleNodeHandler;
import io.mycat.config.ErrorCode;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.SystemConfig;
import io.mycat.net.mysql.FieldPacket;
import io.mycat.route.RouteResultset;
import io.mycat.server.NonBlockingSession;
import io.mycat.server.ServerConnection;
import io.mycat.server.parser.ServerParse;
import io.mycat.server.util.SchemaUtil;
import io.mycat.statistic.stat.QueryResult;
import io.mycat.statistic.stat.QueryResultDispatcher;
/**
 * 
 * 用于解决mysql 协议中com_field_list命令的支持
 * https://dev.mysql.com/doc/internals/en/com-field-list.html
 * 

 * @author stones_he@163.com
 */
public class CommandHandler {
	private static final Logger logger = LoggerFactory.getLogger(CommandHandler.class);
	//
	public static void handle(String stmt, ServerConnection c, int offset) {
		String table = stmt.substring(offset).trim();
		//int sqlType = ServerParse.parse(stmt) & 0xff;
		String db = c.getSchema();
		if (db == null) {
			db = SchemaUtil.detectDefaultDb(stmt, ServerParse.COMMAND);
			if (db == null) {
				c.writeErrMessage(ErrorCode.ER_NO_DB_ERROR, "No database selected");
				return;
			}
		}
		SchemaConfig schema = MycatServer.getInstance().getConfig().getSchemas().get(db);
		if (schema == null) {
			c.writeErrMessage(ErrorCode.ER_BAD_DB_ERROR, "Unknown database '" + db + "'");
			return;
		}
		SystemConfig system = MycatServer.getInstance().getConfig().getSystem();
		RouteResultset rrs = null;
		try {
			rrs = MycatServer	.getInstance()
								.getRouterservice()
								.route(system, schema, ServerParse.COMMAND, stmt, c.getCharset(), c);
			if (rrs == null) {
				return;
			}
		} catch (SQLNonTransientException e) {
			logger.warn("", e);
			c.writeErrMessage(ErrorCode.ERR_HANDLE_DATA, e.toString());
		}
		//
		CommandExecResultHandler handler = new CommandExecResultHandler(rrs, c.getSession2(), table);
		try {
			handler.execute();
		} catch (Exception e1) {
			logger.warn("", e1);
			c.writeErrMessage(ErrorCode.ERR_HANDLE_DATA, e1.toString());
		}
	}
}
class CommandExecResultHandler extends SingleNodeHandler {
	private static final Logger logger = LoggerFactory.getLogger(CommandExecResultHandler.class);
	private String table;
	public CommandExecResultHandler(RouteResultset rrs, NonBlockingSession session, String table) {
		super(rrs, session);
		this.table = table;
	}
	private void rowEofResponse0(byte[] eof, BackendConnection conn) {
		logger.debug("CommandExecResultHandler.rowEofResponse eof: {}", SimpleLogHandler.bytesToHex(eof));
		// 
		ServerConnection source = getSession().getSource();
		conn.recordSql(source.getHost(), source.getSchema(), getRouteResultsetNode().getStatement());
		// 判断是调用存储过程的话不能在这里释放链接
		if (!getRouteResultset().isCallStatement() || (getRouteResultset().isCallStatement() && getRouteResultset()
																													.getProcedure()
																													.isResultSimpleValue())) {
			getSession().releaseConnectionIfSafe(conn, logger.isDebugEnabled(), false);
			endRunning();
		}
		//
		int resultSize = source.getWriteQueue().size() * MycatServer.getInstance()
																	.getConfig()
																	.getSystem()
																	.getBufferPoolPageSize();
		resultSize = resultSize + getBuffer().position();
		if (!errorRepsponsed.get() && !getSession().closed() && source.canResponse()) {
			source.write(getBuffer());
		}
		source.setExecuteSql(null);
		//查询结果派发
        QueryResult queryResult = new QueryResult(getSession().getSource().getSchema(),
                getSession().getSource().getUser(),
                getRouteResultset().getSqlType(), getRouteResultset().getStatement(), affectedRows, netInBytes,
                netOutBytes, startTime, System.currentTimeMillis(), resultSize, source.getHost());
		QueryResultDispatcher.dispatchQuery(queryResult);
	}
	@Override
	public void fieldEofResponse(byte[] header, List fields, byte[] eof, BackendConnection conn) {
		logger.debug("CommandExecResultHandler.fieldEofResponse header: {}", SimpleLogHandler.bytesToHex(header));
		for (byte[] field : fields) {
			logger.debug("CommandExecResultHandler.fieldEofResponse fields: {}", SimpleLogHandler.bytesToHex(field));
		}
		logger.debug("CommandExecResultHandler.fieldEofResponse eof: {}", SimpleLogHandler.bytesToHex(eof));
		//
		fieldEofResponse0(header, fields, eof, conn);
		rowEofResponse0(eof, conn);
	}
	private void fieldEofResponse0(byte[] header, List fields, byte[] eof, BackendConnection conn) {
		byte packetId = 0;
		this.netOutBytes += header.length;
		for (int i = 0, len = fields.size(); i < len; ++i) {
			byte[] field = fields.get(i);
			this.netOutBytes += field.length;
		}
		header[3] = ++packetId;
		ServerConnection source = getSession().getSource();
		buffer = source.writeToBuffer(header, allocBuffer());
		for (int i = 0, len = fields.size(); i < len; ++i) {
			byte[] field = fields.get(i);
			field[3] = ++packetId;
			// 保存field信息
			FieldPacket fieldPk = new FieldPacket();
			fieldPk.read(field);
			fieldPackets.add(fieldPk);
			buffer = source.writeToBuffer(field, buffer);
		}
		fieldCount = fieldPackets.size();
		eof[3] = ++packetId;
		this.netOutBytes += eof.length;
		buffer = source.writeToBuffer(eof, buffer);
	}
	@Override
	public String toString() {
		return "CommandExecResultHandler [node=" + getRouteResultsetNode() + ", table=" + table + "]";
	}
}


================================================
FILE: src/main/java/io/mycat/server/handler/Explain2Handler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server.handler;

import java.nio.ByteBuffer;

import org.slf4j.Logger; import org.slf4j.LoggerFactory;

import io.mycat.backend.mysql.PacketUtil;
import io.mycat.backend.mysql.nio.handler.SingleNodeHandler;
import io.mycat.config.Fields;
import io.mycat.net.mysql.EOFPacket;
import io.mycat.net.mysql.FieldPacket;
import io.mycat.net.mysql.ResultSetHeaderPacket;
import io.mycat.net.mysql.RowDataPacket;
import io.mycat.route.RouteResultset;
import io.mycat.route.RouteResultsetNode;
import io.mycat.server.ServerConnection;
import io.mycat.server.parser.ServerParse;
import io.mycat.util.StringUtil;

/**
 * @author rainbow
 */
public class Explain2Handler {

	private static final Logger logger = LoggerFactory.getLogger(Explain2Handler.class);
    private static final String FORMAT_JSON = "FORMAT=JSON";
	private static final RouteResultsetNode[] EMPTY_ARRAY = new RouteResultsetNode[1];
	private static final int FIELD_COUNT = 2;
	private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT];
	static {
		fields[0] = PacketUtil.getField("SQL",
				Fields.FIELD_TYPE_VAR_STRING);
		fields[1] = PacketUtil.getField("MSG",
				Fields.FIELD_TYPE_VAR_STRING);
	}

	public static void handle(String stmt, ServerConnection c, int offset) {

		try {
			stmt = stmt.substring(offset);
			if(!stmt.toLowerCase().contains("datanode=") || !stmt.toLowerCase().contains("sql=")){
				showerror(stmt, c, "explain2 datanode=? sql=?");
				return ;
			}
            String dataNode = stmt.substring(stmt.indexOf("datanode=") + "datanode=".length(), stmt.indexOf("sql="))
                    .trim();
            boolean isJsonFormat = stmt.toUpperCase().contains(FORMAT_JSON);
            String formatStr = isJsonFormat ? FORMAT_JSON : "";
            String sql = "explain " + formatStr + " " + stmt.substring(stmt.indexOf("sql=") + 4, stmt.length()).trim();
			
			if(dataNode == null || dataNode.isEmpty() || sql == null || sql.isEmpty()){
				showerror(stmt, c, "dataNode or sql is null or empty");
				return;
			}
			
			RouteResultsetNode node = new RouteResultsetNode(dataNode, ServerParse.SELECT, sql);
			RouteResultset	rrs =  new RouteResultset(sql, ServerParse.SELECT);
			node.setSource(rrs);
			EMPTY_ARRAY[0] = node; 
			rrs.setNodes(EMPTY_ARRAY);
			SingleNodeHandler singleNodeHandler = new SingleNodeHandler(rrs, c.getSession2());
			singleNodeHandler.execute();
		} catch (Exception e) {
			logger.error(e.getMessage(), e.getCause());
			showerror(stmt, c, e.getMessage());
		}
	}
	
	private static void showerror(String stmt, ServerConnection c, String msg){
		ByteBuffer buffer = c.allocate();
		// write header
		ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT);
		byte packetId = header.packetId;
		buffer = header.write(buffer, c,true);

		// write fields
		for (FieldPacket field : fields) {
			field.packetId = ++packetId;
			buffer = field.write(buffer, c,true);
		}

		// write eof
		EOFPacket eof = new EOFPacket();
		eof.packetId = ++packetId;
		buffer = eof.write(buffer, c,true);

	
		RowDataPacket row = new RowDataPacket(FIELD_COUNT);
		row.add(StringUtil.encode(stmt, c.getCharset()));
		row.add(StringUtil.encode(msg, c.getCharset()));
		row.packetId = ++packetId;
		buffer = row.write(buffer, c,true);

		// write last eof
		EOFPacket lastEof = new EOFPacket();
		lastEof.packetId = ++packetId;
		buffer = lastEof.write(buffer, c,true);

		// post write
		c.write(buffer);
	}
}


================================================
FILE: src/main/java/io/mycat/server/handler/ExplainHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server.handler;

import java.nio.ByteBuffer;
import java.util.List;
import java.util.regex.Pattern;

import org.slf4j.Logger; import org.slf4j.LoggerFactory;

import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlInsertStatement;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.druid.sql.parser.SQLStatementParser;

import io.mycat.MycatServer;
import io.mycat.backend.mysql.PacketUtil;
import io.mycat.config.ErrorCode;
import io.mycat.config.Fields;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.SystemConfig;
import io.mycat.config.model.TableConfig;
import io.mycat.net.mysql.EOFPacket;
import io.mycat.net.mysql.FieldPacket;
import io.mycat.net.mysql.ResultSetHeaderPacket;
import io.mycat.net.mysql.RowDataPacket;
import io.mycat.route.RouteResultset;
import io.mycat.route.RouteResultsetNode;
import io.mycat.server.ServerConnection;
import io.mycat.server.parser.ServerParse;
import io.mycat.server.util.SchemaUtil;
import io.mycat.util.StringUtil;

/**
 * @author mycat
 */
public class ExplainHandler {

	private static final Logger logger = LoggerFactory.getLogger(ExplainHandler.class);
    private final static Pattern pattern = Pattern.compile("(?:(\\s*next\\s+value\\s+for\\s*MYCATSEQ_(\\w+))(,|\\)|\\s)*)+", Pattern.CASE_INSENSITIVE);
	private static final RouteResultsetNode[] EMPTY_ARRAY = new RouteResultsetNode[0];
	private static final int FIELD_COUNT = 2;
	private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT];
	static {
		fields[0] = PacketUtil.getField("DATA_NODE",
				Fields.FIELD_TYPE_VAR_STRING);
		fields[1] = PacketUtil.getField("SQL", Fields.FIELD_TYPE_VAR_STRING);
	}

	public static void handle(String stmt, ServerConnection c, int offset) {
		stmt = stmt.substring(offset).trim();

		RouteResultset rrs = getRouteResultset(c, stmt);
		if (rrs == null) {
			return;
		}

		ByteBuffer buffer = c.allocate();

		// write header
		ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT);
		byte packetId = header.packetId;
		buffer = header.write(buffer, c,true);

		// write fields
		for (FieldPacket field : fields) {
			field.packetId = ++packetId;
			buffer = field.write(buffer, c,true);
		}

		// write eof
		EOFPacket eof = new EOFPacket();
		eof.packetId = ++packetId;
		buffer = eof.write(buffer, c,true);

		// write rows
		RouteResultsetNode[] rrsn =  rrs.getNodes();
		for (RouteResultsetNode node : rrsn) {
			RowDataPacket row = getRow(node, c.getCharset());
			row.packetId = ++packetId;
			buffer = row.write(buffer, c,true);
		}

		// write last eof
		EOFPacket lastEof = new EOFPacket();
		lastEof.packetId = ++packetId;
		buffer = lastEof.write(buffer, c,true);

		// post write
		c.write(buffer);

	}

	private static RowDataPacket getRow(RouteResultsetNode node, String charset) {
		RowDataPacket row = new RowDataPacket(FIELD_COUNT);
		row.add(StringUtil.encode(node.getName(), charset));
		row.add(StringUtil.encode(node.getStatement().replaceAll("[\\t\\n\\r]", " "), charset));
		return row;
	}

	private static RouteResultset getRouteResultset(ServerConnection c,
			String stmt) {
		String db = c.getSchema();
        int sqlType = ServerParse.parse(stmt) & 0xff;
		if (db == null) {
            db = SchemaUtil.detectDefaultDb(stmt, sqlType);

            if(db==null)
            {
                c.writeErrMessage(ErrorCode.ER_NO_DB_ERROR, "No database selected");
                return null;
            }
		}
		SchemaConfig schema = MycatServer.getInstance().getConfig()
				.getSchemas().get(db);
		if (schema == null) {
			c.writeErrMessage(ErrorCode.ER_BAD_DB_ERROR, "Unknown database '"
					+ db + "'");
			return null;
		}
		try {

            if(ServerParse.INSERT==sqlType&&isMycatSeq(stmt, schema))
            {
                c.writeErrMessage(ErrorCode.ER_PARSE_ERROR, "insert sql using mycat seq,you must provide primaryKey value for explain");
                return null;
            }
            SystemConfig system = MycatServer.getInstance().getConfig().getSystem();
            return MycatServer.getInstance().getRouterservice()
					.route(system,schema, sqlType, stmt, c.getCharset(), c);
		} catch (Exception e) {
			StringBuilder s = new StringBuilder();
			logger.warn(s.append(c).append(stmt).toString()+" error:"+ e);
			String msg = e.getMessage();
			c.writeErrMessage(ErrorCode.ER_PARSE_ERROR, msg == null ? e
					.getClass().getSimpleName() : msg);
			return null;
		}
	}

    private static boolean isMycatSeq(String stmt, SchemaConfig schema)
    {
        if(pattern.matcher(stmt).find()) {
			return true;
		}
        SQLStatementParser parser =new MySqlStatementParser(stmt);
        MySqlInsertStatement statement = (MySqlInsertStatement) parser.parseStatement();
        String tableName=   statement.getTableName().getSimpleName();
        TableConfig tableConfig= schema.getTables().get(tableName.toUpperCase());
        if(tableConfig==null) {
			return false;
		}
        if(tableConfig.isAutoIncrement())
        {
            boolean isHasIdInSql=false;
            String primaryKey = tableConfig.getPrimaryKey();
            List columns = statement.getColumns();
            for (SQLExpr column : columns)
            {
                String columnName = column.toString();
                if(primaryKey.equalsIgnoreCase(columnName))
                {
                    isHasIdInSql = true;
                    break;
                }
            }
            if(!isHasIdInSql) {
				return true;
			}
        }


        return false;
    }

}


================================================
FILE: src/main/java/io/mycat/server/handler/KillHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server.handler;

import io.mycat.MycatServer;
import io.mycat.config.ErrorCode;
import io.mycat.net.FrontendConnection;
import io.mycat.net.NIOProcessor;
import io.mycat.net.mysql.OkPacket;
import io.mycat.server.ServerConnection;
import io.mycat.util.StringUtil;

/**
 * @author mycat
 */
public class KillHandler {

    public static void handle(String stmt, int offset, ServerConnection c) {
        String id = stmt.substring(offset).trim();
        if (StringUtil.isEmpty(id)) {
            c.writeErrMessage(ErrorCode.ER_NO_SUCH_THREAD, "NULL connection id");
        } else {
            // get value
            long value = 0;
            try {
                value = Long.parseLong(id);
            } catch (NumberFormatException e) {
                c.writeErrMessage(ErrorCode.ER_NO_SUCH_THREAD, "Invalid connection id:" + id);
                return;
            }

            // kill myself
            if (value == c.getId()) {
                getOkPacket().write(c);
                c.write(c.allocate());
                return;
            }

            // get connection and close it
            FrontendConnection fc = null;
            NIOProcessor[] processors = MycatServer.getInstance().getProcessors();
            for (NIOProcessor p : processors) {
                if ((fc = p.getFrontends().get(value)) != null) {
                    break;
                }
            }
            if (fc != null) {
                fc.close("killed");
                getOkPacket().write(c);
            } else {
                c.writeErrMessage(ErrorCode.ER_NO_SUCH_THREAD, "Unknown connection id:" + id);
            }
        }
    }

    private static OkPacket getOkPacket() {
        OkPacket packet = new OkPacket();
        packet.packetId = 1;
        packet.affectedRows = 0;
        packet.serverStatus = 2;
        return packet;
    }

}

================================================
FILE: src/main/java/io/mycat/server/handler/MigrateHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese
 * opensource volunteers. you can redistribute it and/or modify it under the
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Any questions about this component can be directed to it's project Web address
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server.handler;

import com.alibaba.fastjson.JSON;
import com.google.common.base.CharMatcher;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import io.mycat.MycatServer;
import io.mycat.backend.datasource.PhysicalDBNode;
import io.mycat.backend.mysql.PacketUtil;
import io.mycat.config.ErrorCode;
import io.mycat.config.Fields;
import io.mycat.config.loader.zkprocess.comm.ZkConfig;
import io.mycat.config.loader.zkprocess.comm.ZkParamCfg;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.SystemConfig;
import io.mycat.config.model.TableConfig;
import io.mycat.migrate.MigrateTask;
import io.mycat.migrate.MigrateTaskWatch;
import io.mycat.migrate.MigrateUtils;
import io.mycat.migrate.TaskNode;
import io.mycat.net.mysql.*;
import io.mycat.route.function.AbstractPartitionAlgorithm;
import io.mycat.route.function.PartitionByCRC32PreSlot;
import io.mycat.route.function.PartitionByCRC32PreSlot.Range;
import io.mycat.server.ServerConnection;
import io.mycat.util.ObjectUtil;
import io.mycat.util.StringUtil;
import io.mycat.util.ZKUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.api.transaction.CuratorTransactionFinal;
import org.apache.curator.framework.recipes.locks.InterProcessMutex;
import org.joda.time.LocalDateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.TimeUnit;

import static io.mycat.config.loader.zkprocess.comm.ZkParamCfg.ZK_CFG_FLAG;

/**
 * todo remove watch
 *
 * @author nange
 */
public final class MigrateHandler {
    private static final Logger LOGGER = LoggerFactory.getLogger("MigrateHandler");

    //可以优化成多个锁
    private static final InterProcessMutex slaveIDsLock = new InterProcessMutex(ZKUtils.getConnection(), ZKUtils.getZKBasePath() + "lock/slaveIDs.lock");
    private static final int FIELD_COUNT = 1;
    private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT];
    private static volatile boolean forceInit = false;


    static {
        fields[0] = PacketUtil.getField("TASK_ID",
                Fields.FIELD_TYPE_VAR_STRING);

    }

    private static String getUUID() {
        String s = UUID.randomUUID().toString();
        //去掉“-”符号
        return s.substring(0, 8) + s.substring(9, 13) + s.substring(14, 18) + s.substring(19, 23) + s.substring(24);
    }

    public static void handle(String stmt, ServerConnection c) {
        Map map = parse(stmt);

        String table = map.get("table");
        String add = map.get("add");
        String timeoutString = map.get("timeout");
        String charset = map.get("charset");
        boolean forceBinlog = false;//这个命令禁止使用因为binlog stream的实现依赖mysqldump
        int timeout = 120;// minute
        String schema = "";
        if (table == null) {
            writeErrMessage(c, "table cannot be null");
            return;
        }
        if (table.contains(".")) {
            String[] split = table.split("\\.");
            schema = split[0];
            table = split[1];
        }
        if (add == null) {
            writeErrMessage(c, "add cannot be null");
            return;
        }
        if (timeoutString != null) {
            try {
                timeout = Integer.parseInt(timeoutString);
                if (timeout <= 0) {
                    throw new NumberFormatException("");
                }
            } catch (Exception e) {
                writeErrMessage(c, String.format("timeout:%s format is wrong,it should be 1-" + Integer.MAX_VALUE + " (unit:minute)", timeoutString));
                return;
            }
        }
        if (StringUtil.isEmpty(charset)) {
            charset = Charset.defaultCharset().name();
        }
        if (!Charset.isSupported(charset)) {
            writeErrMessage(c, "Not support charset " + charset);
            return;
        }
        ZkConfig zkConfig = ZkConfig.getInstance();
        boolean loadZk = "true".equalsIgnoreCase(zkConfig.getValue(ZK_CFG_FLAG));
        boolean force = "true".equalsIgnoreCase(map.get("force"));
        CuratorFramework zk = ZKUtils.getConnection();
        if (!loadZk) {
            if (!force) {
                String msg = "";
                msg += "Mycat can temporarily execute the migration command.If other mycat does not connect to this zookeeper, they will not be able to perceive changes in the migration task.\n";
                msg += "You can command as follow:\n\nmigrate -table=schema.test -add=dn2,dn3 -force=true\n\nto perform the migration.\n";
                LOGGER.error(msg);
                writeErrMessage(c, msg);
                return;
            }
            //因为loadZk在mycat启动时候没有监听zk里migrate路径，这里需要把这个监听补上
            //借用slaveIDsLock对象作为同步锁
            boolean changed = false;
            if (!forceInit) {
                synchronized (slaveIDsLock) {
                    if (!forceInit) {
                        forceInit = true;
                        changed = true;
                    }
                }
            }
            if (changed) {
                MigrateTaskWatch.start();
            }
        }
        if (zk == null) {
            writeErrMessage(c, "Mycat is not connected to zookeeper");
            return;
        }
        String taskID = getUUID();
        try {
            if (StringUtil.isEmpty(schema)) {
                schema = c.getSchema();
            }
            if (StringUtil.isEmpty(schema)) {
                writeErrMessage(c, "No database selected");
                return;
            }
            SchemaConfig schemaConfig = MycatServer.getInstance().getConfig().getSchemas().get(schema);
            if (schemaConfig == null) {
                writeErrMessage(c, String.format("Unknown database '" + schema + "'", table.toUpperCase(), schema));
                return;
            }
            TableConfig tableConfig = schemaConfig.getTables().get(table.toUpperCase());
            if (tableConfig == null) {
                writeErrMessage(c, String.format("Table '%s' doesn't define in schema '%s'\n", table.toUpperCase(), schema));
                return;
            }
            AbstractPartitionAlgorithm algorithm = tableConfig.getRule().getRuleAlgorithm();
            if (!(algorithm instanceof PartitionByCRC32PreSlot)) {
                writeErrMessage(c, "table: " + table + " rule is not be PartitionByCRC32PreSlot");
                return;
            }
            Map> integerListMap = ((PartitionByCRC32PreSlot) algorithm).getRangeMap();
            integerListMap = (Map>) ObjectUtil.copyObject(integerListMap);

            ArrayList oldDataNodes = tableConfig.getDataNodes();
            Map allDataNodes = MycatServer.getInstance().getConfig().getDataNodes();
            List newDataNodes = Splitter.on(",").omitEmptyStrings().trimResults().splitToList(add);
            for (String newDataNode : newDataNodes) {
                if (tableConfig.getDataNodes().contains(newDataNode)) {
                    writeErrMessage(c, "The dataNode " + newDataNode+" that needs to be added already exists\n");
                    return;
                }
                if(!allDataNodes.containsKey(newDataNode)){
                    writeErrMessage(c, "The dataNode " + newDataNode+" does not exist\n");
                    return;
                }
            }

            Map> tasks = MigrateUtils
                    .balanceExpand(table, integerListMap, oldDataNodes, newDataNodes, PartitionByCRC32PreSlot.DEFAULT_SLOTS_NUM);

            CuratorTransactionFinal transactionFinal = null;
            String taskBase = ZKUtils.getZKBasePath() + "migrate/" + schema;
            String taskPath = taskBase + "/" + taskID;
            CuratorFramework client = ZKUtils.getConnection();

            //校验 之前同一个表的迁移任务未完成，则jzhi禁止继续
            if (client.checkExists().forPath(taskBase) != null) {
                List childTaskList = client.getChildren().forPath(taskBase);
                for (String child : childTaskList) {
                    String path = taskBase + "/" + child;
                    String str = new String(ZKUtils.getConnection().getData().forPath(path));
                    if (!isJson(str)) {
                        writeErrMessage(c, path + "in zookeeper is abnormal state,please repair manual!");
                        return;
                    }
                    TaskNode taskNode = JSON
                            .parseObject(str, TaskNode.class);
                    if (taskNode.getSchema().equalsIgnoreCase(schema) && table.equalsIgnoreCase(taskNode.getTable())
                            && taskNode.getStatus() < 5) {
                        writeErrMessage(c, "table: " + table + " previous migrate task is still running,on the same time one table only one task");
                        return;
                    }
                }
            }
            String backupPath = backup();
            client.create().creatingParentsIfNeeded().forPath(taskPath);
            TaskNode taskNode = new TaskNode();
            taskNode.setSchema(schema);
            taskNode.setSql(stmt);
            taskNode.setTable(table);
            taskNode.setAdd(add);
            taskNode.setStatus(0);
            taskNode.setTimeout(timeout);
            taskNode.setCharset(charset);
            taskNode.setForceBinlog(forceBinlog);
            taskNode.setBackupFile(backupPath);

            Map fromNodeSlaveIdMap = new HashMap<>();

            List allTaskList = new ArrayList<>();
            for (Map.Entry> entry : tasks.entrySet()) {
                String key = entry.getKey();
                List value = entry.getValue();
                for (MigrateTask migrateTask : value) {
                    migrateTask.setSchema(schema);

                    //分配slaveid只需要一个dataHost分配一个即可，后续任务执行模拟从节点只需要一个dataHost一个
                    String dataHost = getDataHostNameFromNode(migrateTask.getFrom());
                    if (fromNodeSlaveIdMap.containsKey(dataHost)) {
                        migrateTask.setSlaveId(fromNodeSlaveIdMap.get(dataHost));
                    } else {
                        migrateTask.setSlaveId(getSlaveIdFromZKForDataNode(migrateTask.getFrom()));
                        fromNodeSlaveIdMap.put(dataHost, migrateTask.getSlaveId());
                    }

                }
                allTaskList.addAll(value);

            }


            transactionFinal = client.inTransaction().setData().forPath(taskPath, JSON.toJSONBytes(taskNode)).and();


            //合并成dataHost级别任务
            Map> dataHostMigrateMap = mergerTaskForDataHost(allTaskList);

            String boosterDataHosts = ZkConfig.getInstance().getValue(ZkParamCfg.MYCAT_BOOSTER_DATAHOSTS);
            Set dataNodes = new HashSet<>(Splitter.on(",").trimResults().omitEmptyStrings().splitToList(boosterDataHosts));
            boolean isFirst = true;
            for (String s : dataHostMigrateMap.keySet()) {
                if (!dataNodes.contains(s)) {
                    if (isFirst) {
                        LOGGER.warn("--------------------------------check dataNode--------------------------------");
                        isFirst = false;
                    }
                    LOGGER.warn("dataNode %s will be not participate in migration");
                }
            }

            for (Map.Entry> entry : dataHostMigrateMap.entrySet()) {
                String key = entry.getKey();
                List value = entry.getValue();
                String path = taskPath + "/" + key;
                transactionFinal = transactionFinal.create().forPath(path, JSON.toJSONBytes(value)).and();
            }
            transactionFinal.commit();
        } catch (Exception e) {
            LOGGER.error("migrate error", e);
            writeErrMessage(c, "migrate error:" + e);
            return;
        }

        writePackToClient(c, taskID);
        LOGGER.info("--------------------------------task created success--------------------------------");
        LOGGER.info("task start", new Date());
    }

    private static void writePackToClient(ServerConnection c, String taskID) {
        ByteBuffer buffer = c.allocate();

        // write header
        ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT);
        byte packetId = header.packetId;
        buffer = header.write(buffer, c, true);

        // write fields
        for (FieldPacket field : fields) {
            field.packetId = ++packetId;
            buffer = field.write(buffer, c, true);
        }

        // write eof
        EOFPacket eof = new EOFPacket();
        eof.packetId = ++packetId;
        buffer = eof.write(buffer, c, true);

        RowDataPacket row = new RowDataPacket(FIELD_COUNT);
        row.add(StringUtil.encode(taskID, c.getCharset()));
        row.packetId = ++packetId;
        buffer = row.write(buffer, c, true);

        // write last eof
        EOFPacket lastEof = new EOFPacket();
        lastEof.packetId = ++packetId;
        buffer = lastEof.write(buffer, c, true);

        // post write
        c.write(buffer);
    }


    private static String getDataHostNameFromNode(String dataNode) {
        return MycatServer.getInstance().getConfig().getDataNodes().get(dataNode).getDbPool().getHostName();
    }

    private static Map> mergerTaskForDataHost(List migrateTaskList) {
        Map> taskMap = new HashMap<>();
        for (MigrateTask migrateTask : migrateTaskList) {
            String dataHost = getDataHostNameFromNode(migrateTask.getFrom());
            if (taskMap.containsKey(dataHost)) {
                taskMap.get(dataHost).add(migrateTask);
            } else {
                taskMap.put(dataHost, Lists.newArrayList(migrateTask));
            }
        }


        return taskMap;
    }

    private static int getSlaveIdFromZKForDataNode(String dataNode) {
        PhysicalDBNode dbNode = MycatServer.getInstance().getConfig().getDataNodes().get(dataNode);
        String slaveIDs = dbNode.getDbPool().getSlaveIDs();
        if (Strings.isNullOrEmpty(slaveIDs))
            throw new RuntimeException("dataHost:" + dbNode.getDbPool().getHostName() + " do not config the salveIDs field");

        List allSlaveIDList = parseSlaveIDs(slaveIDs);

        String taskPath = ZKUtils.getZKBasePath() + "slaveIDs/" + dbNode.getDbPool().getHostName();
        try {
            slaveIDsLock.acquire(30, TimeUnit.SECONDS);
            Set zkSlaveIdsSet = new HashSet<>();
            if (ZKUtils.getConnection().checkExists().forPath(taskPath) != null) {
                List zkHasSlaveIDs = ZKUtils.getConnection().getChildren().forPath(taskPath);
                for (String zkHasSlaveID : zkHasSlaveIDs) {
                    zkSlaveIdsSet.add(Integer.parseInt(zkHasSlaveID));
                }
            }
            for (Integer integer : allSlaveIDList) {
                if (!zkSlaveIdsSet.contains(integer)) {
                    ZKUtils.getConnection().create().creatingParentsIfNeeded().forPath(taskPath + "/" + integer);
                    return integer;
                }
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        } finally {
            try {
                slaveIDsLock.release();
            } catch (Exception e) {
                LOGGER.error("error:", e);
            }
        }

        throw new RuntimeException("cannot get the slaveID  for dataHost :" + dbNode.getDbPool().getHostName());
    }

    private static List parseSlaveIDs(String slaveIDs) {
        List allSlaveList = new ArrayList<>();
        List stringList = Splitter.on(",").omitEmptyStrings().trimResults().splitToList(slaveIDs);
        for (String id : stringList) {
            if (id.contains("-")) {
                List idRangeList = Splitter.on("-").omitEmptyStrings().trimResults().splitToList(id);
                if (idRangeList.size() != 2)
                    throw new RuntimeException(id + "slaveIds range must be 2  size");
                for (int i = Integer.parseInt(idRangeList.get(0)); i <= Integer.parseInt(idRangeList.get(1)); i++) {
                    allSlaveList.add(i);
                }

            } else {
                allSlaveList.add(Integer.parseInt(id));
            }
        }
        return allSlaveList;
    }


    private static OkPacket getOkPacket() {
        OkPacket packet = new OkPacket();
        packet.packetId = 1;
        packet.affectedRows = 0;
        packet.serverStatus = 2;
        return packet;
    }

    public static void writeErrMessage(ServerConnection c, String msg) {
        c.writeErrMessage(ErrorCode.ER_UNKNOWN_ERROR, msg);
    }

    public static void main(String[] args) {
        String sql = "migrate    -table=test  -add=dn2,dn3,dn4  " + " \n -additional=\"a=b\"";
        Map map = parse(sql);
        System.out.println();
        for (int i = 0; i < 100; i++) {
            System.out.println(i % 5);
        }

        TaskNode taskNode = new TaskNode();
        taskNode.setSql(sql);


        System.out.println(new String(JSON.toJSONBytes(taskNode)));
    }

    private static Map parse(String sql) {
        Map map = new HashMap<>();
        List rtn = Splitter.on(CharMatcher.whitespace()).omitEmptyStrings().splitToList(sql);
        for (String s : rtn) {
            if (s.contains("=")) {
                int dindex = s.indexOf("=");
                if (s.startsWith("-")) {
                    String key = s.substring(1, dindex).toLowerCase().trim();
                    String value = s.substring(dindex + 1).trim();
                    map.put(key, value);
                } else if (s.startsWith("--")) {
                    String key = s.substring(2, dindex).toLowerCase().trim();
                    String value = s.substring(dindex + 1).trim();
                    map.put(key, value);
                }
            }
        }
        return map;
    }

    public static String backup() throws Exception {
        LocalDateTime now = LocalDateTime.now();
        Path path = Paths.get(SystemConfig.getHomePath()).resolve("backup_" + now.getYear() + "_" + now.getMonthOfYear() + "_" + now.getDayOfMonth() + "_" + now.getHourOfDay() + "_" + now.getMinuteOfHour());
        if (!Files.exists(path)) {
            Files.createDirectory(path);
        }
        List strings = ZKUtils.getConnection().getChildren().forPath(ZKUtils.getZKBasePath() + "ruledata");
        for (String s : strings) {
            byte[] bytes = ZKUtils.getConnection().getData().forPath(ZKUtils.getZKBasePath() + "ruledata/" + s);
            Files.write(path.resolve(s), bytes);
        }
        byte[] bytes = ZKUtils.getConnection().getData().forPath(ZKUtils.getZKBasePath() + "schema/schema");
        Files.write(path.resolve("schema.json"), bytes);
        bytes = ZKUtils.getConnection().getData().forPath(ZKUtils.getZKBasePath() + "rules/function");
        Files.write(path.resolve("function.json"), bytes);
        return path.toAbsolutePath().toString();
    }

    private static boolean isJson(String str) {
        if (StringUtil.isEmpty(str)) return false;
        str = str.trim();
        if (str.startsWith("{") && str.endsWith("}")) return true;
        return false;
    }
}


================================================
FILE: src/main/java/io/mycat/server/handler/MysqlInformationSchemaHandler.java
================================================
package io.mycat.server.handler;

import java.nio.ByteBuffer;

import io.mycat.backend.mysql.PacketUtil;
import io.mycat.config.Fields;
import io.mycat.net.mysql.EOFPacket;
import io.mycat.net.mysql.FieldPacket;
import io.mycat.net.mysql.OkPacket;
import io.mycat.net.mysql.ResultSetHeaderPacket;
import io.mycat.server.ServerConnection;
import io.mycat.server.util.SchemaUtil;


/**
 * 对 PhpAdmin's 控制台操作进行支持 
 * 
 * 如：SELECT * FROM information_schema.CHARACTER_SETS 等相关语句进行模拟返回
 * 
 * @author zhuam
 *
 */
public class MysqlInformationSchemaHandler {
	
	/**
	 * 写入数据包
	 * @param field_count
	 * @param fields
	 * @param c
	 */
	private static void doWrite(int field_count, FieldPacket[] fields, ServerConnection c) {
		
		ByteBuffer buffer = c.allocate();

		// write header
		ResultSetHeaderPacket header = PacketUtil.getHeader(field_count);
		byte packetId = header.packetId;
		buffer = header.write(buffer, c, true);

		// write fields
		for (FieldPacket field : fields) {
			field.packetId = ++packetId;
			buffer = field.write(buffer, c, true);
		}

		// write eof
		EOFPacket eof = new EOFPacket();
		eof.packetId = ++packetId;
		buffer = eof.write(buffer, c, true);

		// write last eof
		EOFPacket lastEof = new EOFPacket();
		lastEof.packetId = ++packetId;
		buffer = lastEof.write(buffer, c, true);

		// post write
		c.write(buffer);
		
	}
	
	public static void handle(String sql, ServerConnection c) {
		
		SchemaUtil.SchemaInfo schemaInfo = SchemaUtil.parseSchema(sql);
		if ( schemaInfo != null ) {
			
			if ( schemaInfo.table.toUpperCase().equals("CHARACTER_SETS") ) {
				
				//模拟列头
				int field_count = 4;
			    FieldPacket[] fields = new FieldPacket[field_count];
			    fields[0] = PacketUtil.getField("CHARACTER_SET_NAME", Fields.FIELD_TYPE_VAR_STRING);
				fields[1] = PacketUtil.getField("DEFAULT_COLLATE_NAME", Fields.FIELD_TYPE_VAR_STRING);
			    fields[2] = PacketUtil.getField("DESCRIPTION", Fields.FIELD_TYPE_VAR_STRING);
				fields[3] = PacketUtil.getField("MAXLEN", Fields.FIELD_TYPE_LONG);
				
				doWrite(field_count, fields, c);				
				
			} else {
				c.write(c.writeToBuffer(OkPacket.OK, c.allocate()));
			}			
			
		} else {
			c.write(c.writeToBuffer(OkPacket.OK, c.allocate()));
		}		
	}
}

================================================
FILE: src/main/java/io/mycat/server/handler/MysqlProcHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server.handler;


import java.nio.ByteBuffer;

import io.mycat.backend.mysql.PacketUtil;
import io.mycat.config.Fields;
import io.mycat.net.mysql.EOFPacket;
import io.mycat.net.mysql.FieldPacket;
import io.mycat.net.mysql.ResultSetHeaderPacket;
import io.mycat.server.ServerConnection;

public class MysqlProcHandler
{
    private static final int FIELD_COUNT = 2;
    private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT];

    static
    {
        fields[0] = PacketUtil.getField("name",
                Fields.FIELD_TYPE_VAR_STRING);
        fields[1] = PacketUtil.getField("type", Fields.FIELD_TYPE_VAR_STRING);
    }

    public static void handle(String stmt, ServerConnection c)
    {

        ByteBuffer buffer = c.allocate();

        // write header
        ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT);
        byte packetId = header.packetId;
        buffer = header.write(buffer, c, true);

        // write fields
        for (FieldPacket field : fields)
        {
            field.packetId = ++packetId;
            buffer = field.write(buffer, c, true);
        }

        // write eof
        EOFPacket eof = new EOFPacket();
        eof.packetId = ++packetId;
        buffer = eof.write(buffer, c, true);


        // write last eof
        EOFPacket lastEof = new EOFPacket();
        lastEof.packetId = ++packetId;
        buffer = lastEof.write(buffer, c, true);

        // post write
        c.write(buffer);

    }


}

================================================
FILE: src/main/java/io/mycat/server/handler/SavepointHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server.handler;

import io.mycat.config.ErrorCode;
import io.mycat.server.ServerConnection;

/**
 * @author mycat
 */
public final class SavepointHandler {

    public static void handle(String stmt, ServerConnection c) {
        c.writeErrMessage(ErrorCode.ER_UNKNOWN_COM_ERROR, "Unsupported statement");
    }

}

================================================
FILE: src/main/java/io/mycat/server/handler/SelectHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server.handler;

import java.util.HashSet;
import java.util.Set;

import io.mycat.route.parser.util.ParseUtil;
import io.mycat.server.ServerConnection;
import io.mycat.server.parser.ServerParse;
import io.mycat.server.parser.ServerParseSelect;
import io.mycat.server.response.ClientHeartbeatResponse;
import io.mycat.server.response.SelectDatabase;
import io.mycat.server.response.SelectIdentity;
import io.mycat.server.response.SelectLastInsertId;
import io.mycat.server.response.SelectTxReadOnly;
import io.mycat.server.response.SelectUser;
import io.mycat.server.response.SelectVersion;
import io.mycat.server.response.SelectVersionComment;
import io.mycat.server.response.SessionIncrement;
import io.mycat.server.response.SessionIsolation;

/**
 * @author mycat
 */
public final class SelectHandler {
    private static final int HEART_BEAT_SQL_MAX_LENGTH = "SELECT 1 FROM DUAL".length();
    private static Set CLIENT_HEART_BEAT_SQLS = new HashSet(2);
    static {
        CLIENT_HEART_BEAT_SQLS.add("SELECT 1");
        CLIENT_HEART_BEAT_SQLS.add("SELECT 1 FROM DUAL");
    }


	public static void handle(String stmt, ServerConnection c, int offs) {
		int offset = offs;
		c.setExecuteSql(null);
		switch (ServerParseSelect.parse(stmt, offs)) {
		case ServerParseSelect.VERSION_COMMENT:
			SelectVersionComment.response(c);
			break;
		case ServerParseSelect.DATABASE:
			SelectDatabase.response(c);
			break;
		case ServerParseSelect.USER:
			SelectUser.response(c);
			break;
		case ServerParseSelect.VERSION:
			SelectVersion.response(c);
			break;
		case ServerParseSelect.SESSION_INCREMENT:
			SessionIncrement.response(c);
			break;
		case ServerParseSelect.SESSION_ISOLATION:
			SessionIsolation.response(c);
			break;
		case ServerParseSelect.LAST_INSERT_ID:
			// offset = ParseUtil.move(stmt, 0, "select".length());
			loop:for (int l=stmt.length(); offset < l; ++offset) {
				switch (stmt.charAt(offset)) {
				case ' ':
					continue;
				case '/':
				case '#':
					offset = ParseUtil.comment(stmt, offset);
					continue;
				case 'L':
				case 'l':
					break loop;
				}
			}
			offset = ServerParseSelect.indexAfterLastInsertIdFunc(stmt, offset);
			offset = ServerParseSelect.skipAs(stmt, offset);
			SelectLastInsertId.response(c, stmt, offset);
			break;
		case ServerParseSelect.IDENTITY:
			// offset = ParseUtil.move(stmt, 0, "select".length());
			loop:for (int l=stmt.length(); offset < l; ++offset) {
				switch (stmt.charAt(offset)) {
				case ' ':
					continue;
				case '/':
				case '#':
					offset = ParseUtil.comment(stmt, offset);
					continue;
				case '@':
					break loop;
				}
			}
			int indexOfAtAt = offset;
			offset += 2;
			offset = ServerParseSelect.indexAfterIdentity(stmt, offset);
			String orgName = stmt.substring(indexOfAtAt, offset);
			offset = ServerParseSelect.skipAs(stmt, offset);
			SelectIdentity.response(c, stmt, offset, orgName);
			break;
            case ServerParseSelect.SELECT_VAR_ALL:
				c.execute(stmt, ServerParse.SELECT);
				break;
			case ServerParseSelect.SESSION_TX_READ_ONLY:
				SelectTxReadOnly.response(c);
				break;
		default:
			c.setExecuteSql(stmt);
            if (isClientHeartbeatSql(stmt)) {
                ClientHeartbeatResponse.response(c);
            } else {
                c.execute(stmt, ServerParse.SELECT);
            }
		}
	}

    private static boolean isClientHeartbeatSql(String sql) {
        if (sql.length() > HEART_BEAT_SQL_MAX_LENGTH) {
            return false;
        } else {
            return CLIENT_HEART_BEAT_SQLS.contains(sql.toUpperCase());
        }
    }
}


================================================
FILE: src/main/java/io/mycat/server/handler/ServerLoadDataInfileHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese
 * opensource volunteers. you can redistribute it and/or modify it under the
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Any questions about this component can be directed to it's project Web address
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server.handler;

import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.expr.*;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlLoadDataInFileStatement;
import com.alibaba.druid.sql.parser.SQLStatementParser;
import com.google.common.collect.Lists;
import com.google.common.io.Files;
import com.univocity.parsers.csv.CsvParser;
import com.univocity.parsers.csv.CsvParserSettings;
import io.mycat.MycatServer;
import io.mycat.cache.LayerCachePool;
import io.mycat.config.ErrorCode;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.SystemConfig;
import io.mycat.config.model.TableConfig;
import io.mycat.net.handler.LoadDataInfileHandler;
import io.mycat.net.mysql.BinaryPacket;
import io.mycat.net.mysql.RequestFilePacket;
import io.mycat.route.RouteResultset;
import io.mycat.route.RouteResultsetNode;
import io.mycat.route.function.SlotFunction;
import io.mycat.route.parser.druid.DruidShardingParseInfo;
import io.mycat.route.parser.druid.MycatStatementParser;
import io.mycat.route.parser.druid.RouteCalculateUnit;
import io.mycat.route.util.RouterUtil;
import io.mycat.server.ServerConnection;
import io.mycat.server.parser.ServerParse;
import io.mycat.sqlengine.mpp.LoadData;
import io.mycat.util.ObjectUtil;
import io.mycat.util.StringUtil;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.sql.SQLNonTransientException;
import java.util.*;

/**
 * mysql命令行客户端也需要启用local file权限，加参数--local-infile=1
 * jdbc则正常，不用设置
 * load data sql中的CHARACTER SET 'gbk'   其中的字符集必须引号括起来，否则druid解析出错
 */
public final class ServerLoadDataInfileHandler implements LoadDataInfileHandler {
    private static final Logger LOGGER = LoggerFactory.getLogger(ServerLoadDataInfileHandler.class);
    private ServerConnection serverConnection;
    private String sql;
    private String fileName;
    private List varColumns;
    private byte packID = 0;
    private MySqlLoadDataInFileStatement statement;

    private Map routeResultMap = new HashMap<>();

    private LoadData loadData;
    private ByteArrayOutputStream tempByteBuffer;
    private long tempByteBuffrSize = 0;
    private String tempFile;
    private boolean isHasStoreToFile = false;
    private String tempPath;
    private String tableName;
    private TableConfig tableConfig;
    private int partitionColumnIndex = -1;
    private LayerCachePool tableId2DataNodeCache;
    private SchemaConfig schema;
    private boolean isStartLoadData = false;

    private boolean shoudAddSlot = false;

    public int getPackID() {
        return packID;
    }

    public void setPackID(byte packID) {
        this.packID = packID;
    }

    public ServerLoadDataInfileHandler(ServerConnection serverConnection) {
        this.serverConnection = serverConnection;

    }

    private static String parseFileName(String sql) {
        if (sql.contains("'")) {
            int beginIndex = sql.indexOf("'");
            return sql.substring(beginIndex + 1, sql.indexOf("'", beginIndex + 1));
        } else if (sql.contains("\"")) {
            int beginIndex = sql.indexOf("\"");
            return sql.substring(beginIndex + 1, sql.indexOf("\"", beginIndex + 1));
        }
        return null;
    }


    private void parseLoadDataPram() {
        loadData = new LoadData();
        SQLTextLiteralExpr rawLineEnd = (SQLTextLiteralExpr) statement.getLinesTerminatedBy();
        String lineTerminatedBy = rawLineEnd == null ? "\n" : rawLineEnd.getText();
        loadData.setLineTerminatedBy(lineTerminatedBy);

        SQLTextLiteralExpr rawFieldEnd = (SQLTextLiteralExpr) statement.getColumnsTerminatedBy();
        String fieldTerminatedBy = rawFieldEnd == null ? "\t" : rawFieldEnd.getText();
        loadData.setFieldTerminatedBy(fieldTerminatedBy);

        SQLTextLiteralExpr rawEnclosed = (SQLTextLiteralExpr) statement.getColumnsEnclosedBy();
        String enclose = rawEnclosed == null ? null : rawEnclosed.getText();
        loadData.setEnclose(enclose);

        SQLTextLiteralExpr escapseExpr = (SQLTextLiteralExpr) statement.getColumnsEscaped();
        String escapse = escapseExpr == null ? "\\" : escapseExpr.getText();
        loadData.setEscape(escapse);
        String charset = statement.getCharset() != null ? statement.getCharset() : serverConnection.getCharset();
        loadData.setCharset(charset);
        loadData.setFileName(fileName);
    }


    @Override
    public void start(String sql) {
        clear();
        this.sql = sql;


        SQLStatementParser parser = new MycatStatementParser(sql);
        statement = (MySqlLoadDataInFileStatement) parser.parseStatement();
        if (statement.getSetList().size() > 0) {
            varColumns = parseParam2RealColumn(statement);
        }
        fileName = parseFileName(sql);

        if (fileName == null) {
            serverConnection.writeErrMessage(ErrorCode.ER_FILE_NOT_FOUND, " file name is null !");
            clear();
            return;
        }
        schema = MycatServer.getInstance().getConfig()
                .getSchemas().get(serverConnection.getSchema());
        if (schema == null) {
            throw new RuntimeException("please sql:use schema  before load data");
        }
        tableId2DataNodeCache = (LayerCachePool) MycatServer.getInstance().getCacheService().getCachePool("TableID2DataNodeCache");
        tableName = statement.getTableName().getSimpleName().toUpperCase();
        tableConfig = schema.getTables().get(tableName);
        if (tableConfig.getRule() != null && tableConfig.getRule().getRuleAlgorithm() instanceof SlotFunction) {
            shoudAddSlot = true;
        }
        tempPath = SystemConfig.getHomePath() + File.separator + "temp" + File.separator + serverConnection.getId() + File.separator;
        tempFile = tempPath + "clientTemp.txt";
        tempByteBuffer = new ByteArrayOutputStream();

        List columns = statement.getColumns();
        if (tableConfig != null) {
            String pColumn = getPartitionColumn();
            //如果有变量set表达式，columns里面会包含变量的名称，改用varColumns以获取真实的列名称
            List columnsTmp = varColumns == null ? columns : varColumns;
            if (pColumn != null && columnsTmp != null && columns.size() > 0) {
                for (int i = 0, columnsSize = columnsTmp.size(); i < columnsSize; i++) {
                    String column = StringUtil.removeBackquote(columnsTmp.get(i).toString());
                    if (pColumn.equalsIgnoreCase(column)) {
                        partitionColumnIndex = i;
                    }
                    if ("_slot".equalsIgnoreCase(column)) {
                        shoudAddSlot = false;
                    }
                }

            }
        }
        if (shoudAddSlot) {
            columns.add(new SQLIdentifierExpr("_slot"));
        }
        parseLoadDataPram();
        if (statement.isLocal()) {
            isStartLoadData = true;
            //向客户端请求发送文件
            ByteBuffer buffer = serverConnection.allocate();
            RequestFilePacket filePacket = new RequestFilePacket();
            filePacket.fileName = fileName.getBytes();
            filePacket.packetId = 1;
            filePacket.write(buffer, serverConnection, true);
        } else {
            if (!new File(fileName).exists()) {
                serverConnection.writeErrMessage(ErrorCode.ER_FILE_NOT_FOUND, fileName + " is not found!");
                clear();
            } else {
                parseFileByLine(fileName, loadData.getCharset(), loadData.getLineTerminatedBy());
                RouteResultset rrs = buildResultSet(routeResultMap);
                if (rrs != null) {
                    flushDataToFile();
                    isStartLoadData = false;
                    serverConnection.getSession2().execute(rrs, ServerParse.LOAD_DATA_INFILE_SQL);
                }

            }
        }
    }

    /**
     *
     * load data infile使用set子句时将用户变量改成真实的列名
     * 比如下面语句LOAD DATA INFILE 'file.txt'
     INTO TABLE t1
     (column1, @var1,@var2)
     SET column2 = @var1/100,columns=100/(@var2/200));
     最后真实的列应该是{column1,column2,columns}
     语法参考： 参考https://dev.mysql.com/doc/refman/5.7/en/load-data.html
     * @param statement
     * @return
     */
    private List parseParam2RealColumn(MySqlLoadDataInFileStatement statement) {
        String HAS_FILTERED = "HAS_FILTERED";
        List realColumnList = new ArrayList<>(statement.getColumns().size());

        for (SQLExpr columnExpr : statement.getColumns()) {
            boolean hasSetColumn = false;
            if (columnExpr instanceof SQLVariantRefExpr) {
                SQLVariantRefExpr varColumn = (SQLVariantRefExpr) columnExpr;
                for (SQLExpr setExpr : statement.getSetList()) {
                    if (setExpr.getAttribute(HAS_FILTERED) != null) {
                        // 已经被过滤过
                        continue;
                    }

                    // 找到包含变量的set表达式
                    if (setExpr.toString().contains(varColumn.toString())) {
                        if (setExpr instanceof SQLBinaryOpExpr) {
                            realColumnList.add(((SQLBinaryOpExpr) setExpr).getLeft());
                            hasSetColumn = true;
                        }
                        setExpr.putAttribute(HAS_FILTERED, true);// 标记已经被过滤过
                        break;
                    }
                }
            }
            if (!hasSetColumn) {
                realColumnList.add(columnExpr);
            }
        }

        return realColumnList;
    }

    @Override
    public void handle(byte[] data) {

        try {
            if (sql == null) {
                serverConnection.writeErrMessage(ErrorCode.ER_UNKNOWN_COM_ERROR,
                        "Unknown command");
                clear();
                return;
            }
            BinaryPacket packet = new BinaryPacket();
            ByteArrayInputStream inputStream = new ByteArrayInputStream(data, 0, data.length);
            packet.read(inputStream);

            saveByteOrToFile(packet.data, false);


        } catch (IOException e) {
            throw new RuntimeException(e);
        }


    }

    private synchronized void saveByteOrToFile(byte[] data, boolean isForce) {

        if (data != null) {
            tempByteBuffrSize = tempByteBuffrSize + data.length;
            try {
                tempByteBuffer.write(data);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }

        if ((isForce && isHasStoreToFile) || tempByteBuffrSize > 200 * 1024 * 1024)    //超过200M 存文件
        {
            FileOutputStream channel = null;
            try {
                File file = new File(tempFile);
                Files.createParentDirs(file);
                channel = new FileOutputStream(file, true);

                tempByteBuffer.writeTo(channel);
                tempByteBuffer = new ByteArrayOutputStream();
                tempByteBuffrSize = 0;
                isHasStoreToFile = true;
            } catch (IOException e) {
                throw new RuntimeException(e);
            } finally {
                try {
                    if (channel != null) {
                        channel.close();
                    }

                } catch (IOException ignored) {

                }
            }


        }
    }


    private RouteResultset tryDirectRoute(String sql, String[] lineList) {

        RouteResultset rrs = new RouteResultset(sql, ServerParse.INSERT);
        rrs.setLoadData(true);
        if (tableConfig == null && schema.getDataNode() != null) {
            //走默认节点
            RouteResultsetNode rrNode = new RouteResultsetNode(schema.getDataNode(), ServerParse.INSERT, sql);
            rrNode.setSource(rrs);
            rrs.setNodes(new RouteResultsetNode[]{rrNode});
            return rrs;
        } else if (tableConfig != null && tableConfig.isGlobalTable()) {
            ArrayList dataNodes = tableConfig.getDataNodes();
            RouteResultsetNode[] rrsNodes = new RouteResultsetNode[dataNodes.size()];
            for (int i = 0, dataNodesSize = dataNodes.size(); i < dataNodesSize; i++) {
                String dataNode = dataNodes.get(i);
                RouteResultsetNode rrNode = new RouteResultsetNode(dataNode, ServerParse.INSERT, sql);
                rrsNodes[i] = rrNode;
                if (rrs.getDataNodeSlotMap().containsKey(dataNode)) {
                    rrsNodes[i].setSlot(rrs.getDataNodeSlotMap().get(dataNode));
                }
                rrsNodes[i].setSource(rrs);
            }

            rrs.setNodes(rrsNodes);
            return rrs;
        } else if (tableConfig != null) {
            DruidShardingParseInfo ctx = new DruidShardingParseInfo();
            ctx.addTable(tableName);


            if (partitionColumnIndex == -1 || partitionColumnIndex >= lineList.length) {
                return null;
            } else {
                String value = lineList[partitionColumnIndex];
                RouteCalculateUnit routeCalculateUnit = new RouteCalculateUnit();
                routeCalculateUnit.addShardingExpr(tableName, getPartitionColumn(), parseFieldString(value, loadData.getEnclose()));
                ctx.addRouteCalculateUnit(routeCalculateUnit);
                try {
                    SortedSet nodeSet = new TreeSet();
                    for (RouteCalculateUnit unit : ctx.getRouteCalculateUnits()) {
                        RouteResultset rrsTmp = RouterUtil.tryRouteForTables(schema, ctx, unit, rrs, false, tableId2DataNodeCache);
                        if (rrsTmp != null) {
                            for (RouteResultsetNode node : rrsTmp.getNodes()) {
                                nodeSet.add(node);
                            }
                        }
                    }

                    RouteResultsetNode[] nodes = new RouteResultsetNode[nodeSet.size()];
                    int i = 0;
                    for (Iterator iterator = nodeSet.iterator(); iterator.hasNext(); ) {
                        nodes[i] = (RouteResultsetNode) iterator.next();
                        i++;
                    }

                    rrs.setNodes(nodes);
                    return rrs;
                } catch (SQLNonTransientException e) {
                    throw new RuntimeException(e);
                }
            }


        }

        return null;
    }


    private void parseOneLine(List columns, String tableName, String[] line, boolean toFile, String lineEnd) {

        RouteResultset rrs = tryDirectRoute(sql, line);
        if (rrs == null || rrs.getNodes() == null || rrs.getNodes().length == 0) {

            String insertSql = makeSimpleInsert(columns, line, tableName, true);
            rrs = serverConnection.routeSQL(insertSql, ServerParse.INSERT);
        }


        if (rrs == null || rrs.getNodes() == null || rrs.getNodes().length == 0) {
            //无路由处理
        } else {
            for (RouteResultsetNode routeResultsetNode : rrs.getNodes()) {
                String name = routeResultsetNode.getName();
                LoadData data = routeResultMap.get(name);
                if (data == null) {
                    data = new LoadData();
                    data.setCharset(loadData.getCharset());
                    data.setEnclose(loadData.getEnclose());
                    data.setFieldTerminatedBy(loadData.getFieldTerminatedBy());
                    data.setLineTerminatedBy(loadData.getLineTerminatedBy());
                    data.setEscape(loadData.getEscape());
                    routeResultMap.put(name, data);
                }

                String jLine = joinField(line, data);
                if (shoudAddSlot) {
                    jLine = jLine + loadData.getFieldTerminatedBy() + routeResultsetNode.getSlot();
                }
                if (data.getData() == null) {
                    data.setData(Lists.newArrayList(jLine));
                } else {

                    data.getData().add(jLine);

                }

                if (toFile
                        //避免当导入数据跨多分片时内存溢出的情况
                        && data.getData().size() > 10000) {
                    saveDataToFile(data, name);
                }

            }
        }
    }

    private void flushDataToFile() {
        for (Map.Entry stringLoadDataEntry : routeResultMap.entrySet()) {
            LoadData value = stringLoadDataEntry.getValue();
            if (value.getFileName() != null && value.getData() != null && value.getData().size() > 0) {
                saveDataToFile(value, stringLoadDataEntry.getKey());
            }
        }

    }

    private void saveDataToFile(LoadData data, String dnName) {
        if (data.getFileName() == null) {
            String dnPath = tempPath + dnName + ".txt";
            data.setFileName(dnPath);
        }

        File dnFile = new File(data.getFileName());
        try {
            if (!dnFile.exists()) {
                Files.createParentDirs(dnFile);
            }
            Files.append(joinLine(data.getData(), data), dnFile, Charset.forName(loadData.getCharset()));

        } catch (IOException e) {
            throw new RuntimeException(e);
        } finally {
            data.setData(null);

        }


    }

    private String joinLine(List data, LoadData loadData) {
        StringBuilder sb = new StringBuilder();
        for (String s : data) {
            sb.append(s).append(loadData.getLineTerminatedBy());
        }
        return sb.toString();
    }


    private String joinField(String[] src, LoadData loadData) {
        StringBuilder sb = new StringBuilder();
        for (int i = 0, srcLength = src.length; i < srcLength; i++) {
            String s = src[i] != null ? src[i] : "";
            if (loadData.getEnclose() == null) {
                sb.append(s);
            } else {
                sb.append(loadData.getEnclose()).append(s.replace(loadData.getEnclose(), loadData.getEscape() + loadData.getEnclose())).append(loadData.getEnclose());
            }
            if (i != srcLength - 1) {
                sb.append(loadData.getFieldTerminatedBy());
            }
        }

        return sb.toString();

    }


    private RouteResultset buildResultSet(Map routeMap) {
        statement.setLocal(true);//强制local
        SQLLiteralExpr fn = new SQLCharExpr(fileName);    //默认druid会过滤掉路径的分隔符，所以这里重新设置下
        statement.setFileName(fn);
        String srcStatement = statement.toString();
        RouteResultset rrs = new RouteResultset(srcStatement, ServerParse.LOAD_DATA_INFILE_SQL);
        rrs.setLoadData(true);
        rrs.setStatement(srcStatement);
        rrs.setAutocommit(serverConnection.isAutocommit());
        rrs.setFinishedRoute(true);
        int size = routeMap.size();
        RouteResultsetNode[] routeResultsetNodes = new RouteResultsetNode[size];
        int index = 0;
        for (String dn : routeMap.keySet()) {
            RouteResultsetNode rrNode = new RouteResultsetNode(dn, ServerParse.LOAD_DATA_INFILE_SQL, srcStatement);
            rrNode.setSource(rrs);
            rrNode.setTotalNodeSize(size);
            rrNode.setStatement(srcStatement);
            LoadData newLoadData = new LoadData();
            ObjectUtil.copyProperties(loadData, newLoadData);
            newLoadData.setLocal(true);
            LoadData loadData1 = routeMap.get(dn);
            //  if (isHasStoreToFile)
            if (loadData1.getFileName() != null)//此处判断是否有保存分库load的临时文件dn1.txt/dn2.txt，不是判断是否有clientTemp.txt
            {
                newLoadData.setFileName(loadData1.getFileName());
            } else {
                newLoadData.setData(loadData1.getData());
            }
            rrNode.setLoadData(newLoadData);

            routeResultsetNodes[index] = rrNode;
            index++;
        }
        rrs.setNodes(routeResultsetNodes);
        return rrs;
    }


    private String makeSimpleInsert(List columns, String[] fields, String table, boolean isAddEncose) {
        StringBuilder sb = new StringBuilder();
        sb.append(LoadData.loadDataHint).append("insert into ").append(table.toUpperCase());
        if (columns != null && columns.size() > 0) {
            sb.append("(");
            for (int i = 0, columnsSize = columns.size(); i < columnsSize; i++) {
                SQLExpr column = columns.get(i);
                sb.append(column.toString());
                if (i != columnsSize - 1) {
                    sb.append(",");
                }
            }
            sb.append(") ");
        }

        sb.append(" values (");
        for (int i = 0, columnsSize = fields.length; i < columnsSize; i++) {
            String column = fields[i];
            if (isAddEncose) {
                sb.append("'").append(parseFieldString(column, loadData.getEnclose())).append("'");
            } else {
                sb.append(column);
            }
            if (i != columnsSize - 1) {
                sb.append(",");
            }
        }
        sb.append(")");
        return sb.toString();
    }

    private String parseFieldString(String value, String encose) {
        if (encose == null || "".equals(encose) || value == null) {
            return value;
        } else if (value.startsWith(encose) && value.endsWith(encose)) {
            return value.substring(encose.length() - 1, value.length() - encose.length());
        }
        return value;
    }


    @Override
    public void end(byte packID) {
        isStartLoadData = false;
        this.packID = packID;
        //load in data空包 结束
        saveByteOrToFile(null, true);
        List columns = statement.getColumns();
        String tableName = statement.getTableName().getSimpleName();
        if (isHasStoreToFile) {
            parseFileByLine(tempFile, loadData.getCharset(), loadData.getLineTerminatedBy());
        } else {
            String reader = new String(tempByteBuffer.toByteArray(), Charset.forName(loadData.getCharset()));

                    CSVFormat csvFormat = CSVFormat.newFormat(
                    loadData.getFieldTerminatedBy().charAt(0))
                    .withRecordSeparator(loadData.getLineTerminatedBy()).
                            withSkipHeaderRecord(false).withTrim(false);

            if (loadData.getEnclose() != null) {
                csvFormat = csvFormat.withQuote(loadData.getEnclose().charAt(0));
            }
            if (loadData.getEscape() != null) {
                csvFormat = csvFormat.withQuote(loadData.getEscape().charAt(0));
            }
            /*
             *  fix bug #1074 : LOAD DATA local INFILE导入的所有Boolean类型全部变成了false
             *  不可见字符将在CsvParser被当成whitespace过滤掉, 使用settings.trimValues(false)来避免被过滤掉
             *  TODO : 设置trimValues(false)之后, 会引起字段值前后的空白字符无法被过滤!
             */
            try {
                Iterable records = csvFormat.parse(new StringReader(reader));
                String[] row = null;

                for (CSVRecord record : records) {
                    int size = record.size();
                    row = new String[size];
                    for (int i = 0; i < size; i++) {
                        row[i] = record.get(i);
                    }
                    parseOneLine(columns, tableName, row, false, null);
                }
            } catch (Throwable e) {
                LOGGER.error("解析csv格式错误", e);
            }
        }

        RouteResultset rrs = buildResultSet(routeResultMap);
        if (rrs != null) {
            flushDataToFile();
            serverConnection.getSession2().execute(rrs, ServerParse.LOAD_DATA_INFILE_SQL);
        }


        // sendOk(++packID);


    }


    private void parseFileByLine(String file, String encode, String split) {
        List columns = statement.getColumns();
        CsvParserSettings settings = new CsvParserSettings();
        settings.setMaxColumns(65535);
        settings.setMaxCharsPerColumn(65535);
        settings.getFormat().setLineSeparator(loadData.getLineTerminatedBy());
        settings.getFormat().setDelimiter(loadData.getFieldTerminatedBy().charAt(0));
        if (loadData.getEnclose() != null) {
            settings.getFormat().setQuote(loadData.getEnclose().charAt(0));
        }
        if (loadData.getEscape() != null) {
            settings.getFormat().setQuoteEscape(loadData.getEscape().charAt(0));
        }
        settings.getFormat().setNormalizedNewline(loadData.getLineTerminatedBy().charAt(0));
        /*
         *  fix #1074 : LOAD DATA local INFILE导入的所有Boolean类型全部变成了false
         *  不可见字符将在CsvParser被当成whitespace过滤掉, 使用settings.trimValues(false)来避免被过滤掉
         *  TODO : 设置trimValues(false)之后, 会引起字段值前后的空白字符无法被过滤!
         */
        settings.trimValues(false);
        CsvParser parser = new CsvParser(settings);
        InputStreamReader reader = null;
        FileInputStream fileInputStream = null;
        try {

            fileInputStream = new FileInputStream(file);
            reader = new InputStreamReader(fileInputStream, encode);
            parser.beginParsing(reader);
            String[] row = null;

            while ((row = parser.parseNext()) != null) {
                parseOneLine(columns, tableName, row, true, loadData.getLineTerminatedBy());
            }


        } catch (FileNotFoundException | UnsupportedEncodingException e) {
            throw new RuntimeException(e);
        } finally {
            parser.stopParsing();
            if (fileInputStream != null) {
                try {
                    fileInputStream.close();
                } catch (IOException e) {
                    throw new RuntimeException(e);
                }
            }
            if (reader != null) {
                try {
                    reader.close();
                } catch (IOException e) {
                    throw new RuntimeException(e);
                }
            }

        }

    }


    public void clear() {
        isStartLoadData = false;
        tableId2DataNodeCache = null;
        schema = null;
        tableConfig = null;
        isHasStoreToFile = false;
        packID = 0;
        tempByteBuffrSize = 0;
        tableName = null;
        partitionColumnIndex = -1;
        varColumns = null;
        if (tempFile != null) {
            File temp = new File(tempFile);
            if (temp.exists()) {
                temp.delete();
            }
        }
        if (tempPath != null && new File(tempPath).exists()) {
            deleteFile(tempPath);
        }
        tempByteBuffer = null;
        loadData = null;
        sql = null;
        fileName = null;
        statement = null;
        routeResultMap.clear();
    }

    @Override
    public byte getLastPackId() {
        return packID;
    }

    @Override
    public boolean isStartLoadData() {
        return isStartLoadData;
    }

    private String getPartitionColumn() {
        String pColumn;
        if (tableConfig.isSecondLevel()
                && tableConfig.getParentTC().getPartitionColumn()
                .equals(tableConfig.getParentKey())) {
            pColumn = tableConfig.getJoinKey();
        } else {
            pColumn = tableConfig.getPartitionColumn();
        }
        return pColumn;
    }

    /**
     * 删除目录及其所有子目录和文件
     *
     * @param dirPath 要删除的目录路径
     * @throws Exception
     */
    private static void deleteFile(String dirPath) {
        File fileDirToDel = new File(dirPath);
        if (!fileDirToDel.exists()) {
            return;
        }
        if (fileDirToDel.isFile()) {
            fileDirToDel.delete();
            return;
        }
        File[] fileList = fileDirToDel.listFiles();

        for (int i = 0; i < fileList.length; i++) {
            File file = fileList[i];
            if (file.isFile() && file.exists()) {
                boolean delete = file.delete();
            } else if (file.isDirectory()) {
                deleteFile(file.getAbsolutePath());
                file.delete();
            }
        }
        fileDirToDel.delete();
    }


}


================================================
FILE: src/main/java/io/mycat/server/handler/ServerPrepareHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese
 * opensource volunteers. you can redistribute it and/or modify it under the
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Any questions about this component can be directed to it's project Web address
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server.handler;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;

import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLReplaceable;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.*;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.MySqlKey;
import com.alibaba.druid.sql.dialect.mysql.ast.MySqlPrimaryKey;
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.escape.Escaper;
import com.google.common.escape.Escapers;
import com.google.common.escape.Escapers.Builder;

import io.mycat.backend.mysql.BindValue;
import io.mycat.backend.mysql.ByteUtil;
import io.mycat.backend.mysql.PreparedStatement;
import io.mycat.backend.mysql.nio.handler.PrepareRequestHandler;
import io.mycat.backend.mysql.nio.handler.PrepareRequestHandler.PrepareRequestCallback;
import io.mycat.config.ErrorCode;
import io.mycat.config.Fields;
import io.mycat.net.handler.FrontendPrepareHandler;
import io.mycat.net.mysql.ExecutePacket;
import io.mycat.net.mysql.FieldPacket;
import io.mycat.net.mysql.LongDataPacket;
import io.mycat.net.mysql.OkPacket;
import io.mycat.net.mysql.ResetPacket;
import io.mycat.server.ServerConnection;
import io.mycat.server.response.PreparedStmtResponse;
import io.mycat.util.HexFormatUtil;

/**
 * @author mycat, CrazyPig, zhuam
 */
public class ServerPrepareHandler implements FrontendPrepareHandler {

    private static final Logger LOGGER = LoggerFactory.getLogger(ServerPrepareHandler.class);

    private static Escaper varcharEscaper = null;

    static {
        Builder escapeBuilder = Escapers.builder();
        escapeBuilder.addEscape('\0', "\\0");
        escapeBuilder.addEscape('\'', "\\'");
        escapeBuilder.addEscape('\b', "\\b");
        escapeBuilder.addEscape('\n', "\\n");
        escapeBuilder.addEscape('\r', "\\r");
        escapeBuilder.addEscape('\"', "\\\"");
        escapeBuilder.addEscape('$', "\\$");
        escapeBuilder.addEscape('\\', "\\\\");
        varcharEscaper = escapeBuilder.build();
    }

    private ServerConnection source;

    // java int是32位，long是64位；mysql协议里面定义的statementId是32位，因此用Integer
    private static final AtomicInteger PSTMT_ID_GENERATOR = new AtomicInteger(0);
    //    private static final Map pstmtForSql = new ConcurrentHashMap<>();
    private static final Map pstmtForId = new ConcurrentHashMap<>();
    private int maxPreparedStmtCount;

    public ServerPrepareHandler(ServerConnection source, int maxPreparedStmtCount) {
        this.source = source;
        this.maxPreparedStmtCount = maxPreparedStmtCount;
    }

    @Override
    public void prepare(String sql) {

        LOGGER.debug("use server prepare, sql: " + sql);
        PreparedStatement pstmt = null;
        if (pstmt == null) {
            // 解析获取字段个数和参数个数
            int columnCount = 0;
            int paramCount = getParamCount(sql);
            if (paramCount > maxPreparedStmtCount) {
                source.writeErrMessage(ErrorCode.ER_PS_MANY_PARAM,
                        "Prepared statement contains too many placeholders");
                return;
            }
            pstmt = new PreparedStatement(PSTMT_ID_GENERATOR.incrementAndGet(), sql,
                    paramCount);
            pstmtForId.put(pstmt.getId(), pstmt);
            LOGGER.info("preparestatement  parepare id:{}", pstmt.getId());
        }
        PreparedStmtResponse.response(pstmt, source);
    }


    @Override
    public void sendLongData(byte[] data) {
        LongDataPacket packet = new LongDataPacket();
        packet.read(data);
        long pstmtId = packet.getPstmtId();
        LOGGER.info("preparestatement  long data id:{}", pstmtId);
        PreparedStatement pstmt = pstmtForId.get(pstmtId);
        if (pstmt != null) {
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug("send long data to prepare sql : " + pstmtForId.get(pstmtId));
            }
            long paramId = packet.getParamId();
            try {
                pstmt.appendLongData(paramId, packet.getLongData());
            } catch (IOException e) {
                source.writeErrMessage(ErrorCode.ERR_FOUND_EXCEPTION, e.getMessage());
            }
        }
    }

    @Override
    public void reset(byte[] data) {
        ResetPacket packet = new ResetPacket();
        packet.read(data);
        long pstmtId = packet.getPstmtId();
        LOGGER.info("preparestatement  long data id:{}", pstmtId);
        PreparedStatement pstmt = pstmtForId.get(pstmtId);
        if (pstmt != null) {
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug("reset prepare sql : " + pstmtForId.get(pstmtId));
            }
            pstmt.resetLongData();
            source.write(OkPacket.OK);
        } else {
            source.writeErrMessage(ErrorCode.ERR_FOUND_EXCEPTION,
                    "can not reset prepare statement : " + pstmtForId.get(pstmtId));
        }
    }

    @Override
    public void execute(byte[] data) {
        long pstmtId = ByteUtil.readUB4(data, 5);
        PreparedStatement pstmt = null;
        LOGGER.info("preparestatement  execute id:{}", pstmtId);
        if ((pstmt = pstmtForId.get(pstmtId)) == null) {
            source.writeErrMessage(ErrorCode.ER_ERROR_WHEN_EXECUTING_COMMAND,
                    "Unknown pstmtId when executing.");
        } else {
            ExecutePacket packet = new ExecutePacket(pstmt);
            try {
                packet.read(data, source.getCharset());
            } catch (UnsupportedEncodingException e) {
                source.writeErrMessage(ErrorCode.ER_ERROR_WHEN_EXECUTING_COMMAND, e.getMessage());
                return;
            }
            BindValue[] bindValues = packet.values;
            // 还原sql中的动态参数为实际参数值
            String sql = prepareStmtBindValue(pstmt, bindValues);
            // 执行sql
            source.getSession2().setPrepared(true);
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug("execute prepare sql: " + sql);
            }

            pstmt.resetLongData();
            source.query(sql);
        }
    }


    @Override
    public void close(byte[] data) {
        long pstmtId = ByteUtil.readUB4(data, 5); // 获取prepare stmt id
        LOGGER.info("preparestatement  close id:{}", pstmtId);
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("close prepare stmt, stmtId = " + pstmtId);
        }
        PreparedStatement pstmt = pstmtForId.remove(pstmtId);
    }

    @Override
    public void clear() {
        this.pstmtForId.clear();
//    this.pstmtForSql.clear();
    }


    // 获取预处理sql中预处理参数个数
    private int getParamCount(String sql) {
        char[] cArr = sql.toCharArray();
        int count = 0;
        for (int i = 0; i < cArr.length; i++) {
            if (cArr[i] == '?') {
                count++;
            }
        }
        return count;
    }

    /**
     * 组装sql语句,替换动态参数为实际参数值
     */
    private String prepareStmtBindValue(PreparedStatement pstmt, BindValue[] bindValues) {
        String sql = pstmt.getStatement();
        SQLStatement sqlStatement = SQLUtils.parseSingleMysqlStatement(sql);
        boolean primitiveArg = !(sqlStatement instanceof SQLSelectStatement);
        int[] paramTypes = pstmt.getParametersType();
        sqlStatement.accept(new MySqlASTVisitorAdapter() {
            @Override
            public boolean visit(SQLVariantRefExpr x) {
                BindValue bindValue = bindValues[x.getIndex()];
                Object o = null;
                if (bindValue.isNull) {
                    SQLReplaceable parent = (SQLReplaceable) x.getParent();
                    parent.replace(x, new SQLNullExpr());
                    return false;
                } else {
                    if (primitiveArg && bindValue.value instanceof byte[]) {
                        SQLReplaceable parent = (SQLReplaceable) x.getParent();
                        parent.replace(x, new SQLHexExpr(HexFormatUtil.bytesToHexString((byte[]) bindValue.value)));
                        return false;
                    }
                    switch (paramTypes[x.getIndex()] & 0xff) {
                        case Fields.FIELD_TYPE_TINY:
                            o = bindValue.byteBinding;
                            break;
                        case Fields.FIELD_TYPE_SHORT:
                            o = bindValue.shortBinding;
                            break;
                        case Fields.FIELD_TYPE_LONG:
                            o = bindValue.intBinding;
                            break;
                        case Fields.FIELD_TYPE_LONGLONG:
                            o = (bindValue.longBinding);
                            break;
                        case Fields.FIELD_TYPE_FLOAT:
                            o = bindValue.floatBinding;
                            break;
                        case Fields.FIELD_TYPE_DOUBLE:
                            o = bindValue.doubleBinding;
                            break;
                        case Fields.FIELD_TYPE_TIME:
                        case Fields.FIELD_TYPE_DATE:
                        case Fields.FIELD_TYPE_DATETIME:
                        case Fields.FIELD_TYPE_TIMESTAMP:
                            o = bindValue.value;
                            break;
                        default:
                            if (bindValue.value instanceof byte[]) {
                                SQLReplaceable parent = (SQLReplaceable) x.getParent();
                                byte[] bytes = (byte[]) bindValue.value;
                                parent.replace(x, new SQLCharExpr(new String(bytes)));
                                return false;
                            }
                            if (bindValue.value instanceof String) {
                                SQLReplaceable parent = (SQLReplaceable) x.getParent();
                                String value = (String) bindValue.value;
                                parent.replace(x, new SQLCharExpr(value));
                                return false;
                            }
                            throw new UnsupportedOperationException("unsupport " + bindValue.value);
                    }

                    SQLReplaceable parent = (SQLReplaceable) x.getParent();
                    parent.replace(x, SQLExprUtils.fromJavaObject(o));
                    return false;
                }
            }

        });
        return sqlStatement.toString();
    }
}

================================================
FILE: src/main/java/io/mycat/server/handler/SetHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server.handler;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import io.mycat.config.ErrorCode;
import io.mycat.config.Isolations;
import io.mycat.net.mysql.OkPacket;
import io.mycat.route.parser.util.ParseUtil;
import io.mycat.server.ServerConnection;
import io.mycat.server.parser.ServerParse;
import io.mycat.server.parser.ServerParseSet;
import io.mycat.server.response.CharacterSet;
import io.mycat.util.SetIgnoreUtil;

import static io.mycat.server.parser.ServerParseSet.AUTOCOMMIT_OFF;
import static io.mycat.server.parser.ServerParseSet.AUTOCOMMIT_ON;
import static io.mycat.server.parser.ServerParseSet.CHARACTER_SET_CLIENT;
import static io.mycat.server.parser.ServerParseSet.CHARACTER_SET_CONNECTION;
import static io.mycat.server.parser.ServerParseSet.CHARACTER_SET_RESULTS;
import static io.mycat.server.parser.ServerParseSet.NAMES;
import static io.mycat.server.parser.ServerParseSet.SQL_SELECT_LIMIT;
import static io.mycat.server.parser.ServerParseSet.TX_READONLY;
import static io.mycat.server.parser.ServerParseSet.TX_READWRITE;
import static io.mycat.server.parser.ServerParseSet.TX_READ_COMMITTED;
import static io.mycat.server.parser.ServerParseSet.TX_READ_UNCOMMITTED;
import static io.mycat.server.parser.ServerParseSet.TX_REPEATED_READ;
import static io.mycat.server.parser.ServerParseSet.TX_SERIALIZABLE;
import static io.mycat.server.parser.ServerParseSet.XA_FLAG_OFF;
import static io.mycat.server.parser.ServerParseSet.XA_FLAG_ON;

/**
 * SET 语句处理
 * 
 * @author mycat
 * @author zhuam
 */
public final class SetHandler {
	
	private static final Logger logger = LoggerFactory.getLogger(SetHandler.class);
	
	private static final byte[] AC_OFF = new byte[] { 7, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 };
		
	public static void handle(String stmt, ServerConnection c, int offset) {
		// System.out.println("SetHandler: "+stmt);
		int rs = ServerParseSet.parse(stmt, offset);
		switch (rs & 0xff) {
		case AUTOCOMMIT_ON:
			if (c.isAutocommit()) {
				c.write(c.writeToBuffer(OkPacket.OK, c.allocate()));
			} else {
				c.setPreAcStates(true);
				c.commit();
				c.setAutocommit(true);
			}
			break;
		case AUTOCOMMIT_OFF: {
			if (c.isAutocommit()) {
				c.setAutocommit(false);
				c.setPreAcStates(false);
			}
			c.write(c.writeToBuffer(AC_OFF, c.allocate()));
			break;
		}
		case XA_FLAG_ON: {
			if (c.isAutocommit()) {
				c.writeErrMessage(ErrorCode.ERR_WRONG_USED,
						"set xa cmd on can't used in autocommit connection ");
				return;
			}
			c.getSession2().setXATXEnabled(true);
			c.write(c.writeToBuffer(OkPacket.OK, c.allocate()));
			break;
		}
		case XA_FLAG_OFF: {
			c.writeErrMessage(ErrorCode.ERR_WRONG_USED,
					"set xa cmd off not for external use ");
			return;
		}
		case TX_READ_UNCOMMITTED: {
			c.setTxIsolation(Isolations.READ_UNCOMMITTED);
			c.write(c.writeToBuffer(OkPacket.OK, c.allocate()));
			break;
		}
		case TX_READ_COMMITTED: {
			c.setTxIsolation(Isolations.READ_COMMITTED);
			c.write(c.writeToBuffer(OkPacket.OK, c.allocate()));
			break;
		}
		case TX_REPEATED_READ: {
			c.setTxIsolation(Isolations.REPEATED_READ);
			c.write(c.writeToBuffer(OkPacket.OK, c.allocate()));
			break;
		}
		case TX_SERIALIZABLE: {
			c.setTxIsolation(Isolations.SERIALIZABLE);
			c.write(c.writeToBuffer(OkPacket.OK, c.allocate()));
			break;
		}
		case TX_READONLY: {
			c.setTxReadonly(true);
			c.write(c.writeToBuffer(OkPacket.OK, c.allocate()));
			break;
		}
		case TX_READWRITE: {
			c.setTxReadonly(false);
			c.write(c.writeToBuffer(OkPacket.OK, c.allocate()));
			break;
		}
		case NAMES:
			String charset = stmt.substring(rs >>> 8).trim();
		   int index=	charset.indexOf(",")  ;
			if(index>-1) {
				//支持rails框架自动生成的SET NAMES utf8,  @@SESSION.sql_auto_is_null = 0, @@SESSION.wait_timeout = 2147483, @@SESSION.sql_mode = 'STRICT_ALL_TABLES'
			charset=charset.substring(0,index)	;
			}
			if(charset.startsWith("'")&&charset.endsWith("'"))
			{
				charset=charset.substring(1,charset.length()-1)  ;
			}
			if (c.setCharset(charset)) {
				c.write(c.writeToBuffer(OkPacket.OK, c.allocate()));
			} else {
				
				/**
				 * TODO：修复 phpAyAdmin's 的发包问题
				 * 如： SET NAMES 'utf8' COLLATE 'utf8_general_ci' 错误
				 */	
				int beginIndex = stmt.toLowerCase().indexOf("names");
				int endIndex = stmt.toLowerCase().indexOf("collate");
				if ( beginIndex > -1 && endIndex > -1 ) {					
					charset = stmt.substring(beginIndex + "names".length(), endIndex);					
					//重试一次
					if (c.setCharset( charset.trim() )) {
						c.write(c.writeToBuffer(OkPacket.OK, c.allocate()));
					} else {
						c.writeErrMessage(ErrorCode.ER_UNKNOWN_CHARACTER_SET, "Unknown charset '" + charset + "'");
					}	
					
				} else {				
					c.writeErrMessage(ErrorCode.ER_UNKNOWN_CHARACTER_SET, "Unknown charset '" + charset + "'");
				}
			}
			break;
		case SQL_SELECT_LIMIT:
			String limit = ParseUtil.parseString(stmt);
			int sqlSelectLimit = -1;
			if ("default".equalsIgnoreCase(limit)) {
				sqlSelectLimit = -1;
			} else {
				try{
					sqlSelectLimit = Integer.parseInt(limit);
				} catch ( Exception  ex) {
					c.writeErrMessage(ErrorCode.ER_YES, "Unsupported statement:"+ex.getMessage());
					break;
				}

			}
			c.setSqlSelectLimit(sqlSelectLimit);
			c.write(c.writeToBuffer(OkPacket.OK, c.allocate()));
			break;
		case CHARACTER_SET_CLIENT:
		case CHARACTER_SET_CONNECTION:
		case CHARACTER_SET_RESULTS:
			CharacterSet.response(stmt, c, rs);
			break;
		default:			
			 boolean ignore = SetIgnoreUtil.isIgnoreStmt(stmt);
             if ( ignore ) {
     			StringBuilder s = new StringBuilder();
    			logger.warn(s.append(c).append(stmt).append(" is not recoginized and ignored").toString());
				 c.write(c.writeToBuffer(OkPacket.OK, c.allocate()));
             }else{
				 c.execute(stmt, ServerParse.UPDATE);
			 }
		}
	}

}


================================================
FILE: src/main/java/io/mycat/server/handler/ShowCache.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server.handler;

import java.nio.ByteBuffer;
import java.util.Map;

import io.mycat.MycatServer;
import io.mycat.backend.mysql.PacketUtil;
import io.mycat.cache.CachePool;
import io.mycat.cache.CacheService;
import io.mycat.cache.CacheStatic;
import io.mycat.cache.LayerCachePool;
import io.mycat.config.Fields;
import io.mycat.manager.ManagerConnection;
import io.mycat.net.mysql.EOFPacket;
import io.mycat.net.mysql.FieldPacket;
import io.mycat.net.mysql.ResultSetHeaderPacket;
import io.mycat.net.mysql.RowDataPacket;
import io.mycat.util.LongUtil;
import io.mycat.util.StringUtil;

public class ShowCache {

	private static final int FIELD_COUNT = 8;
	private static final ResultSetHeaderPacket header = PacketUtil
			.getHeader(FIELD_COUNT);
	private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT];
	private static final EOFPacket eof = new EOFPacket();
	static {
		int i = 0;
		byte packetId = 0;
		header.packetId = ++packetId;

		fields[i] = PacketUtil.getField("CACHE", Fields.FIELD_TYPE_VAR_STRING);
		fields[i++].packetId = ++packetId;
		fields[i] = PacketUtil.getField("MAX", Fields.FIELD_TYPE_LONG);
		fields[i++].packetId = ++packetId;
		fields[i] = PacketUtil.getField("CUR", Fields.FIELD_TYPE_LONG);
		fields[i++].packetId = ++packetId;
		fields[i] = PacketUtil.getField("ACCESS", Fields.FIELD_TYPE_LONG);
		fields[i++].packetId = ++packetId;
		fields[i] = PacketUtil.getField("HIT", Fields.FIELD_TYPE_LONG);
		fields[i++].packetId = ++packetId;
		fields[i] = PacketUtil.getField("PUT", Fields.FIELD_TYPE_LONG);
		fields[i++].packetId = ++packetId;
		fields[i] = PacketUtil.getField("LAST_ACCESS", Fields.FIELD_TYPE_LONG);
		fields[i++].packetId = ++packetId;
		fields[i] = PacketUtil.getField("LAST_PUT", Fields.FIELD_TYPE_LONG);
		fields[i++].packetId = ++packetId;
		eof.packetId = ++packetId;
	}

	public static void execute(ManagerConnection c) {

		ByteBuffer buffer = c.allocate();

		// write header
		buffer = header.write(buffer, c,true);

		// write fields
		for (FieldPacket field : fields) {
			buffer = field.write(buffer, c,true);
		}

		// write eof
		buffer = eof.write(buffer, c,true);

		// write rows
		byte packetId = eof.packetId;
		CacheService cacheService = MycatServer.getInstance().getCacheService();
		for (Map.Entry entry : cacheService
				.getAllCachePools().entrySet()) {
			String cacheName=entry.getKey();
			CachePool cachePool = entry.getValue();
			if (cachePool instanceof LayerCachePool) {
				for (Map.Entry staticsEntry : ((LayerCachePool) cachePool)
						.getAllCacheStatic().entrySet()) {
					RowDataPacket row = getRow(cacheName+'.'+staticsEntry.getKey(),
							staticsEntry.getValue(), c.getCharset());
					row.packetId = ++packetId;
					buffer = row.write(buffer, c,true);
				}
			} else {
				RowDataPacket row = getRow(cacheName,
						cachePool.getCacheStatic(), c.getCharset());
				row.packetId = ++packetId;
				buffer = row.write(buffer, c,true);
			}
		}

		// write last eof
		EOFPacket lastEof = new EOFPacket();
		lastEof.packetId = ++packetId;
		buffer = lastEof.write(buffer, c,true);

		// write buffer
		c.write(buffer);
	}

	private static RowDataPacket getRow(String poolName,
			CacheStatic cacheStatic, String charset) {
		RowDataPacket row = new RowDataPacket(FIELD_COUNT);
		row.add(StringUtil.encode(poolName, charset));
		// max size
		row.add(LongUtil.toBytes(cacheStatic.getMaxSize()));
		row.add(LongUtil.toBytes(cacheStatic.getItemSize()));
		row.add(LongUtil.toBytes(cacheStatic.getAccessTimes()));
		row.add(LongUtil.toBytes(cacheStatic.getHitTimes()));
		row.add(LongUtil.toBytes(cacheStatic.getPutTimes()));
		row.add(LongUtil.toBytes(cacheStatic.getLastAccesTime()));
		row.add(LongUtil.toBytes(cacheStatic.getLastPutTime()));
		return row;
	}

}

================================================
FILE: src/main/java/io/mycat/server/handler/ShowHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server.handler;

import io.mycat.server.ServerConnection;
import io.mycat.server.parser.ServerParse;
import io.mycat.server.parser.ServerParseShow;
import io.mycat.server.response.*;
import io.mycat.util.StringUtil;

/**
 * @author mycat
 */
public final class ShowHandler {

	public static void handle(final String originalStmt, ServerConnection c, int offset) {

		// 排除 “ ` ” 符号
		String stmt = StringUtil.replaceChars(originalStmt, "`", null,true);

		int type = ServerParseShow.parse(stmt, offset);
		switch (type) {
		case ServerParseShow.DATABASES:
			ShowDatabases.response(c);
			break;
		case ServerParseShow.TABLES:
			ShowTables.response(c, stmt,type);
			break;
        case ServerParseShow.FULLTABLES:
            ShowFullTables.response(c, stmt, type);
            break;
		case ServerParseShow.MYCAT_STATUS:
			ShowMyCatStatus.response(c);
			break;
		case ServerParseShow.MYCAT_CLUSTER:
			ShowMyCATCluster.response(c);
			break;
		default:
			c.execute(originalStmt, ServerParse.SHOW);
		}
	}

}

================================================
FILE: src/main/java/io/mycat/server/handler/StartHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server.handler;

import io.mycat.config.ErrorCode;
import io.mycat.server.ServerConnection;
import io.mycat.server.parser.ServerParse;
import io.mycat.server.parser.ServerParseStart;

/**
 * @author mycat
 */
public final class StartHandler {
    private static final byte[] AC_OFF = new byte[] { 7, 0, 0, 1, 0, 0, 0, 0,
            0, 0, 0 };
    public static void handle(String stmt, ServerConnection c, int offset) {
        switch (ServerParseStart.parse(stmt, offset)) {
        case ServerParseStart.TRANSACTION:
            if (c.isAutocommit())
            {
                c.write(c.writeToBuffer(AC_OFF, c.allocate()));
            }else
            {
                c.getSession2().commit() ;
            }
            c.setAutocommit(false);
            break;
        default:
            c.execute(stmt, ServerParse.START);
        }
    }

}

================================================
FILE: src/main/java/io/mycat/server/handler/UseHandler.java
================================================
/*
 * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software;Designed and Developed mainly by many Chinese 
 * opensource volunteers. you can redistribute it and/or modify it under the 
 * terms of the GNU General Public License version 2 only, as published by the
 * Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 * 
 * Any questions about this component can be directed to it's project Web address 
 * https://code.google.com/p/opencloudb/.
 *
 */
package io.mycat.server.handler;

import java.nio.ByteBuffer;
import java.util.Set;

import io.mycat.config.ErrorCode;
import io.mycat.net.handler.FrontendPrivileges;
import io.mycat.net.mysql.OkPacket;
import io.mycat.server.ServerConnection;
import io.mycat.util.StringUtil;

/**
 * @author mycat
 */
public final class UseHandler {

    public static void handle(String sql, ServerConnection c, int offset) {
        String schema = sql.substring(offset).trim();
        int length = schema.length();
        if (length > 0) {
        	//许多客户端工具断链重连后会批量发送SQL,如下:
            //USE `TESTDB`;\nSELECT SYSDATE(),CURRENT_USER()
            //modify by jeff.cao 2018/3/31
            int end = schema.indexOf(";");
            if (end > 0) {
                schema = schema.substring(0, end - 1);
            }
            schema = StringUtil.replaceChars(schema, "`", null,true);
            length = schema.length();
            if (schema.charAt(0) == '\'' && schema.charAt(length - 1) == '\'') {
                schema = schema.substring(1, length - 1);
            }
        }
        // 检查schema的有效性
        FrontendPrivileges privileges = c.getPrivileges();
        if (schema == null || !privileges.schemaExists(schema)) {
            c.writeErrMessage(ErrorCode.ER_BAD_DB_ERROR, "Unknown database '" + schema + "'");
            return;
        }
        String user = c.getUser();
        if (!privileges.userExists(user, c.getHost())) {
            c.writeErrMessage(ErrorCode.ER_ACCESS_DENIED_ERROR, "Access denied for user '" + c.getUser() + "'");
            return;
        }
        Set schemas = privileges.getUserSchemas(user);
        if (schemas == null || schemas.size() == 0 || schemas.contains(schema)) {
            c.setSchema(schema);
            ByteBuffer buffer = c.allocate();
            c.write(c.writeToBuffer(OkPacket.OK, buffer));
        } else {
            String msg = "Access denied for user '" + c.getUser() + "' to database '" + schema + "'";
            c.writeErrMessage(ErrorCode.ER_DBACCESS_DENIED_ERROR, msg);
        }
    }

}


================================================
FILE: src/main/java/io/mycat/server/interceptor/SQLInterceptor.java
================================================
package io.mycat.server.interceptor;
/**
 * used for interceptor sql before execute ,can modify sql befor execute
 * @author wuzhih
 *
 */
public interface SQLInterceptor {

	/**
	 * return new sql to handler,ca't modify sql's type 
	 * @param sql
	 * @param sqlType
	 * @return new sql
	 */
	String interceptSQL(String sql ,int sqlType);
}


================================================
FILE: src/main/java/io/mycat/server/interceptor/impl/DefaultSqlInterceptor.java
================================================
package io.mycat.server.interceptor.impl;

import io.mycat.MycatServer;
import io.mycat.config.model.SystemConfig;
import io.mycat.server.interceptor.SQLInterceptor;

public class DefaultSqlInterceptor implements SQLInterceptor {
	private static final char ESCAPE_CHAR = '\\';

	private static final int TARGET_STRING_LENGTH = 2;

	/**
	 * mysql driver对'转义与\',解析前改为foundationdb parser支持的'' add by sky
	 * 
	 * @param sql
	 * @update by jason@dayima.com replace regex with general string walking
	 * avoid sql being destroyed in case of some mismatch
	 * maybe some performance enchanced
	 * @return
	 */
	public static String processEscape(String sql) {
		int firstIndex = -1;
		if ((sql == null) || ((firstIndex = sql.indexOf(ESCAPE_CHAR)) == -1)) {
			return sql;
		} else {
			int lastIndex = sql.lastIndexOf(ESCAPE_CHAR, sql.length() - 2) + TARGET_STRING_LENGTH;
			StringBuilder sb = new StringBuilder(sql);
			for (int i = firstIndex; i < lastIndex; i ++) {
				if (sb.charAt(i) == '\\') {
					if (i + 1 < lastIndex
							&& sb.charAt(i + 1) == '\'') {
							//replace
							sb.setCharAt(i, '\'');
					}
					//roll over
					i ++;
				}
			}
			return sb.toString();
		}
	}

	/**
	 * escape mysql escape letter sql type ServerParse.UPDATE,ServerParse.INSERT
	 * etc
	 */
	@Override
	public String interceptSQL(String sql, int sqlType) {
		if("fdbparser".equals(MycatServer.getInstance().getConfig().getSystem().getDefaultSqlParser())) {
			sql = processEscape(sql);
		}
		
		// 全局表一致性 sql 改写拦截
		SystemConfig system = MycatServer.getInstance().getConfig().getSystem();
		if(system != null && system.getUseGlobleTableCheck() == 1) // 全局表一致性检测是否开启
			sql = GlobalTableUtil.interceptSQL(sql, sqlType);
		
		// other interceptors put in here ....
		
		return sql;
	}

}


================================================
FILE: src/main/java/io/mycat/server/interceptor/impl/GlobalTableUtil.java
================================================
package io.mycat.server.interceptor.impl;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;

import com.alibaba.druid.sql.ast.statement.*;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLLimit;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.SQLOrderBy;
import com.alibaba.druid.sql.ast.SQLOrderingSpecification;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLCharExpr;
import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
import com.alibaba.druid.sql.ast.expr.SQLInSubQueryExpr;
import com.alibaba.druid.sql.ast.statement.SQLInsertStatement.ValuesClause;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlInsertStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlUpdateStatement;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.fastjson.JSON;

import io.mycat.MycatServer;
import io.mycat.backend.datasource.PhysicalDBNode;
import io.mycat.backend.datasource.PhysicalDBPool;
import io.mycat.backend.datasource.PhysicalDatasource;
import io.mycat.backend.heartbeat.MySQLConsistencyChecker;
import io.mycat.backend.mysql.nio.MySQLDataSource;
import io.mycat.config.MycatConfig;
import io.mycat.config.model.SchemaConfig;
import io.mycat.config.model.TableConfig;
import io.mycat.server.parser.ServerParse;
import io.mycat.sqlengine.SQLQueryResult;
import io.mycat.util.StringUtil;

/**
 * @author digdeep@126.com
 * 全局表一致性检查 和 拦截
 */
public class GlobalTableUtil{
	private static final Logger LOGGER = LoggerFactory.getLogger(GlobalTableUtil.class);
	private static Map globalTableMap = new ConcurrentHashMap<>();
	/** 全局表 保存修改时间戳 的字段名，用于全局表一致性检查 */
	public static final String GLOBAL_TABLE_MYCAT_COLUMN = "_mycat_op_time";
	public static final String COUNT_COLUMN = "record_count";
	public static final String MAX_COLUMN = "max_timestamp";   
	public static final String INNER_COLUMN = "inner_col_exist";   
	private static String operationTimestamp = String.valueOf(new Date().getTime());
	private static volatile int isInnerColumnCheckFinished = 0;
	private static volatile int isColumnCountCheckFinished = 0;
	private static final ReentrantLock lock = new ReentrantLock(false);
	private static List>> innerColumnNotExist = new ArrayList<>();
	private static Map tableColumsMap = new ConcurrentHashMap<>();
	
	public static Map getGlobalTableMap() {
		return globalTableMap;
	}

	static {
		getGlobalTable();	// 初始化 globalTableMap
	}
	
	public static String interceptSQL(String sql, int sqlType){
		return GlobalTableUtil.consistencyInterceptor(sql, sqlType);
	}
	
	public static String consistencyInterceptor(String sql, int sqlType){
		// 统一使用mycat-server所在机器的时间，防止不同mysqld时间不同步
		operationTimestamp = String.valueOf(new Date().getTime());
		
		LOGGER.debug("before intercept: " +  sql);
		
		if(sqlType == ServerParse.INSERT){
			sql =  convertInsertSQL(sql);
		}
		if(sqlType == ServerParse.UPDATE){
			sql = convertUpdateSQL(sql);
		}
		if(sqlType == ServerParse.DDL){
			LOGGER.info(" DDL to modify global table.");
			sql = handleDDLSQL(sql);	
		}
		
		LOGGER.debug("after intercept: " +  sql);
		/*
		   目前  mycat-server不支持 replace 语句，报错如下：
		 ERROR 1064 (HY000):  ReplaceStatement can't be supported,
		 use insert into ...on duplicate key update... instead
		 
		if(sqlType == ServerParse.REPLACE){
			return convertReplaceSQL(sql);
		}
		*/
		return sql;
	}
	
	/*
	 * Name: 'ALTER TABLE'
		Description:
		Syntax:
		ALTER [IGNORE] TABLE tbl_name
		    [alter_specification [, alter_specification] ...]
		    [partition_options]
	       如果 DDL 修改了表结构，需要重新获得表的列list
	 */
	private static String handleDDLSQL(String sql){
		MySqlStatementParser parser = new MySqlStatementParser(sql);	 
		SQLStatement statement = parser.parseStatement();
		// druid高版本去掉了 MySqlAlterTableStatement，在其父类 SQLAlterTableStatement 直接支持 mysql alter table 语句
//			MySqlAlterTableStatement alter = (MySqlAlterTableStatement)statement;
		SQLExprTableSource source = getDDLTableSource(statement);
		if (source == null)
			return sql;
		String tableName = StringUtil.removeBackquote(source.toString());
		if(StringUtils.isNotBlank(tableName))
			tableName = tableName.trim();
		else
			return sql;
		
		if(!isGlobalTable(tableName))
			return sql;
		
		//增加对全局表create语句的解析，如果是建表语句创建的是全局表，且表中不含"_mycat_op_time"列
		//则为其增加"_mycat_op_time"列，方便导入数据。
		sql = addColumnIfCreate(sql, statement);
		
		final String tn = tableName;
		MycatServer.getInstance().getListeningExecutorService().execute(new Runnable() {
			public void run() {
				try {
					TimeUnit.SECONDS.sleep(3);	// DDL发出之后，等待3秒让DDL分发完成
				} catch (InterruptedException e) {
				} 
				reGetColumnsForTable(tn); // DDL 语句可能会增删 列，所以需要重新获取 全局表的 列list
			}
		});
		
		MycatServer.getInstance().getListeningExecutorService().execute(new Runnable() {
			public void run() {
				try {
					TimeUnit.MINUTES.sleep(10);	// DDL发出之后，等待10分钟再次执行，全局表一般很小，DDL耗时不会超过10分钟
				} catch (InterruptedException e) {
				} 
				reGetColumnsForTable(tn); // DDL 语句可能会增删 列，所以需要重新获取 全局表的 列list
			}
		});
		return sql;
	}

	static String addColumnIfCreate(String sql, SQLStatement statement) {
		if (isCreate(statement) && sql.trim().toUpperCase().startsWith("CREATE TABLE ") && !hasGlobalColumn(statement)) {
			SQLColumnDefinition column = new SQLColumnDefinition();
			column.setDataType(new SQLCharacterDataType("bigint"));
			column.setName(new SQLIdentifierExpr(GLOBAL_TABLE_MYCAT_COLUMN));
			column.setComment(new SQLCharExpr("全局表保存修改时间戳的字段名"));
			((SQLCreateTableStatement)statement).getTableElementList().add(column);
		}
		return statement.toString();
	}
	
	private static boolean hasGlobalColumn(SQLStatement statement){
		for (SQLTableElement tableElement : ((SQLCreateTableStatement)statement).getTableElementList()) {
			SQLName sqlName = null;
			if (tableElement instanceof SQLColumnDefinition) {
				sqlName = ((SQLColumnDefinition)tableElement).getName();
			}
			if (sqlName != null) {
				String simpleName = sqlName.getSimpleName();
				simpleName = StringUtil.removeBackquote(simpleName);
				if (tableElement instanceof SQLColumnDefinition && GLOBAL_TABLE_MYCAT_COLUMN.equalsIgnoreCase(simpleName)) {
					return true;
				}
			}
		}
		return false;
	}

	private static SQLExprTableSource getDDLTableSource(SQLStatement statement) {
		SQLExprTableSource source = null;
		if (statement instanceof SQLAlterTableStatement) {
			source = ((SQLAlterTableStatement)statement).getTableSource();
			
		} else if (isCreate(statement)) {
			source = ((SQLCreateTableStatement)statement).getTableSource();
		}
		return source;
	}

	private static boolean isCreate(SQLStatement statement) {
		return statement instanceof SQLCreateTableStatement;
	}
	
	/**
	 * Syntax:
		INSERT [LOW_PRIORITY | DELAYED | HIGH_PRIORITY] [IGNORE]
	    [INTO] tbl_name
	    [PARTITION (partition_name,...)]
	    [(col_name,...)]
	    {VALUES | VALUE} ({expr | DEFAULT},...),(...),...
	    [ ON DUPLICATE KEY UPDATE
	      col_name=expr
	        [, col_name=expr] ... ]
	
		Or:
	
		INSERT [LOW_PRIORITY | DELAYED | HIGH_PRIORITY] [IGNORE]
	    [INTO] tbl_name
	    [PARTITION (partition_name,...)]
	    SET col_name={expr | DEFAULT}, ...
	    [ ON DUPLICATE KEY UPDATE
	      col_name=expr
	        [, col_name=expr] ... ]
	
		Or:
	
		INSERT [LOW_PRIORITY | HIGH_PRIORITY] [IGNORE]
	    [INTO] tbl_name
	    [PARTITION (partition_name,...)]
	    [(col_name,...)]
	    SELECT ...
	    [ ON DUPLICATE KEY UPDATE
	      col_name=expr
        [, col_name=expr] ... ]
        mysql> insert user value (33333333,'ddd');
		mysql> insert into user value (333333,'ddd');
		mysql> insert user values (3333,'ddd');
     * insert into user(id,name) valueS(1111,'dig'),
     * (1111,  'dig'), (1111,'dig') ,(1111,'dig');
	 * @param sql
	 * @return
	 */
	private static String convertInsertSQL(String sql){
		try{
			MySqlStatementParser parser = new MySqlStatementParser(sql);	 
			SQLStatement statement = parser.parseStatement();
			MySqlInsertStatement insert = (MySqlInsertStatement)statement; 
	        String tableName = StringUtil.removeBackquote(insert.getTableName().getSimpleName());
	        if(!isGlobalTable(tableName))
				return sql;
	        if(!isInnerColExist(tableName))
	        	return sql;
	        	
	        if(insert.getQuery() != null)	// insert into tab select 
	        	return sql;
	        
	        StringBuilder sb = new StringBuilder(200)	// 指定初始容量可以提高性能
			.append("insert into ").append(tableName);
	        
	        List columns = insert.getColumns();
	        
	        int idx = -1;	
	        int colSize = -1;
	        
	        if(columns == null || columns.size() <= 0){ // insert 没有带列名：insert into t values(xxx,xxx)
	        	String columnsList = tableColumsMap.get(tableName.toUpperCase());
	        	if(StringUtils.isNotBlank(columnsList)){ //"id,name,_mycat_op_time"
	        		//newSQL = "insert into t(id,name,_mycat_op_time)";
	        		// 构建一个虚拟newSQL来寻找 内部列的索引位置
	        		String newSQL = "insert into " + tableName + "(" + columnsList + ")";
	        		MySqlStatementParser newParser = new MySqlStatementParser(newSQL);	 
	        		SQLStatement newStatement = newParser.parseStatement();
	        		MySqlInsertStatement newInsert = (MySqlInsertStatement)newStatement; 
	        		List newColumns = newInsert.getColumns();
	        		for(int i = 0; i < newColumns.size(); i++) {
						String column = StringUtil.removeBackquote(newInsert.getColumns().get(i).toString());
						if(column.equalsIgnoreCase(GLOBAL_TABLE_MYCAT_COLUMN))
							idx = i;	// 找到 内部列的索引位置
					}
	        		colSize = newColumns.size();
	        		sb.append("(").append(columnsList).append(")");
	        	}else{	// tableName 是全局表，但是 tableColumsMap 没有其对应的列list，这种情况不应该存在
		        	LOGGER.warn("you'd better do not use 'insert into t values(a,b)' Syntax (without column list) on global table, "
    				+ "If you do. Then you must make sure inner column '_mycat_op_time' is last column of global table: " 
    				+ tableName + " in all database. Good luck. ^_^");
		        	// 我们假定 内部列位于表中所有列的最后，后面我们在values 子句的最后 给他附加上时间戳
	        	}
	        }else{	// insert 语句带有 列名
	        	sb.append("(");
				for(int i = 0; i < columns.size(); i++) {
					if(i < columns.size() - 1)
						sb.append(columns.get(i).toString()).append(",");
					else
						sb.append(columns.get(i).toString());
					String column = StringUtil.removeBackquote(insert.getColumns().get(i).toString());
					if(column.equalsIgnoreCase(GLOBAL_TABLE_MYCAT_COLUMN))
						idx = i;
				}
				if(idx <= -1)
					sb.append(",").append(GLOBAL_TABLE_MYCAT_COLUMN);
				sb.append(")");
				colSize = columns.size();
	        }
			
			sb.append(" values");
			List vcl = insert.getValuesList();
			if(vcl != null && vcl.size() > 1){	// 批量insert
				for(int j=0; j valuse = insert.getValues().getValues();
				appendValues(valuse, sb, idx, colSize);
			}
			
			List dku = insert.getDuplicateKeyUpdate();
			if(dku != null && dku.size() > 0){
				sb.append(" on duplicate key update ");
				for(int i=0; i columns = insert.getColumns();
//		System.out.println(columns.size());
		
		String sql = "alter table t add colomn name varchar(30)";
		System.out.println(handleDDLSQL(sql));
	}
	
	private static boolean isInnerColExist(String tableName){
		if(innerColumnNotExist.size() > 0){
			for(SQLQueryResult> map : innerColumnNotExist){
				if(map != null && tableName.equalsIgnoreCase(map.getTableName())){
					StringBuilder warnStr = new StringBuilder(map.getDataNode())
							.append(".").append(tableName).append(" inner column: ")
							.append(GLOBAL_TABLE_MYCAT_COLUMN)
							.append(" is not exist.");
					LOGGER.warn(warnStr.toString());
					return false;	// tableName 全局表没有内部列
				}
			}
		}
		return true;	// tableName 有内部列
	}
	
	private static StringBuilder appendValues(List valuse, StringBuilder sb, int idx, int colSize){
		int size = valuse.size();
		if(size < colSize)
			size = colSize;
		
		sb.append("(");
		for(int i = 0; i < size; i++) {
    		if(i < size - 1){
    			if(i != idx)
    				sb.append(valuse.get(i).toString()).append(",");
    			else
    				sb.append(operationTimestamp).append(",");
    		}else{
    			if(i != idx){
    				sb.append(valuse.get(i).toString());
    			}else{
    				sb.append(operationTimestamp);
    			}
    		}
		}
		if(idx <= -1)
    	   sb.append(",").append(operationTimestamp);
		return sb.append(")");
	}
	
	/**
	 * UPDATE [LOW_PRIORITY] [IGNORE] table_reference
    	SET col_name1={expr1|DEFAULT} [, col_name2={expr2|DEFAULT}] ...
    	[WHERE where_condition]
    	[ORDER BY ...]
    	[LIMIT row_count]

		Multiple-table syntax:

		UPDATE [LOW_PRIORITY] [IGNORE] table_references
    	SET col_name1={expr1|DEFAULT} [, col_name2={expr2|DEFAULT}] ...
    	[WHERE where_condition]
    	
    	update user, tuser set user.name='dddd',tuser.pwd='aaa' 
    	where user.id=2 and tuser.id=0;
	 * @param sql update tuser set pwd='aaa', name='digdee' where id=0;
	 * @return
	 */
	public static String convertUpdateSQL(String sql){
		try{
			MySqlStatementParser parser = new MySqlStatementParser(sql);	 
			SQLStatement stmt = parser.parseStatement();
			if (!(stmt instanceof MySqlUpdateStatement)){
				return sql;
			}
			MySqlUpdateStatement update = (MySqlUpdateStatement)stmt;
			SQLTableSource ts = update.getTableSource();
			if(ts != null && ts.toString().contains(",")){
				System.out.println(ts.toString());
				LOGGER.warn("Do not support Multiple-table udpate syntax...");
				return sql;
			}
			
			String tableName = StringUtil.removeBackquote(update.getTableName().getSimpleName());
	        if(!isGlobalTable(tableName))
				return sql;
	        if(!isInnerColExist(tableName))
	        	return sql;		// 没有内部列
	        
			StringBuilder sb = new StringBuilder(150);
			
			SQLExpr se = update.getWhere();
			// where中有子查询： update company set name='com' where id in (select id from xxx where ...)
			if(se instanceof SQLInSubQueryExpr){
				// return sql;
				int idx = sql.toUpperCase().indexOf(" SET ") + 5;
				sb.append(sql.substring(0, idx)).append(GLOBAL_TABLE_MYCAT_COLUMN)
				.append("=").append(operationTimestamp)
				.append(",").append(sql.substring(idx));
				return sb.toString();
			}
			String where = null;
			if(update.getWhere() != null)
				where = update.getWhere().toString();
			
			SQLOrderBy orderBy = update.getOrderBy();
			SQLLimit limit = update.getLimit();
			
			sb.append("update ").append(tableName).append(" set ");
			List items = update.getItems();
			boolean flag = false;
			for(int i=0; i 0){
				sb.append(" order by ");
				for(int i=0; i schemaMap = config.getSchemas();
		SchemaConfig schemaMconfig = null;
		for(String key : schemaMap.keySet()){
			if(schemaMap.get(key) != null){
				schemaMconfig = schemaMap.get(key);
				Map tableMap = schemaMconfig.getTables();
				if(tableMap != null){
					for(String k : tableMap.keySet()){
						TableConfig table = tableMap.get(k);
						if(table != null && table.isGlobalTable()){
							globalTableMap.put(table.getName().toUpperCase(), table);
						}
					}
				}
			}
		}
	}
	
	/**
	 * 重新获得table 的列list
	 * @param tableName
	 */
	private static void reGetColumnsForTable(String tableName){
		MycatConfig config = MycatServer.getInstance().getConfig();
		if(globalTableMap != null 
						&& globalTableMap.get(tableName.toUpperCase()) != null){
			
			TableConfig tableConfig = globalTableMap.get(tableName.toUpperCase());
			if(tableConfig == null || isInnerColumnCheckFinished != 1)	// consistencyCheck 在运行中
				return;
			
			String nodeName = tableConfig.getDataNodes().get(0);
			
			Map map = config.getDataNodes();
			for(String k2 : map.keySet()){
				PhysicalDBNode dBnode = map.get(k2);
				if(nodeName.equals(dBnode.getName())){
					PhysicalDBPool pool = dBnode.getDbPool();
					List dsList = (List)pool.genAllDataSources();
					for(PhysicalDatasource ds : dsList){
						if(ds instanceof MySQLDataSource){
							MySQLDataSource mds = (MySQLDataSource)dsList.get(0);
							MySQLConsistencyChecker checker = 
									new MySQLConsistencyChecker(mds, tableConfig.getName());
							checker.checkInnerColumnExist();
							return; // 运行一次就行了，不需要像consistencyCheck那样每个db都运行一次
						}
					}
				}
			}
		}
	}
	
	public static void consistencyCheck() {
		MycatConfig config = MycatServer.getInstance().getConfig();
		for(String key : globalTableMap.keySet()){
			TableConfig table = globalTableMap.get(key);
			//
dataNodeList = table.getDataNodes(); // 记录本次已经执行的datanode // 多个 datanode 对应到同一个 PhysicalDatasource 只执行一次 Map executedMap = new HashMap<>(); for(String nodeName : dataNodeList){ Map map = config.getDataNodes(); for(String k2 : map.keySet()){ // PhysicalDBNode dBnode = map.get(k2); if(nodeName.equals(dBnode.getName())){ // dn1,dn2,dn3 PhysicalDBPool pool = dBnode.getDbPool(); Collection allDS = pool.genAllDataSources(); for(PhysicalDatasource pds : allDS){ if(pds instanceof MySQLDataSource){ MySQLDataSource mds = (MySQLDataSource)pds; if(executedMap.get(pds.getName()) == null){ MySQLConsistencyChecker checker = new MySQLConsistencyChecker(mds, table.getName()); isInnerColumnCheckFinished = 0; checker.checkInnerColumnExist(); while(isInnerColumnCheckFinished <= 0){ LOGGER.debug("isInnerColumnCheckFinished:" + isInnerColumnCheckFinished); try { TimeUnit.SECONDS.sleep(1); } catch (InterruptedException e) { LOGGER.warn(e.getMessage()); } } LOGGER.debug("isInnerColumnCheckFinished:" + isInnerColumnCheckFinished); // 一种 check 完成之后，再进行另一种 check checker = new MySQLConsistencyChecker(mds, table.getName()); isColumnCountCheckFinished = 0; checker.checkRecordCout(); while(isColumnCountCheckFinished <= 0){ LOGGER.debug("isColumnCountCheckFinished:" + isColumnCountCheckFinished); try { TimeUnit.SECONDS.sleep(1); } catch (InterruptedException e) { LOGGER.warn(e.getMessage()); } } LOGGER.debug("isColumnCountCheckFinished:" + isColumnCountCheckFinished); checker = new MySQLConsistencyChecker(mds, table.getName()); checker.checkMaxTimeStamp(); executedMap.put(pds.getName(), nodeName); } } } } } } } } /** * 每次处理 一种 check 的结果，不会交叉同时处理 多种不同 check 的结果 * @param list * @return */ public static List>> finished(List>> list){ lock.lock(); try{ //[{"dataNode":"db3","result":{"count(*)":"1"},"success":true,"tableName":"COMPANY"}] LOGGER.debug("list:::::::::::" + JSON.toJSONString(list)); for(SQLQueryResult> map : list){ Map row = map.getResult(); if(row != null){ if(row.containsKey(GlobalTableUtil.MAX_COLUMN)){ LOGGER.info(map.getDataNode() + "." + map.getTableName() + "." + GlobalTableUtil.MAX_COLUMN + ": "+ map.getResult().get(GlobalTableUtil.MAX_COLUMN)); } if(row.containsKey(GlobalTableUtil.COUNT_COLUMN)){ LOGGER.info(map.getDataNode() + "." + map.getTableName() + "." + GlobalTableUtil.COUNT_COLUMN + ": "+ map.getResult().get(GlobalTableUtil.COUNT_COLUMN)); } if(row.containsKey(GlobalTableUtil.INNER_COLUMN)){ String columnsList = null; try{ if(StringUtils.isNotBlank(row.get(GlobalTableUtil.INNER_COLUMN))) columnsList = row.get(GlobalTableUtil.INNER_COLUMN); // id,name,_mycat_op_time LOGGER.debug("columnsList: " + columnsList); }catch(Exception e){ LOGGER.warn(row.get(GlobalTableUtil.INNER_COLUMN) + ", " + e.getMessage()); }finally{ if(columnsList == null || columnsList.indexOf(GlobalTableUtil.GLOBAL_TABLE_MYCAT_COLUMN) == -1){ LOGGER.warn(map.getDataNode() + "." + map.getTableName() + " inner column: " + GlobalTableUtil.GLOBAL_TABLE_MYCAT_COLUMN + " is not exist."); if(StringUtils.isNotBlank(map.getTableName())){ for(SQLQueryResult> sqr : innerColumnNotExist){ String name = map.getTableName(); String node = map.getDataNode(); if(name != null && !name.equalsIgnoreCase(sqr.getTableName()) || node != null && !node.equalsIgnoreCase(sqr.getDataNode())){ innerColumnNotExist.add(map); } } } }else{ LOGGER.debug("columnsList: " + columnsList); // COMPANY -> "id,name,_mycat_op_time"，获得了全局表的所有列，并且知道了全局表是否有内部列 // 所有列，在 insert into t values(xx,yy) 语法中需要用到 tableColumsMap.put(map.getTableName().toUpperCase(), columnsList); } // isInnerColumnCheckFinished = 1; } } } } }finally{ isInnerColumnCheckFinished = 1; isColumnCountCheckFinished = 1; lock.unlock(); } return list; } private static boolean isGlobalTable(String tableName){ if(globalTableMap != null && globalTableMap.size() > 0){ return globalTableMap.get(tableName.toUpperCase()) != null; } return false; } public static Map getTableColumsMap() { return tableColumsMap; } } ================================================ FILE: src/main/java/io/mycat/server/interceptor/impl/StatSqlInterceptor.java ================================================ package io.mycat.server.interceptor.impl; import io.mycat.server.interceptor.SQLInterceptor; public class StatSqlInterceptor implements SQLInterceptor { @Override public String interceptSQL(String sql, int sqlType) { // TODO Auto-generated method stub final int atype = sqlType; final String sqls = DefaultSqlInterceptor.processEscape(sql); return sql; } } ================================================ FILE: src/main/java/io/mycat/server/interceptor/impl/StatisticsSqlInterceptor.java ================================================ package io.mycat.server.interceptor.impl; import java.io.FileWriter; import java.io.IOException; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.HashMap; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.config.model.SystemConfig; import io.mycat.server.interceptor.SQLInterceptor; import io.mycat.server.parser.ServerParse; import java.io.File; public class StatisticsSqlInterceptor implements SQLInterceptor { private final class StatisticsSqlRunner implements Runnable { private int sqltype = 0; private String sqls = ""; public StatisticsSqlRunner(int sqltype, String sqls) { this.sqltype = sqltype; this.sqls = sqls; } public void run() { try { SystemConfig sysconfig = MycatServer.getInstance().getConfig().getSystem(); String sqlInterceptorType = sysconfig.getSqlInterceptorType(); String sqlInterceptorFile = sysconfig.getSqlInterceptorFile(); String[] sqlInterceptorTypes = sqlInterceptorType.split(","); for (String type : sqlInterceptorTypes) { if (StatisticsSqlInterceptor.parseType(type.toUpperCase()) == sqltype) { switch (sqltype) { case ServerParse.SELECT: StatisticsSqlInterceptor.appendFile(sqlInterceptorFile, "SELECT:" + sqls + ""); break; case ServerParse.UPDATE: StatisticsSqlInterceptor.appendFile(sqlInterceptorFile, "UPDATE:" + sqls); break; case ServerParse.INSERT: StatisticsSqlInterceptor.appendFile(sqlInterceptorFile, "INSERT:" + sqls); break; case ServerParse.DELETE: StatisticsSqlInterceptor.appendFile(sqlInterceptorFile, "DELETE:" + sqls); break; default: break; } } } } catch (Exception e) { LOGGER.error("interceptSQL error:" + e.getMessage(),e); } } } private static final Logger LOGGER = LoggerFactory.getLogger(StatisticsSqlInterceptor.class); private static Map typeMap = new HashMap(); static { typeMap.put("SELECT", 7); typeMap.put("UPDATE", 11); typeMap.put("INSERT", 4); typeMap.put("DELETE", 3); } public static int parseType(String type) { return typeMap.get(type); } /** * 方法追加文件：使用FileWriter */ private static synchronized void appendFile(String fileName, String content) { Calendar calendar = Calendar.getInstance(); DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); String dayFile = dateFormat.format(calendar.getTime()); FileWriter writer = null; try { String newFileName = fileName; //打开一个写文件器，构造函数中的第二个参数true表示以追加形式写文件 String[] title = newFileName.split("\\."); if (title.length == 2) { newFileName = title[0] + dayFile + "." + title[1]; } File file = new File(newFileName); if (!file.exists()) { file.createNewFile(); } writer = new FileWriter(file, true); String newContent = content.replaceAll("[\\t\\n\\r]", "") + System.getProperty("line.separator"); writer.write(newContent); writer.flush(); } catch (IOException e) { LOGGER.error("appendFile error:" + e.getMessage(),e); } finally { if(writer != null ){ try { writer.close(); } catch (IOException e) { LOGGER.error("close file error:" + e.getMessage(),e); } } } } /** * interceptSQL , * type :insert,delete,update,select * exectime:xxx ms * log content : select:select 1 from table,exectime:100ms,shared:1 * etc */ @Override public String interceptSQL(String sql, int sqlType) { LOGGER.debug("sql interceptSQL:"); final int sqltype = sqlType; final String sqls = DefaultSqlInterceptor.processEscape(sql); MycatServer.getInstance().getBusinessExecutor() .execute(new StatisticsSqlRunner(sqltype, sqls)); return sql; } } ================================================ FILE: src/main/java/io/mycat/server/parser/ServerParse.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.parser; import java.util.regex.Matcher; import java.util.regex.Pattern; import io.mycat.route.parser.util.ParseUtil; /** * @author mycat */ public final class ServerParse { public static final int OTHER = -1; public static final int BEGIN = 1; public static final int COMMIT = 2; public static final int DELETE = 3; public static final int INSERT = 4; public static final int REPLACE = 5; public static final int ROLLBACK = 6; public static final int SELECT = 7; public static final int SET = 8; public static final int SHOW = 9; public static final int START = 10; public static final int UPDATE = 11; public static final int KILL = 12; public static final int SAVEPOINT = 13; public static final int USE = 14; public static final int EXPLAIN = 15; public static final int EXPLAIN2 = 151; public static final int KILL_QUERY = 16; public static final int HELP = 17; public static final int MYSQL_CMD_COMMENT = 18; public static final int MYSQL_COMMENT = 19; public static final int CALL = 20; public static final int DESCRIBE = 21; public static final int LOCK = 22; public static final int UNLOCK = 23; public static final int LOAD_DATA_INFILE_SQL = 99; public static final int DDL = 100; public static final int COMMAND = 101; public static final String COM_FIELD_LIST_FLAG="select @@command "; public static final int MIGRATE = 203; private static final Pattern pattern = Pattern.compile("(load)+\\s+(data)+\\s+\\w*\\s*(infile)+",Pattern.CASE_INSENSITIVE); private static final Pattern callPattern = Pattern.compile("\\w*\\;\\s*\\s*(call)+\\s+\\w*\\s*",Pattern.CASE_INSENSITIVE); public static int parse(String stmt) { int length = stmt.length(); for (int i = 0; i < length; ++i) { switch (stmt.charAt(i)) { case ' ': case '\t': case '\r': case '\n': continue; case '/': // such as /*!40101 SET character_set_client = @saved_cs_client // */; if (i == 0 && stmt.charAt(1) == '*' && stmt.charAt(2) == '!' && stmt.charAt(length - 2) == '*' && stmt.charAt(length - 1) == '/') { return MYSQL_CMD_COMMENT; } case '#': i = ParseUtil.comment(stmt, i); if (i + 1 == length) { return MYSQL_COMMENT; } continue; case 'A': case 'a': return aCheck(stmt, i); case 'B': case 'b': return beginCheck(stmt, i); case 'C': case 'c': return commitOrCallCheckOrCreate(stmt, i); case 'D': case 'd': return deleteOrdCheck(stmt, i); case 'E': case 'e': return explainCheck(stmt, i); case 'I': case 'i': return insertCheck(stmt, i); case 'M': case 'm': return migrateCheck(stmt, i); case 'R': case 'r': return rCheck(stmt, i); case 'S': case 's': return sCheck(stmt, i); case 'T': case 't': return tCheck(stmt, i); case 'U': case 'u': return uCheck(stmt, i); case 'K': case 'k': return killCheck(stmt, i); case 'H': case 'h': return helpCheck(stmt, i); case 'L': case 'l': return lCheck(stmt, i); default: return OTHER; } } return OTHER; } static int lCheck(String stmt, int offset) { if (stmt.length() > offset + 3) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); if ((c1 == 'O' || c1 == 'o') && (c2 == 'A' || c2 == 'a') && (c3 == 'D' || c3 == 'd')) { Matcher matcher = pattern.matcher(stmt); return matcher.find() ? LOAD_DATA_INFILE_SQL : OTHER; } else if ((c1 == 'O' || c1 == 'o') && (c2 == 'C' || c2 == 'c') && (c3 == 'K' || c3 == 'k')){ return LOCK; } } return OTHER; } private static int migrateCheck(String stmt, int offset) { if (stmt.length() > offset + 7) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); if ((c1 == 'i' || c1 == 'I') && (c2 == 'g' || c2 == 'G') && (c3 == 'r' || c3 == 'R') && (c4 == 'a' || c4 == 'A') && (c5 == 't' || c5 == 'T') && (c6 == 'e' || c6 == 'E')) { return MIGRATE; } } return OTHER; } //truncate private static int tCheck(String stmt, int offset) { if (stmt.length() > offset + 7) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); char c7 = stmt.charAt(++offset); char c8 = stmt.charAt(++offset); if ((c1 == 'R' || c1 == 'r') && (c2 == 'U' || c2 == 'u') && (c3 == 'N' || c3 == 'n') && (c4 == 'C' || c4 == 'c') && (c5 == 'A' || c5 == 'a') && (c6 == 'T' || c6 == 't') && (c7 == 'E' || c7 == 'e') && (c8 == ' ' || c8 == '\t' || c8 == '\r' || c8 == '\n')) { return DDL; } } return OTHER; } //alter table/view/... private static int aCheck(String stmt, int offset) { if (stmt.length() > offset + 4) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); if ((c1 == 'L' || c1 == 'l') && (c2 == 'T' || c2 == 't') && (c3 == 'E' || c3 == 'e') && (c4 == 'R' || c4 == 'r') && (c5 == ' ' || c5 == '\t' || c5 == '\r' || c5 == '\n')) { return DDL; } } return OTHER; } //create table/view/... private static int createCheck(String stmt, int offset) { if (stmt.length() > offset + 5) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); if ((c1 == 'R' || c1 == 'r') && (c2 == 'E' || c2 == 'e') && (c3 == 'A' || c3 == 'a') && (c4 == 'T' || c4 == 't') && (c5 == 'E' || c5 == 'e') && (c6 == ' ' || c6 == '\t' || c6 == '\r' || c6 == '\n')) { return DDL; } } return OTHER; } //drop private static int dropCheck(String stmt, int offset) { if (stmt.length() > offset + 3) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); if ((c1 == 'R' || c1 == 'r') && (c2 == 'O' || c2 == 'o') && (c3 == 'P' || c3 == 'p') && (c4 == ' ' || c4 == '\t' || c4 == '\r' || c4 == '\n')) { return DDL; } } return OTHER; } // delete or drop static int deleteOrdCheck(String stmt, int offset){ int sqlType = OTHER; switch (stmt.charAt((offset + 1))) { case 'E': case 'e': sqlType = dCheck(stmt, offset); break; case 'R': case 'r': sqlType = dropCheck(stmt, offset); break; default: sqlType = OTHER; } return sqlType; } // HELP' ' static int helpCheck(String stmt, int offset) { if (stmt.length() > offset + "ELP ".length()) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); if ((c1 == 'E' || c1 == 'e') && (c2 == 'L' || c2 == 'l') && (c3 == 'P' || c3 == 'p')) { return (offset << 8) | HELP; } } return OTHER; } // EXPLAIN' ' static int explainCheck(String stmt, int offset) { if (stmt.length() > offset + "XPLAIN ".length()) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); char c7 = stmt.charAt(++offset); if ((c1 == 'X' || c1 == 'x') && (c2 == 'P' || c2 == 'p') && (c3 == 'L' || c3 == 'l') && (c4 == 'A' || c4 == 'a') && (c5 == 'I' || c5 == 'i') && (c6 == 'N' || c6 == 'n') && (c7 == ' ' || c7 == '\t' || c7 == '\r' || c7 == '\n')) { return (offset << 8) | EXPLAIN; } } if(stmt != null && stmt.toLowerCase().startsWith("explain2")){ return (offset << 8) | EXPLAIN2; } return OTHER; } // KILL' ' static int killCheck(String stmt, int offset) { if (stmt.length() > offset + "ILL ".length()) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); if ((c1 == 'I' || c1 == 'i') && (c2 == 'L' || c2 == 'l') && (c3 == 'L' || c3 == 'l') && (c4 == ' ' || c4 == '\t' || c4 == '\r' || c4 == '\n')) { while (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case ' ': case '\t': case '\r': case '\n': continue; case 'Q': case 'q': return killQueryCheck(stmt, offset); default: return (offset << 8) | KILL; } } return OTHER; } } return OTHER; } // KILL QUERY' ' static int killQueryCheck(String stmt, int offset) { if (stmt.length() > offset + "UERY ".length()) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); if ((c1 == 'U' || c1 == 'u') && (c2 == 'E' || c2 == 'e') && (c3 == 'R' || c3 == 'r') && (c4 == 'Y' || c4 == 'y') && (c5 == ' ' || c5 == '\t' || c5 == '\r' || c5 == '\n')) { while (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case ' ': case '\t': case '\r': case '\n': continue; default: return (offset << 8) | KILL_QUERY; } } return OTHER; } } return OTHER; } // BEGIN static int beginCheck(String stmt, int offset) { if (stmt.length() > offset + 4) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); if ((c1 == 'E' || c1 == 'e') && (c2 == 'G' || c2 == 'g') && (c3 == 'I' || c3 == 'i') && (c4 == 'N' || c4 == 'n') && (stmt.length() == ++offset || ParseUtil.isEOF(stmt .charAt(offset)))) { return BEGIN; } } return OTHER; } // COMMIT static int commitCheck(String stmt, int offset) { if (stmt.length() > offset + 5) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); if ((c1 == 'O' || c1 == 'o') && (c2 == 'M' || c2 == 'm') && (c3 == 'M' || c3 == 'm') && (c4 == 'I' || c4 == 'i') && (c5 == 'T' || c5 == 't') && (stmt.length() == ++offset || ParseUtil.isEOF(stmt .charAt(offset)))) { return COMMIT; } } return OTHER; } // CALL static int callCheck(String stmt, int offset) { if (stmt.length() > offset + 3) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); if ((c1 == 'A' || c1 == 'a') && (c2 == 'L' || c2 == 'l') && (c3 == 'L' || c3 == 'l')) { return CALL; } } return OTHER; } static int commitOrCallCheckOrCreate(String stmt, int offset) { int sqlType = OTHER; switch (stmt.charAt((offset + 1))) { case 'O': case 'o': sqlType = commitCheck(stmt, offset); break; case 'A': case 'a': sqlType = callCheck(stmt, offset); break; case 'R': case 'r': sqlType = createCheck(stmt, offset); break; default: sqlType = OTHER; } return sqlType; } // DESCRIBE or desc or DELETE' ' static int dCheck(String stmt, int offset) { if (stmt.length() > offset + 4) { int res = describeCheck(stmt, offset); if (res == DESCRIBE) { return res; } } // continue check if (stmt.length() > offset + 6) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); if ((c1 == 'E' || c1 == 'e') && (c2 == 'L' || c2 == 'l') && (c3 == 'E' || c3 == 'e') && (c4 == 'T' || c4 == 't') && (c5 == 'E' || c5 == 'e') && (c6 == ' ' || c6 == '\t' || c6 == '\r' || c6 == '\n')) { return DELETE; } } return OTHER; } // DESCRIBE' ' 或 desc' ' static int describeCheck(String stmt, int offset) { //desc if (stmt.length() > offset + 4) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); if ((c1 == 'E' || c1 == 'e') && (c2 == 'S' || c2 == 's') && (c3 == 'C' || c3 == 'c') && (c4 == ' ' || c4 == '\t' || c4 == '\r' || c4 == '\n')) { return DESCRIBE; } //describe if (stmt.length() > offset + 4) { char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); char c7 = stmt.charAt(++offset); char c8 = stmt.charAt(++offset); if ((c1 == 'E' || c1 == 'e') && (c2 == 'S' || c2 == 's') && (c3 == 'C' || c3 == 'c') && (c4 == 'R' || c4 == 'r') && (c5 == 'I' || c5 == 'i') && (c6 == 'B' || c6 == 'b') && (c7 == 'E' || c7 == 'e') && (c8 == ' ' || c8 == '\t' || c8 == '\r' || c8 == '\n')) { return DESCRIBE; } } } return OTHER; } // INSERT' ' static int insertCheck(String stmt, int offset) { if (stmt.length() > offset + 6) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); if ((c1 == 'N' || c1 == 'n') && (c2 == 'S' || c2 == 's') && (c3 == 'E' || c3 == 'e') && (c4 == 'R' || c4 == 'r') && (c5 == 'T' || c5 == 't') && (c6 == ' ' || c6 == '\t' || c6 == '\r' || c6 == '\n')) { return INSERT; } } return OTHER; } static int rCheck(String stmt, int offset) { if (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case 'E': case 'e': return replaceCheck(stmt, offset); case 'O': case 'o': return rollabckCheck(stmt, offset); default: return OTHER; } } return OTHER; } // REPLACE' ' static int replaceCheck(String stmt, int offset) { if (stmt.length() > offset + 6) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); if ((c1 == 'P' || c1 == 'p') && (c2 == 'L' || c2 == 'l') && (c3 == 'A' || c3 == 'a') && (c4 == 'C' || c4 == 'c') && (c5 == 'E' || c5 == 'e') && (c6 == ' ' || c6 == '\t' || c6 == '\r' || c6 == '\n')) { return REPLACE; } } return OTHER; } // ROLLBACK static int rollabckCheck(String stmt, int offset) { if (stmt.length() > offset + 6) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); if ((c1 == 'L' || c1 == 'l') && (c2 == 'L' || c2 == 'l') && (c3 == 'B' || c3 == 'b') && (c4 == 'A' || c4 == 'a') && (c5 == 'C' || c5 == 'c') && (c6 == 'K' || c6 == 'k') && (stmt.length() == ++offset || ParseUtil.isEOF(stmt .charAt(offset)))) { return ROLLBACK; } } return OTHER; } static int sCheck(String stmt, int offset) { if (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case 'A': case 'a': return savepointCheck(stmt, offset); case 'E': case 'e': return seCheck(stmt, offset); case 'H': case 'h': return showCheck(stmt, offset); case 'T': case 't': return startCheck(stmt, offset); default: return OTHER; } } return OTHER; } // SAVEPOINT static int savepointCheck(String stmt, int offset) { if (stmt.length() > offset + 8) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); char c7 = stmt.charAt(++offset); char c8 = stmt.charAt(++offset); if ((c1 == 'V' || c1 == 'v') && (c2 == 'E' || c2 == 'e') && (c3 == 'P' || c3 == 'p') && (c4 == 'O' || c4 == 'o') && (c5 == 'I' || c5 == 'i') && (c6 == 'N' || c6 == 'n') && (c7 == 'T' || c7 == 't') && (c8 == ' ' || c8 == '\t' || c8 == '\r' || c8 == '\n')) { return SAVEPOINT; } } return OTHER; } static int seCheck(String stmt, int offset) { if (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case 'L': case 'l': return selectCheck(stmt, offset); case 'T': case 't': if (stmt.length() > ++offset) { //支持一下语句 // /*!mycat: sql=SELECT * FROM test where id=99 */set @pin=1; // call p_test(@pin,@pout); // select @pout; if(stmt.startsWith("/*!mycat:")||stmt.startsWith("/*#mycat:")||stmt.startsWith("/*mycat:")||stmt.startsWith("/* mycat:")) { Matcher matcher = callPattern.matcher(stmt); if (matcher.find()) { return CALL; } } char c = stmt.charAt(offset); if (c == ' ' || c == '\r' || c == '\n' || c == '\t' || c == '/' || c == '#') { return (offset << 8) | SET; } } return OTHER; default: return OTHER; } } return OTHER; } // SELECT' ' static int selectCheck(String stmt, int offset) { // SELECT @@command ,对应mysql协议是0x04 if(stmt.startsWith(COM_FIELD_LIST_FLAG)) { return COMMAND; } if (stmt.length() > offset + 4) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); if ((c1 == 'E' || c1 == 'e') && (c2 == 'C' || c2 == 'c') && (c3 == 'T' || c3 == 't') && (c4 == ' ' || c4 == '\t' || c4 == '\r' || c4 == '\n' || c4 == '/' || c4 == '#' || c4 == '*')) { return (offset << 8) | SELECT; } } return OTHER; } // SHOW' ' static int showCheck(String stmt, int offset) { if (stmt.length() > offset + 3) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); if ((c1 == 'O' || c1 == 'o') && (c2 == 'W' || c2 == 'w') && (c3 == ' ' || c3 == '\t' || c3 == '\r' || c3 == '\n')) { return (offset << 8) | SHOW; } } return OTHER; } // START' ' static int startCheck(String stmt, int offset) { if (stmt.length() > offset + 4) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); if ((c1 == 'A' || c1 == 'a') && (c2 == 'R' || c2 == 'r') && (c3 == 'T' || c3 == 't') && (c4 == ' ' || c4 == '\t' || c4 == '\r' || c4 == '\n')) { return (offset << 8) | START; } } return OTHER; } // UPDATE' ' | USE' ' static int uCheck(String stmt, int offset) { if (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case 'P': case 'p': if (stmt.length() > offset + 5) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); if ((c1 == 'D' || c1 == 'd') && (c2 == 'A' || c2 == 'a') && (c3 == 'T' || c3 == 't') && (c4 == 'E' || c4 == 'e') && (c5 == ' ' || c5 == '\t' || c5 == '\r' || c5 == '\n')) { return UPDATE; } } break; case 'S': case 's': if (stmt.length() > offset + 2) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); if ((c1 == 'E' || c1 == 'e') && (c2 == ' ' || c2 == '\t' || c2 == '\r' || c2 == '\n')) { return (offset << 8) | USE; } } break; case 'N': case 'n': if (stmt.length() > offset + 5) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); if ((c1 == 'L' || c1 == 'l') && (c2 == 'O' || c2 == 'o') && (c3 == 'C' || c3 == 'c') && (c4 == 'K' || c4 == 'k') && (c5 == ' ' || c5 == '\t' || c5 == '\r' || c5 == '\n')) { return UNLOCK; } } break; default: return OTHER; } } return OTHER; } } ================================================ FILE: src/main/java/io/mycat/server/parser/ServerParseSelect.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.parser; import io.mycat.route.parser.util.CharTypes; import io.mycat.route.parser.util.ParseUtil; /** * @author mycat */ public final class ServerParseSelect { public static final int OTHER = -1; public static final int VERSION_COMMENT = 1; public static final int DATABASE = 2; public static final int USER = 3; public static final int LAST_INSERT_ID = 4; public static final int IDENTITY = 5; public static final int VERSION = 6; public static final int SESSION_INCREMENT = 7; public static final int SESSION_ISOLATION = 8; public static final int SELECT_VAR_ALL = 9; public static final int SESSION_TX_READ_ONLY = 10; private static final char[] _VERSION_COMMENT = "VERSION_COMMENT" .toCharArray(); private static final char[] _IDENTITY = "IDENTITY".toCharArray(); private static final char[] _LAST_INSERT_ID = "LAST_INSERT_ID" .toCharArray(); private static final char[] _DATABASE = "DATABASE()".toCharArray(); private static final char[] _CURRENT_USER = "CURRENT_USER()".toCharArray(); public static int parse(String stmt, int offset) { int i = offset; for (; i < stmt.length(); ++i) { switch (stmt.charAt(i)) { case ' ': continue; case '/': case '#': i = ParseUtil.comment(stmt, i); continue; case '@': return select2Check(stmt, i); case 'D': case 'd': return databaseCheck(stmt, i); case 'L': case 'l': return lastInsertCheck(stmt, i); case 'U': case 'u': return userCheck(stmt, i); case 'C': case 'c': return currentUserCheck(stmt, i); case 'V': case 'v': return versionCheck(stmt, i); default: return OTHER; } } return OTHER; } /** * SELECT @@session.auto_increment_increment * * @param stmt * @param offset * @return */ private static int sessionVarCheck(String stmt, int offset) { String s = stmt.substring(offset).toLowerCase(); if (s.startsWith("session.auto_increment_increment")) { if(s.contains("@@")) { return SELECT_VAR_ALL; } return SESSION_INCREMENT; } else if (s .startsWith("session.tx_isolation")) { return SESSION_ISOLATION; } else if (s .startsWith("session.tx_read_only")) { return SESSION_TX_READ_ONLY; } else { return OTHER; } } // SELECT VERSION private static int versionCheck(String stmt, int offset) { if (stmt.length() > offset + "ERSION".length()) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); if ((c1 == 'E' || c1 == 'e') && (c2 == 'R' || c2 == 'r') && (c3 == 'S' || c3 == 's') && (c4 == 'I' || c4 == 'i') && (c5 == 'O' || c5 == 'o') && (c6 == 'N' || c6 == 'n')) { while (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case ' ': case '\t': case '\r': case '\n': continue; case '(': return versionParenthesisCheck(stmt, offset); default: return OTHER; } } } } return OTHER; } // SELECT VERSION ( private static int versionParenthesisCheck(String stmt, int offset) { while (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case ' ': case '\t': case '\r': case '\n': continue; case ')': while (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case ' ': case '\t': case '\r': case '\n': continue; default: return OTHER; } } return VERSION; default: return OTHER; } } return OTHER; } /** * SELECT LAST_INSERT_ID() AS id, * * @param offset * index of 'i', offset == stmt.length() is possible * @return index of ','. return stmt.length() is possible. -1 if not alias */ private static int skipAlias(String stmt, int offset) { offset = ParseUtil.move(stmt, offset, 0); if (offset >= stmt.length()) { return offset; } switch (stmt.charAt(offset)) { case '\'': return skipString(stmt, offset); case '"': return skipString2(stmt, offset); case '`': return skipIdentifierEscape(stmt, offset); default: if (CharTypes.isIdentifierChar(stmt.charAt(offset))) { for (; offset < stmt.length() && CharTypes.isIdentifierChar(stmt.charAt(offset)); ++offset) { } return offset; } } return -1; } /** * `abc`d * * @param offset * index of first ` * @return index of 'd'. return stmt.length() is possible. -1 if string * invalid */ private static int skipIdentifierEscape(String stmt, int offset) { for (++offset; offset < stmt.length(); ++offset) { if (stmt.charAt(offset) == '`' && (++offset >= stmt.length() || stmt.charAt(offset) != '`')) { return offset; } } return -1; } /** * "abc"d * * @param offset * index of first " * @return index of 'd'. return stmt.length() is possible. -1 if string * invalid */ private static int skipString2(String stmt, int offset) { int state = 0; for (++offset; offset < stmt.length(); ++offset) { char c = stmt.charAt(offset); switch (state) { case 0: switch (c) { case '\\': state = 1; break; case '"': state = 2; break; } break; case 1: state = 0; break; case 2: switch (c) { case '"': state = 0; break; default: return offset; } break; } } if (offset == stmt.length() && state == 2) { return stmt.length(); } return -1; } /** * 'abc'd * * @param offset * index of first ' * @return index of 'd'. return stmt.length() is possible. -1 if string * invalid */ private static int skipString(String stmt, int offset) { int state = 0; for (++offset; offset < stmt.length(); ++offset) { char c = stmt.charAt(offset); switch (state) { case 0: switch (c) { case '\\': state = 1; break; case '\'': state = 2; break; } break; case 1: state = 0; break; case 2: switch (c) { case '\'': state = 0; break; default: return offset; } break; } } if (offset == stmt.length() && state == 2) { return stmt.length(); } return -1; } /** * SELECT LAST_INSERT_ID() AS id * * @param offset * index of first ' ' after LAST_INSERT_ID(), offset == * stmt.length() is possible * @return index of 'i'. return stmt.length() is possible */ public static int skipAs(String stmt, int offset) { offset = ParseUtil.move(stmt, offset, 0); if (stmt.length() > offset + "AS".length() && (stmt.charAt(offset) == 'A' || stmt.charAt(offset) == 'a') && (stmt.charAt(offset + 1) == 'S' || stmt.charAt(offset + 1) == 's') && (stmt.charAt(offset + 2) == ' ' || stmt.charAt(offset + 2) == '\r' || stmt.charAt(offset + 2) == '\n' || stmt.charAt(offset + 2) == '\t' || stmt.charAt(offset + 2) == '/' || stmt .charAt(offset + 2) == '#')) { offset = ParseUtil.move(stmt, offset + 2, 0); } return offset; } /** * @param offset * stmt.charAt(offset) == first 'L' OR 'l' * @return index after LAST_INSERT_ID(), might equals to length. -1 if not * LAST_INSERT_ID */ public static int indexAfterLastInsertIdFunc(String stmt, int offset) { if (stmt.length() >= offset + "LAST_INSERT_ID()".length() && ParseUtil.compare(stmt, offset, _LAST_INSERT_ID)) { offset = ParseUtil.move(stmt, offset + _LAST_INSERT_ID.length, 0); if (offset + 1 < stmt.length() && stmt.charAt(offset) == '(') { offset = ParseUtil.move(stmt, offset + 1, 0); if (offset < stmt.length() && stmt.charAt(offset) == ')') { return ++offset; } } } return -1; } /** * @param offset * stmt.charAt(offset) == first '`' OR 'i' OR 'I' OR '\'' OR '"' * @return index after identity or `identity` or "identity" or 'identity', * might equals to length. -1 if not identity or `identity` or * "identity" or 'identity' */ public static int indexAfterIdentity(String stmt, int offset) { char first = stmt.charAt(offset); switch (first) { case '`': case '\'': case '"': if (stmt.length() < offset + "identity".length() + 2) { return -1; } if (stmt.charAt(offset + "identity".length() + 1) != first) { return -1; } ++offset; break; case 'i': case 'I': if (stmt.length() < offset + "identity".length()) { return -1; } break; default: return -1; } if (ParseUtil.compare(stmt, offset, _IDENTITY)) { offset += _IDENTITY.length; switch (first) { case '`': case '\'': case '"': return ++offset; } return offset; } return -1; } /** * SELECT LAST_INSERT_ID() */ static int lastInsertCheck(String stmt, int offset) { offset = indexAfterLastInsertIdFunc(stmt, offset); if (offset < 0) { return OTHER; } offset = skipAs(stmt, offset); offset = skipAlias(stmt, offset); if (offset < 0) { return OTHER; } offset = ParseUtil.move(stmt, offset, 0); if (offset < stmt.length()) { return OTHER; } return LAST_INSERT_ID; } /** * select @@identity
* select @@identiTy aS iD */ static int identityCheck(String stmt, int offset) { offset = indexAfterIdentity(stmt, offset); if (offset < 0) { return OTHER; } offset = skipAs(stmt, offset); offset = skipAlias(stmt, offset); if (offset < 0) { return OTHER; } offset = ParseUtil.move(stmt, offset, 0); if (offset < stmt.length()) { return OTHER; } return IDENTITY; } static int select2Check(String stmt, int offset) { if (stmt.length() > ++offset && stmt.charAt(offset) == '@' && stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case 'V': case 'v': return versionCommentCheck(stmt, offset); case 'i': case 'I': return identityCheck(stmt, offset); case 's': case 'S': return sessionVarCheck(stmt, offset); default: return OTHER; } } return OTHER; } /** * SELECT DATABASE() */ static int databaseCheck(String stmt, int offset) { int length = offset + _DATABASE.length; if (stmt.length() >= length && ParseUtil.compare(stmt, offset, _DATABASE)) { if (stmt.length() > length && stmt.charAt(length) != ' ') { return OTHER; } else { return DATABASE; } } return OTHER; } /** * SELECT USER() */ static int userCheck(String stmt, int offset) { if (stmt.length() > offset + 5) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); if ((c1 == 'S' || c1 == 's') && (c2 == 'E' || c2 == 'e') && (c3 == 'R' || c3 == 'r') && (c4 == '(') && (c5 == ')') && (stmt.length() == ++offset || ParseUtil.isEOF(stmt .charAt(offset)))) { return USER; } } return OTHER; } /** * SELECT USER() */ static int currentUserCheck(String stmt, int offset) { int length = offset + _CURRENT_USER.length; if (stmt.length() >= length && ParseUtil.compare(stmt, offset, _CURRENT_USER)) { if (stmt.length() > length && stmt.charAt(length) != ' ') { return OTHER; } return USER; } return OTHER; } /** * SELECT @@VERSION_COMMENT */ static int versionCommentCheck(String stmt, int offset) { int length = offset + _VERSION_COMMENT.length; if (stmt.length() >= length && ParseUtil.compare(stmt, offset, _VERSION_COMMENT)) { if (stmt.length() > length && stmt.charAt(length) != ' ') { return OTHER; } else { return VERSION_COMMENT; } } return OTHER; } } ================================================ FILE: src/main/java/io/mycat/server/parser/ServerParseSet.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.parser; import io.mycat.route.parser.util.ParseUtil; /** * @author mycat */ public final class ServerParseSet { public static final int OTHER = -1; public static final int AUTOCOMMIT_ON = 1; public static final int AUTOCOMMIT_OFF = 2; public static final int TX_READ_UNCOMMITTED = 3; public static final int TX_READ_COMMITTED = 4; public static final int TX_REPEATED_READ = 5; public static final int TX_SERIALIZABLE = 6; public static final int NAMES = 7; public static final int CHARACTER_SET_CLIENT = 8; public static final int CHARACTER_SET_CONNECTION = 9; public static final int CHARACTER_SET_RESULTS = 10; public static final int XA_FLAG_ON = 11; public static final int XA_FLAG_OFF = 12; public static final int TX_READONLY = 13; public static final int TX_READWRITE = 14; public static final int SQL_SELECT_LIMIT = 15; public static int parse(String stmt, int offset) { if (stmt.contains("@@session.")){ stmt = stmt.replaceAll("@@session.",""); } int i = offset; for (; i < stmt.length(); i++) { switch (stmt.charAt(i)) { case ' ': case '\r': case '\n': case '\t': continue; case '/': case '#': i = ParseUtil.comment(stmt, i); continue; case 'A': case 'a': return autocommit(stmt, i); case 'C': case 'c': return characterSet(stmt, i, 0); case 'N': case 'n': return names(stmt, i); case 'S': case 's': return parseS(stmt, i); case 'T': case 't': return transaction(stmt, i); case 'X': case 'x': return xaFlag(stmt, i); default: return OTHER; } } return OTHER; } // set xa=1 private static int xaFlag(String stmt, int offset) { if (stmt.length() > offset + 1) { char c1 = stmt.charAt(++offset); if ((c1 == 'A' || c1 == 'a')) { while (stmt.length() >= ++ offset) { switch (stmt.charAt(offset)) { case ' ': case '\r': case '\n': case '\t': continue; case '=': int value = autocommitValue(stmt, offset); if (value == AUTOCOMMIT_ON) { return XA_FLAG_ON; } else if (value == AUTOCOMMIT_OFF) { return XA_FLAG_OFF; } else { return OTHER; } default: return OTHER; } } } } return OTHER; } // SET AUTOCOMMIT(' '=) private static int autocommit(String stmt, int offset) { if (stmt.length() > offset + 9) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); char c7 = stmt.charAt(++offset); char c8 = stmt.charAt(++offset); char c9 = stmt.charAt(++offset); if ((c1 == 'U' || c1 == 'u') && (c2 == 'T' || c2 == 't') && (c3 == 'O' || c3 == 'o') && (c4 == 'C' || c4 == 'c') && (c5 == 'O' || c5 == 'o') && (c6 == 'M' || c6 == 'm') && (c7 == 'M' || c7 == 'm') && (c8 == 'I' || c8 == 'i') && (c9 == 'T' || c9 == 't')) { while (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case ' ': case '\r': case '\n': case '\t': continue; case '=': return autocommitValue(stmt, offset); default: return OTHER; } } } } return OTHER; } private static int autocommitValue(String stmt, int offset) { for (;;) { offset++; if (stmt.length() <= offset) { return OTHER; } switch (stmt.charAt(offset)) { case ' ': case '\r': case '\n': case '\t': continue; case '1': if (stmt.length() == ++offset || ParseUtil.isEOF(stmt.charAt(offset))) { return AUTOCOMMIT_ON; } else { return OTHER; } case '0': if (stmt.length() == ++offset || ParseUtil.isEOF(stmt.charAt(offset))) { return AUTOCOMMIT_OFF; } else { return OTHER; } case 'O': case 'o': return autocommitOn(stmt, offset); default: return OTHER; } } } private static int autocommitOn(String stmt, int offset) { if (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case 'N': case 'n': if (stmt.length() == ++offset || ParseUtil.isEOF(stmt.charAt(offset))) { return AUTOCOMMIT_ON; } else { return OTHER; } case 'F': case 'f': return autocommitOff(stmt, offset); default: return OTHER; } } return OTHER; } // SET AUTOCOMMIT = OFF private static int autocommitOff(String stmt, int offset) { if (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case 'F': case 'f': if (stmt.length() == ++offset || ParseUtil.isEOF(stmt.charAt(offset))) { return AUTOCOMMIT_OFF; } else { return OTHER; } default: return OTHER; } } return OTHER; } // SET NAMES' ' private static int names(String stmt, int offset) { if (stmt.length() > offset + 5) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); if ((c1 == 'A' || c1 == 'a') && (c2 == 'M' || c2 == 'm') && (c3 == 'E' || c3 == 'e') && (c4 == 'S' || c4 == 's') && stmt.charAt(++offset) == ' ') { return (offset << 8) | NAMES; } } return OTHER; } // SET CHARACTER_SET_ private static int characterSet(String stmt, int offset, int depth) { if (stmt.length() > offset + 14) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); char c7 = stmt.charAt(++offset); char c8 = stmt.charAt(++offset); char c9 = stmt.charAt(++offset); char c10 = stmt.charAt(++offset); char c11 = stmt.charAt(++offset); char c12 = stmt.charAt(++offset); char c13 = stmt.charAt(++offset); char c14 = stmt.charAt(++offset); if ((c1 == 'H' || c1 == 'h') && (c2 == 'A' || c2 == 'a') && (c3 == 'R' || c3 == 'r') && (c4 == 'A' || c4 == 'a') && (c5 == 'C' || c5 == 'c') && (c6 == 'T' || c6 == 't') && (c7 == 'E' || c7 == 'e') && (c8 == 'R' || c8 == 'r') && (c9 == '_') && (c10 == 'S' || c10 == 's') && (c11 == 'E' || c11 == 'e') && (c12 == 'T' || c12 == 't') && (c13 == '_')) { switch (c14) { case 'R': case 'r': return characterSetResults(stmt, offset); case 'C': case 'c': return characterSetC(stmt, offset); default: return OTHER; } } } return OTHER; } // SET CHARACTER_SET_RESULTS = private static int characterSetResults(String stmt, int offset) { if (stmt.length() > offset + 6) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); if ((c1 == 'E' || c1 == 'e') && (c2 == 'S' || c2 == 's') && (c3 == 'U' || c3 == 'u') && (c4 == 'L' || c4 == 'l') && (c5 == 'T' || c5 == 't') && (c6 == 'S' || c6 == 's')) { while (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case ' ': case '\r': case '\n': case '\t': continue; case '=': while (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case ' ': case '\r': case '\n': case '\t': continue; default: return (offset << 8) | CHARACTER_SET_RESULTS; } } return OTHER; default: return OTHER; } } } } return OTHER; } // SET CHARACTER_SET_C private static int characterSetC(String stmt, int offset) { if (stmt.length() > offset + 1) { char c1 = stmt.charAt(++offset); switch (c1) { case 'o': case 'O': return characterSetConnection(stmt, offset); case 'l': case 'L': return characterSetClient(stmt, offset); default: return OTHER; } } return OTHER; } // SET CHARACTER_SET_CONNECTION = private static int characterSetConnection(String stmt, int offset) { if (stmt.length() > offset + 8) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); char c7 = stmt.charAt(++offset); char c8 = stmt.charAt(++offset); if ((c1 == 'N' || c1 == 'n') && (c2 == 'N' || c2 == 'n') && (c3 == 'E' || c3 == 'e') && (c4 == 'C' || c4 == 'c') && (c5 == 'T' || c5 == 't') && (c6 == 'I' || c6 == 'i') && (c7 == 'O' || c7 == 'o') && (c8 == 'N' || c8 == 'n')) { while (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case ' ': case '\r': case '\n': case '\t': continue; case '=': while (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case ' ': case '\r': case '\n': case '\t': continue; default: return (offset << 8) | CHARACTER_SET_CONNECTION; } } return OTHER; default: return OTHER; } } } } return OTHER; } // SET CHARACTER_SET_CLIENT = private static int characterSetClient(String stmt, int offset) { if (stmt.length() > offset + 4) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); if ((c1 == 'I' || c1 == 'i') && (c2 == 'E' || c2 == 'e') && (c3 == 'N' || c3 == 'n') && (c4 == 'T' || c4 == 't')) { while (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case ' ': case '\r': case '\n': case '\t': continue; case '=': while (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case ' ': case '\r': case '\n': case '\t': continue; default: return (offset << 8) | CHARACTER_SET_CLIENT; } } return OTHER; default: return OTHER; } } } } return OTHER; } private static int parseS(String stmt, int offset) { if (stmt.length() > offset + 1) { char c1 = stmt.charAt(offset + 1); if(c1=='E' || c1=='e'){ return session(stmt, offset); } if(c1=='Q' || c1=='q'){ return sqlSelectLimit(stmt, offset); } } return OTHER; } // SET SQL_SELECT_LIMIT=N|DEFAULT private static int sqlSelectLimit(String stmt, int offset) { // SET SQL_SELECT_LIMIT=N|DEFAULT if (stmt.length() > offset + 15) { String var = stmt.substring(offset, offset+16); if(var.equalsIgnoreCase("SQL_SELECT_LIMIT")) { return SQL_SELECT_LIMIT; } } return OTHER; } // SET SESSION' ' // SET SQL_SELECT_LIMIT=N|DEFAULT private static int session(String stmt, int offset) { if (stmt.length() > offset + 7) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); if ((c1 == 'E' || c1 == 'e') && (c2 == 'S' || c2 == 's') && (c3 == 'S' || c3 == 's') && (c4 == 'I' || c4 == 'i') && (c5 == 'O' || c5 == 'o') && (c6 == 'N' || c6 == 'n') && stmt.charAt(++offset) == ' ') { // for (;;) { while (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case ' ': case '\r': case '\n': case '\t': continue; case 'T': case 't': return transaction(stmt, offset); default: return OTHER; } } // return OTHER; // } } } return OTHER; } // SET [SESSION] TRANSACTION ISOLATION LEVEL private static int transaction(String stmt, int offset) { if (stmt.length() > offset + 11) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); char c7 = stmt.charAt(++offset); char c8 = stmt.charAt(++offset); char c9 = stmt.charAt(++offset); char c10 = stmt.charAt(++offset); if ((c1 == 'R' || c1 == 'r') && (c2 == 'A' || c2 == 'a') && (c3 == 'N' || c3 == 'n') && (c4 == 'S' || c4 == 's') && (c5 == 'A' || c5 == 'a') && (c6 == 'C' || c6 == 'c') && (c7 == 'T' || c7 == 't') && (c8 == 'I' || c8 == 'i') && (c9 == 'O' || c9 == 'o') && (c10 == 'N' || c10 == 'n') && stmt.charAt(++offset) == ' ') { // for (;;) { while (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case ' ': case '\r': case '\n': case '\t': continue; case 'I': case 'i': return isolation(stmt, offset); case 'R': case 'r': return transactionRead(stmt, offset); default: return OTHER; } } // return OTHER; // } } } return OTHER; } // SET [SESSION] TRANSACTION READ WRITE|ONLY private static int transactionRead(String stmt, int offset) { if (stmt.length() > offset + 4) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); if ((c1 == 'E' || c1 == 'e') && (c2 == 'A' || c2 == 'a') && (c3 == 'D' || c3 == 'd')) { // for (;;) { while (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case ' ': case '\r': case '\n': case '\t': continue; case 'W': case 'w': return transactionReadWrite(stmt, offset); case 'O': case 'o': return transactionReadOnly(stmt, offset); default: return OTHER; } } // return OTHER; // } } } return OTHER; } // SET [SESSION] TRANSACTION READ WRITE private static int transactionReadWrite(String stmt, int offset) { if (stmt.length() > offset + 4) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); if ((c1 == 'R' || c1 == 'r') && (c2 == 'I' || c2 == 'i') && (c3 == 'T' || c3 == 't') && (c4 == 'E' || c4 == 'e') && (stmt.length() == ++offset || ParseUtil.isEOF(stmt .charAt(offset)))) { return TX_READWRITE; } } return OTHER; } // SET [SESSION] TRANSACTION READ ONLY private static int transactionReadOnly(String stmt, int offset) { if (stmt.length() > offset + 3) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); if ((c1 == 'N' || c1 == 'n') && (c2 == 'L' || c2 == 'l') && (c3 == 'Y' || c3 == 'y') && (stmt.length() == ++offset || ParseUtil.isEOF(stmt .charAt(offset)))) { return TX_READONLY; } } return OTHER; } // SET [SESSION] TRANSACTION ISOLATION LEVEL private static int isolation(String stmt, int offset) { if (stmt.length() > offset + 9) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); char c7 = stmt.charAt(++offset); char c8 = stmt.charAt(++offset); if ((c1 == 'S' || c1 == 's') && (c2 == 'O' || c2 == 'o') && (c3 == 'L' || c3 == 'l') && (c4 == 'A' || c4 == 'a') && (c5 == 'T' || c5 == 't') && (c6 == 'I' || c6 == 'i') && (c7 == 'O' || c7 == 'o') && (c8 == 'N' || c8 == 'n') && stmt.charAt(++offset) == ' ') { // for (;;) { while (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case ' ': case '\r': case '\n': case '\t': continue; case 'L': case 'l': return level(stmt, offset); default: return OTHER; } } // return OTHER; // } } } return OTHER; } // SET [SESSION] TRANSACTION ISOLATION LEVEL' ' private static int level(String stmt, int offset) { if (stmt.length() > offset + 5) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); if ((c1 == 'E' || c1 == 'e') && (c2 == 'V' || c2 == 'v') && (c3 == 'E' || c3 == 'e') && (c4 == 'L' || c4 == 'l') && stmt.charAt(++offset) == ' ') { // for (;;) { while (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case ' ': case '\r': case '\n': case '\t': continue; case 'R': case 'r': return rCheck(stmt, offset); case 'S': case 's': return serializable(stmt, offset); default: return OTHER; } } // return OTHER; // } } } return OTHER; } // SET [SESSION] TRANSACTION ISOLATION LEVEL SERIALIZABLE private static int serializable(String stmt, int offset) { if (stmt.length() > offset + 11) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); char c7 = stmt.charAt(++offset); char c8 = stmt.charAt(++offset); char c9 = stmt.charAt(++offset); char c10 = stmt.charAt(++offset); char c11 = stmt.charAt(++offset); if ((c1 == 'E' || c1 == 'e') && (c2 == 'R' || c2 == 'r') && (c3 == 'I' || c3 == 'i') && (c4 == 'A' || c4 == 'a') && (c5 == 'L' || c5 == 'l') && (c6 == 'I' || c6 == 'i') && (c7 == 'Z' || c7 == 'z') && (c8 == 'A' || c8 == 'a') && (c9 == 'B' || c9 == 'b') && (c10 == 'L' || c10 == 'l') && (c11 == 'E' || c11 == 'e') && (stmt.length() == ++offset || ParseUtil.isEOF(stmt .charAt(offset)))) { return TX_SERIALIZABLE; } } return OTHER; } // READ' '|REPEATABLE private static int rCheck(String stmt, int offset) { if (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case 'E': case 'e': return eCheck(stmt, offset); default: return OTHER; } } return OTHER; } // READ' '|REPEATABLE private static int eCheck(String stmt, int offset) { if (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case 'A': case 'a': return aCheck(stmt, offset); case 'P': case 'p': return pCheck(stmt, offset); default: return OTHER; } } return OTHER; } // READ' ' private static int aCheck(String stmt, int offset) { if (stmt.length() > offset + 2) { char c1 = stmt.charAt(++offset); if ((c1 == 'D' || c1 == 'd') && stmt.charAt(++offset) == ' ') { // for (;;) { while (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case ' ': case '\r': case '\n': case '\t': continue; case 'C': case 'c': return committed(stmt, offset); case 'U': case 'u': return uncommitted(stmt, offset); default: return OTHER; } } // return OTHER; // } } } return OTHER; } // COMMITTED private static int committed(String stmt, int offset) { if (stmt.length() > offset + 8) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); char c7 = stmt.charAt(++offset); char c8 = stmt.charAt(++offset); if ((c1 == 'O' || c1 == 'o') && (c2 == 'M' || c2 == 'm') && (c3 == 'M' || c3 == 'm') && (c4 == 'I' || c4 == 'i') && (c5 == 'T' || c5 == 't') && (c6 == 'T' || c6 == 't') && (c7 == 'E' || c7 == 'e') && (c8 == 'D' || c8 == 'd') && (stmt.length() == ++offset || ParseUtil.isEOF(stmt .charAt(offset)))) { return TX_READ_COMMITTED; } } return OTHER; } // UNCOMMITTED private static int uncommitted(String stmt, int offset) { if (stmt.length() > offset + 10) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); char c7 = stmt.charAt(++offset); char c8 = stmt.charAt(++offset); char c9 = stmt.charAt(++offset); char c10 = stmt.charAt(++offset); if ((c1 == 'N' || c1 == 'n') && (c2 == 'C' || c2 == 'c') && (c3 == 'O' || c3 == 'o') && (c4 == 'M' || c4 == 'm') && (c5 == 'M' || c5 == 'm') && (c6 == 'I' || c6 == 'i') && (c7 == 'T' || c7 == 't') && (c8 == 'T' || c8 == 't') && (c9 == 'E' || c9 == 'e') && (c10 == 'D' || c10 == 'd') && (stmt.length() == ++offset || ParseUtil.isEOF(stmt .charAt(offset)))) { return TX_READ_UNCOMMITTED; } } return OTHER; } // REPEATABLE private static int pCheck(String stmt, int offset) { if (stmt.length() > offset + 8) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); char c7 = stmt.charAt(++offset); if ((c1 == 'E' || c1 == 'e') && (c2 == 'A' || c2 == 'a') && (c3 == 'T' || c3 == 't') && (c4 == 'A' || c4 == 'a') && (c5 == 'B' || c5 == 'b') && (c6 == 'L' || c6 == 'l') && (c7 == 'E' || c7 == 'e') && stmt.charAt(++offset) == ' ') { // for (;;) { while (stmt.length() > ++offset) { switch (stmt.charAt(offset)) { case ' ': case '\r': case '\n': case '\t': continue; case 'R': case 'r': return prCheck(stmt, offset); default: return OTHER; } } // return OTHER; // } } } return OTHER; } // READ private static int prCheck(String stmt, int offset) { if (stmt.length() > offset + 3) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); if ((c1 == 'E' || c1 == 'e') && (c2 == 'A' || c2 == 'a') && (c3 == 'D' || c3 == 'd') && (stmt.length() == ++offset || ParseUtil.isEOF(stmt .charAt(offset)))) { return TX_REPEATED_READ; } } return OTHER; } } ================================================ FILE: src/main/java/io/mycat/server/parser/ServerParseShow.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.parser; import java.util.regex.Pattern; import io.mycat.route.parser.util.ParseUtil; /** * @author mycat */ public final class ServerParseShow { public static final int OTHER = -1; public static final int DATABASES = 1; public static final int DATASOURCES = 2; public static final int MYCAT_STATUS = 3; public static final int MYCAT_CLUSTER = 4; public static final int TABLES = 5; public static final int FULLTABLES =65; private static final Pattern SHOWTablePattern = Pattern.compile( "^\\s*(SHOW)(\\s+(full|all))?\\s+(TABLES)(\\s+(FROM)\\s+([a-zA-Z_0-9]+))?((\\s+(like)\\s+'((. *)*)'\\s*)|(\\s+(where)\\s+((. *)*)\\s*))?", Pattern.CASE_INSENSITIVE); public static int parse(String stmt, int offset) { int i = offset; for (; i < stmt.length(); i++) { switch (stmt.charAt(i)) { case ' ': continue; case 'F': case 'f': return fullTableCheck(stmt,i) ; case '/': case '#': i = ParseUtil.comment(stmt, i); continue; case 'M': case 'm': return mycatCheck(stmt, i); case 'D': case 'd': return dataCheck(stmt, i); case 'T': case 't': return tableCheck(stmt, i); default: return OTHER; } } return OTHER; } // SHOW MYCAT_ static int mycatCheck(String stmt, int offset) { if (stmt.length() > offset + "ycat_?".length()) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); if ((c1 == 'Y' || c1 == 'y') && (c2 == 'C' || c2 == 'c') && (c3 == 'A' || c3 == 'a') && (c4 == 'T' || c4 == 't') && (c5 == '_')) { switch (stmt.charAt(++offset)) { case 'S': case 's': return showMyCatStatus(stmt, offset); case 'C': case 'c': return showMyCatCluster(stmt, offset); default: return OTHER; } } } return OTHER; } // SHOW MyCat_STATUS static int showMyCatStatus(String stmt, int offset) { if (stmt.length() > offset + "tatus".length()) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); if ((c1 == 't' || c1 == 'T') && (c2 == 'a' || c2 == 'A') && (c3 == 't' || c3 == 'T') && (c4 == 'u' || c4 == 'U') && (c5 == 's' || c5 == 'S') && (stmt.length() == ++offset || ParseUtil.isEOF(stmt .charAt(offset)))) { return MYCAT_STATUS; } } return OTHER; } // SHOW MyCat_CLUSTER static int showMyCatCluster(String stmt, int offset) { if (stmt.length() > offset + "luster".length()) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); if ((c1 == 'L' || c1 == 'l') && (c2 == 'U' || c2 == 'u') && (c3 == 'S' || c3 == 's') && (c4 == 'T' || c4 == 't') && (c5 == 'E' || c5 == 'e') && (c6 == 'R' || c6 == 'r') && (stmt.length() == ++offset || ParseUtil.isEOF(stmt .charAt(offset)))) { return MYCAT_CLUSTER; } } return OTHER; } // SHOW DATA static int dataCheck(String stmt, int offset) { if (stmt.length() > offset + "ata?".length()) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); if ((c1 == 'A' || c1 == 'a') && (c2 == 'T' || c2 == 't') && (c3 == 'A' || c3 == 'a')) { switch (stmt.charAt(++offset)) { case 'B': case 'b': return showDatabases(stmt, offset); case 'S': case 's': return showDataSources(stmt, offset); default: return OTHER; } } } return OTHER; } public static int fullTableCheck(String stmt, int offset) { if (SHOWTablePattern.matcher(stmt).matches()) { return FULLTABLES; } return OTHER; } // SHOW TABLE public static int tableCheck(String stmt, int offset) { if (SHOWTablePattern.matcher(stmt).matches()) { return TABLES; } return OTHER; } // SHOW DATABASES static int showDatabases(String stmt, int offset) { if (stmt.length() > offset + "ases".length()) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); if ((c1 == 'A' || c1 == 'a') && (c2 == 'S' || c2 == 's') && (c3 == 'E' || c3 == 'e') && (c4 == 'S' || c4 == 's') && (stmt.length() == ++offset || ParseUtil.isEOF(stmt .charAt(offset)))) { return DATABASES; } } return OTHER; } // SHOW DATASOURCES static int showDataSources(String stmt, int offset) { if (stmt.length() > offset + "ources".length()) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); if ((c1 == 'O' || c1 == 'o') && (c2 == 'U' || c2 == 'u') && (c3 == 'R' || c3 == 'r') && (c4 == 'C' || c4 == 'c') && (c5 == 'E' || c5 == 'e') && (c6 == 'S' || c6 == 's') && (stmt.length() == ++offset || ParseUtil.isEOF(stmt .charAt(offset)))) { return DATASOURCES; } } return OTHER; } } ================================================ FILE: src/main/java/io/mycat/server/parser/ServerParseStart.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.parser; import io.mycat.route.parser.util.ParseUtil; /** * @author mycat */ public final class ServerParseStart { public static final int OTHER = -1; public static final int TRANSACTION = 1; public static int parse(String stmt, int offset) { int i = offset; for (; i < stmt.length(); i++) { switch (stmt.charAt(i)) { case ' ': continue; case '/': case '#': i = ParseUtil.comment(stmt, i); continue; case 'T': case 't': return transactionCheck(stmt, i); default: return OTHER; } } return OTHER; } // START TRANSACTION static int transactionCheck(String stmt, int offset) { if (stmt.length() > offset + "ransaction".length()) { char c1 = stmt.charAt(++offset); char c2 = stmt.charAt(++offset); char c3 = stmt.charAt(++offset); char c4 = stmt.charAt(++offset); char c5 = stmt.charAt(++offset); char c6 = stmt.charAt(++offset); char c7 = stmt.charAt(++offset); char c8 = stmt.charAt(++offset); char c9 = stmt.charAt(++offset); char c10 = stmt.charAt(++offset); if ((c1 == 'R' || c1 == 'r') && (c2 == 'A' || c2 == 'a') && (c3 == 'N' || c3 == 'n') && (c4 == 'S' || c4 == 's') && (c5 == 'A' || c5 == 'a') && (c6 == 'C' || c6 == 'c') && (c7 == 'T' || c7 == 't') && (c8 == 'I' || c8 == 'i') && (c9 == 'O' || c9 == 'o') && (c10 == 'N' || c10 == 'n') && (stmt.length() == ++offset || ParseUtil.isEOF(stmt.charAt(offset)))) { return TRANSACTION; } } return OTHER; } } ================================================ FILE: src/main/java/io/mycat/server/response/CharacterSet.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import static io.mycat.server.parser.ServerParseSet.CHARACTER_SET_CLIENT; import static io.mycat.server.parser.ServerParseSet.CHARACTER_SET_CONNECTION; import static io.mycat.server.parser.ServerParseSet.CHARACTER_SET_RESULTS; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.config.ErrorCode; import io.mycat.net.mysql.OkPacket; import io.mycat.server.ServerConnection; import io.mycat.server.parser.ServerParseSet; import io.mycat.util.SetIgnoreUtil; import io.mycat.util.SplitUtil; /** * 字符集属性设置 */ /** * @author mycat */ public class CharacterSet { private static final Logger logger = LoggerFactory.getLogger(CharacterSet.class); public static void response(String stmt, ServerConnection c, int rs) { if (-1 == stmt.indexOf(',')) { /* 单个属性 */ oneSetResponse(stmt, c, rs); } else { /* 多个属性 ,但是只关注CHARACTER_SET_RESULTS，CHARACTER_SET_CONNECTION */ multiSetResponse(stmt, c, rs); } } private static void oneSetResponse(String stmt, ServerConnection c, int rs) { if ((rs & 0xff) == CHARACTER_SET_CLIENT) { /* 忽略client属性设置 */ c.write(c.writeToBuffer(OkPacket.OK, c.allocate())); } else { String charset = stmt.substring(rs >>> 8).trim(); if (charset.endsWith(";")) { /* 结尾为 ; 标识符 */ charset = charset.substring(0, charset.length() - 1); } if (charset.startsWith("'") || charset.startsWith("`")) { /* 与mysql保持一致，引号里的字符集不做trim操作 */ charset = charset.substring(1, charset.length() - 1); } // 设置字符集 setCharset(charset, c); } } private static void multiSetResponse(String stmt, ServerConnection c, int rs) { String charResult = "null"; String charConnection = "null"; String[] sqlList = SplitUtil.split(stmt, ',', false); // check first switch (rs & 0xff) { case CHARACTER_SET_RESULTS: charResult = sqlList[0].substring(rs >>> 8).trim(); break; case CHARACTER_SET_CONNECTION: charConnection = sqlList[0].substring(rs >>> 8).trim(); break; } // check remaining for (int i = 1; i < sqlList.length; i++) { String sql = new StringBuilder("set ").append(sqlList[i]).toString(); if ((i + 1 == sqlList.length) && sql.endsWith(";")) { /* 去掉末尾的 ‘;’ */ sql = sql.substring(0, sql.length() - 1); } int rs2 = ServerParseSet.parse(sql, "set".length()); switch (rs2 & 0xff) { case CHARACTER_SET_RESULTS: charResult = sql.substring(rs2 >>> 8).trim(); break; case CHARACTER_SET_CONNECTION: charConnection = sql.substring(rs2 >>> 8).trim(); break; case CHARACTER_SET_CLIENT: break; default: boolean ignore = SetIgnoreUtil.isIgnoreStmt( sql ); if ( !ignore ) { StringBuilder s = new StringBuilder(); logger.warn(s.append(c).append(sql).append(" is not executed").toString()); } } } if (charResult.startsWith("'") || charResult.startsWith("`")) { charResult = charResult.substring(1, charResult.length() - 1); } if (charConnection.startsWith("'") || charConnection.startsWith("`")) { charConnection = charConnection.substring(1, charConnection.length() - 1); } // 如果其中一个为null，则以另一个为准。 if ("null".equalsIgnoreCase(charResult)) { setCharset(charConnection, c); return; } if ("null".equalsIgnoreCase(charConnection)) { setCharset(charResult, c); return; } if (charConnection.equalsIgnoreCase(charResult)) { setCharset(charConnection, c); } else { StringBuilder sb = new StringBuilder(); sb.append("charset is not consistent:[connection=").append(charConnection); sb.append(",results=").append(charResult).append(']'); c.writeErrMessage(ErrorCode.ER_UNKNOWN_CHARACTER_SET, sb.toString()); } } private static void setCharset(String charset, ServerConnection c) { if ("null".equalsIgnoreCase(charset)) { /* 忽略字符集为null的属性设置 */ c.write(c.writeToBuffer(OkPacket.OK, c.allocate())); } else if (c.setCharset(charset)) { c.write(c.writeToBuffer(OkPacket.OK, c.allocate())); } else { try { if (c.setCharsetIndex(Integer.parseInt(charset))) { c.write(c.writeToBuffer(OkPacket.OK, c.allocate())); } else { c.writeErrMessage(ErrorCode.ER_UNKNOWN_CHARACTER_SET, "Unknown charset :" + charset); } } catch (RuntimeException e) { c.writeErrMessage(ErrorCode.ER_UNKNOWN_CHARACTER_SET, "Unknown charset :" + charset); } } } } ================================================ FILE: src/main/java/io/mycat/server/response/ClientHeartbeatResponse.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import java.nio.ByteBuffer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.ServerConnection; import io.mycat.util.StringUtil; /** * 构建select 1语句返回包： * 客户端连接池给mycat发送select 1或select 1 from dual心跳语句,mycat拦截，用该类直接返回包 */ public class ClientHeartbeatResponse { private static byte[] RESPONSE_DATA = null; public static void response(ServerConnection c) { if (RESPONSE_DATA == null) { synchronized (ClientHeartbeatResponse.class) { if (RESPONSE_DATA == null) { RESPONSE_DATA = buildPacket(c); } } } ByteBuffer buffer = c.allocate(); buffer.put(RESPONSE_DATA); c.write(buffer); } private static byte[] buildPacket(ServerConnection c) { byte packetId = 0; final int FIELD_COUNT = 1; final String FIELD_NAME = "1"; final String FIELD_VALUE = "1"; ByteBuffer buffer = c.allocate(); // write header ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); header.packetId = ++packetId; buffer = header.write(buffer, c, false); FieldPacket field = PacketUtil.getField(FIELD_NAME, Fields.FIELD_TYPE_VAR_STRING); field.packetId = ++packetId; buffer = field.write(buffer, c, false); EOFPacket eof = new EOFPacket(); eof.packetId = ++packetId; buffer = eof.write(buffer, c, false); // write body RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(FIELD_VALUE, c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c, false); EOFPacket rowEof = new EOFPacket(); rowEof.packetId = ++packetId; buffer = rowEof.write(buffer, c, false); buffer.flip(); byte[] data = new byte[buffer.limit()]; buffer.get(data); // recycle buffer c.recycle(buffer); return data; } } ================================================ FILE: src/main/java/io/mycat/server/response/Heartbeat.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.config.ErrorCode; import io.mycat.net.mysql.ErrorPacket; import io.mycat.net.mysql.HeartbeatPacket; import io.mycat.net.mysql.OkPacket; import io.mycat.server.ServerConnection; import io.mycat.util.TimeUtil; /** * @author mycat */ public class Heartbeat { private static final Logger HEARTBEAT = LoggerFactory.getLogger("heartbeat"); public static void response(ServerConnection c, byte[] data) { HeartbeatPacket hp = new HeartbeatPacket(); hp.read(data); if (MycatServer.getInstance().isOnline()) { OkPacket ok = new OkPacket(); ok.packetId = 1; ok.affectedRows = hp.id; ok.serverStatus = 2; ok.write(c); if (HEARTBEAT.isInfoEnabled()) { HEARTBEAT.info(responseMessage("OK", c, hp.id)); } } else { ErrorPacket error = new ErrorPacket(); error.packetId = 1; error.errno = ErrorCode.ER_SERVER_SHUTDOWN; error.message = String.valueOf(hp.id).getBytes(); error.write(c); if (HEARTBEAT.isInfoEnabled()) { HEARTBEAT.info(responseMessage("ERROR", c, hp.id)); } } } private static String responseMessage(String action, ServerConnection c, long id) { return new StringBuilder("RESPONSE:").append(action).append(", id=").append(id).append(", host=") .append(c.getHost()).append(", port=").append(c.getPort()).append(", time=") .append(TimeUtil.currentTimeMillis()).toString(); } } ================================================ FILE: src/main/java/io/mycat/server/response/InformationSchemaProfiling.java ================================================ package io.mycat.server.response; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.server.ServerConnection; import java.nio.ByteBuffer; public class InformationSchemaProfiling { private static final int FIELD_COUNT = 3; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); /** * response method. * @param c */ public static void response(ServerConnection c) { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("State" , Fields.FIELD_TYPE_VAR_STRING); fields[i].packetId = ++packetId; fields[i+1] = PacketUtil.getField("Duration" , Fields.FIELD_TYPE_DECIMAL); fields[i+1].packetId = ++packetId; fields[i+2] = PacketUtil.getField("Percentage" , Fields.FIELD_TYPE_DECIMAL); fields[i+2].packetId = ++packetId; eof.packetId = ++packetId; ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows packetId = eof.packetId; // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } } ================================================ FILE: src/main/java/io/mycat/server/response/InformationSchemaProfilingSqlyog.java ================================================ package io.mycat.server.response; import java.nio.ByteBuffer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.server.ServerConnection; public class InformationSchemaProfilingSqlyog { private static final int FIELD_COUNT = 2; /** * response method. * @param c */ public static void response(ServerConnection c) { ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; EOFPacket eof = new EOFPacket(); int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("State" , Fields.FIELD_TYPE_VAR_STRING); fields[i].packetId = ++packetId; fields[i+1] = PacketUtil.getField("duration (summed) in sec" , Fields.FIELD_TYPE_DECIMAL); fields[i+1].packetId = ++packetId; eof.packetId = ++packetId; ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows packetId = eof.packetId; // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } } ================================================ FILE: src/main/java/io/mycat/server/response/Ping.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.net.FrontendConnection; import io.mycat.net.mysql.ErrorPacket; import io.mycat.net.mysql.OkPacket; /** * 加入了offline状态推送，用于心跳语句。 * * @author mycat */ public class Ping { private static final ErrorPacket error = PacketUtil.getShutdown(); public static void response(FrontendConnection c) { if (MycatServer.getInstance().isOnline()) { c.write(c.writeToBuffer(OkPacket.OK, c.allocate())); } else { error.write(c); } } } ================================================ FILE: src/main/java/io/mycat/server/response/PreparedStmtResponse.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import java.nio.ByteBuffer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.backend.mysql.PreparedStatement; import io.mycat.net.FrontendConnection; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.PreparedOkPacket; /** * @author mycat */ public class PreparedStmtResponse { private static final Logger LOGGER = LoggerFactory.getLogger(PreparedStmtResponse.class); public static void response(PreparedStatement pstmt, FrontendConnection c) { if (pstmt.getParams() != null || pstmt.getFields() != null) { LOGGER.info("prepared metadata from backend mysql data and is complete"); responseCompleteMetaData(pstmt, c); } else { LOGGER.info("prepared metadata comes from self-construction and is incomplete"); responseIncompleteMetaData(pstmt, c); } } private static void responseCompleteMetaData(PreparedStatement pstmt, FrontendConnection c) { byte packetId = 0; // write preparedOk packet PreparedOkPacket preparedOk = new PreparedOkPacket(); preparedOk.packetId = ++packetId; preparedOk.statementId = pstmt.getId(); preparedOk.columnsNumber = pstmt.getColumnsNumber(); preparedOk.parametersNumber = pstmt.getParametersNumber(); ByteBuffer buffer = preparedOk.write(c.allocate(), c, true); // write parameter field packet int parametersNumber = pstmt.getParametersNumber(); if (parametersNumber > 0) { for (FieldPacket param : pstmt.getParams()) { param.packetId = ++packetId; buffer = param.write(buffer, c, true); } EOFPacket eof = new EOFPacket(); eof.packetId = ++packetId; buffer = eof.write(buffer, c, true); } // write column field packet int columnsNumber = pstmt.getColumnsNumber(); if (columnsNumber > 0) { for (FieldPacket field : pstmt.getFields()) { field.packetId = ++packetId; buffer = field.write(buffer, c, true); } EOFPacket eof = new EOFPacket(); eof.packetId = ++packetId; buffer = eof.write(buffer, c, true); } // send buffer c.write(buffer); } /** * 这种返回结果，元数据不全。用c驱动读取元数据信息会报错，比如下面的用法： * prepare_meta_result = mysql_stmt_result_metadata(stmt); * column_count= mysql_num_fields(prepare_meta_result); * @param pstmt * @param c */ private static void responseIncompleteMetaData(PreparedStatement pstmt, FrontendConnection c) { byte packetId = 0; // write preparedOk packet PreparedOkPacket preparedOk = new PreparedOkPacket(); preparedOk.packetId = ++packetId; preparedOk.statementId = pstmt.getId(); preparedOk.columnsNumber = pstmt.getColumnsNumber(); preparedOk.parametersNumber = pstmt.getParametersNumber(); ByteBuffer buffer = preparedOk.write(c.allocate(), c, true); // write parameter field packet int parametersNumber = preparedOk.parametersNumber; if (parametersNumber > 0) { for (int i = 0; i < parametersNumber; i++) { FieldPacket field = new FieldPacket(); field.packetId = ++packetId; buffer = field.write(buffer, c,true); } EOFPacket eof = new EOFPacket(); eof.packetId = ++packetId; buffer = eof.write(buffer, c,true); } // write column field packet int columnsNumber = preparedOk.columnsNumber; if (columnsNumber > 0) { String[] columnNames = pstmt.getColumnNames(); for (int i = 0; i < columnsNumber; i++) { FieldPacket field = new FieldPacket(); field.name= columnNames[i].getBytes(); field.packetId = ++packetId; buffer = field.write(buffer, c,true); } EOFPacket eof = new EOFPacket(); eof.packetId = ++packetId; buffer = eof.write(buffer, c,true); } // send buffer c.write(buffer); } } ================================================ FILE: src/main/java/io/mycat/server/response/SelectConnnectID.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import java.nio.ByteBuffer; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.ErrorPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.ServerConnection; import io.mycat.util.RandomUtil; import io.mycat.util.StringUtil; /** * @author mycat */ public class SelectConnnectID { private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); private static final ErrorPacket error = PacketUtil.getShutdown(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("CONNECTION_ID()", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void response(ServerConnection c) { if (MycatServer.getInstance().isOnline()) { ByteBuffer buffer = c.allocate(); buffer = header.write(buffer, c,true); for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } buffer = eof.write(buffer, c,true); byte packetId = eof.packetId; RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(getConnectID(c)); row.packetId = ++packetId; buffer = row.write(buffer, c,true); EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); c.write(buffer); } else { error.write(c); } } private static byte[] getConnectID(ServerConnection c) { StringBuilder sb = new StringBuilder(); sb.append(new String(RandomUtil.randomBytes(10000))); return StringUtil.encode(sb.toString(), c.getCharset()); } } ================================================ FILE: src/main/java/io/mycat/server/response/SelectDatabase.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import java.nio.ByteBuffer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.ServerConnection; import io.mycat.util.StringUtil; /** * @author mycat */ public class SelectDatabase { private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("DATABASE()", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void response(ServerConnection c) { ByteBuffer buffer = c.allocate(); buffer = header.write(buffer, c,true); for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } buffer = eof.write(buffer, c,true); byte packetId = eof.packetId; RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(c.getSchema(), c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); c.write(buffer); } } ================================================ FILE: src/main/java/io/mycat/server/response/SelectIdentity.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import java.nio.ByteBuffer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.route.parser.util.ParseUtil; import io.mycat.server.ServerConnection; import io.mycat.util.LongUtil; /** * @author mycat */ public class SelectIdentity { private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); static { byte packetId = 0; header.packetId = ++packetId; } public static void response(ServerConnection c, String stmt, int aliasIndex, final String orgName) { String alias = ParseUtil.parseAlias(stmt, aliasIndex); if (alias == null) { alias = orgName; } ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields byte packetId = header.packetId; FieldPacket field = PacketUtil.getField(alias, orgName, Fields.FIELD_TYPE_LONGLONG); field.packetId = ++packetId; buffer = field.write(buffer, c,true); // write eof EOFPacket eof = new EOFPacket(); eof.packetId = ++packetId; buffer = eof.write(buffer, c,true); // write rows RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(LongUtil.toBytes(c.getLastInsertId())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } } ================================================ FILE: src/main/java/io/mycat/server/response/SelectLastInsertId.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import java.nio.ByteBuffer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.route.parser.util.ParseUtil; import io.mycat.server.ServerConnection; import io.mycat.util.LongUtil; /** * @author mycat */ public class SelectLastInsertId { private static final String ORG_NAME = "LAST_INSERT_ID()"; private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); static { byte packetId = 0; header.packetId = ++packetId; } public static void response(ServerConnection c, String stmt, int aliasIndex) { String alias = ParseUtil.parseAlias(stmt, aliasIndex); if (alias == null) { alias = ORG_NAME; } ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields byte packetId = header.packetId; FieldPacket field = PacketUtil.getField(alias, ORG_NAME, Fields.FIELD_TYPE_LONGLONG); field.packetId = ++packetId; buffer = field.write(buffer, c,true); // write eof EOFPacket eof = new EOFPacket(); eof.packetId = ++packetId; buffer = eof.write(buffer, c,true); // write rows RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(LongUtil.toBytes(c.getLastInsertId())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } } ================================================ FILE: src/main/java/io/mycat/server/response/SelectTxReadOnly.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.ServerConnection; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; import java.nio.ByteBuffer; /** * @author mycat */ public class SelectTxReadOnly { private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); private static byte[] longbt= LongUtil.toBytes(0) ; static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("@@session.tx_read_only", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void response(ServerConnection c) { ByteBuffer buffer = c.allocate(); buffer = header.write(buffer, c,true); for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } buffer = eof.write(buffer, c,true); byte packetId = eof.packetId; RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(longbt); row.packetId = ++packetId; buffer = row.write(buffer, c,true); EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); c.write(buffer); } } ================================================ FILE: src/main/java/io/mycat/server/response/SelectUser.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import java.nio.ByteBuffer; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.ErrorPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.ServerConnection; import io.mycat.util.StringUtil; /** * @author mycat */ public class SelectUser { private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); private static final ErrorPacket error = PacketUtil.getShutdown(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("USER()", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void response(ServerConnection c) { if (MycatServer.getInstance().isOnline()) { ByteBuffer buffer = c.allocate(); buffer = header.write(buffer, c,true); for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } buffer = eof.write(buffer, c,true); byte packetId = eof.packetId; RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(getUser(c)); row.packetId = ++packetId; buffer = row.write(buffer, c,true); EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); c.write(buffer); } else { error.write(c); } } private static byte[] getUser(ServerConnection c) { StringBuilder sb = new StringBuilder(); sb.append(c.getUser()).append('@').append(c.getHost()); return StringUtil.encode(sb.toString(), c.getCharset()); } } ================================================ FILE: src/main/java/io/mycat/server/response/SelectVariables.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import com.google.common.base.Splitter; import io.mycat.backend.BackendConnection; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.NonBlockingSession; import io.mycat.server.ServerConnection; import io.mycat.util.LongUtil; import io.mycat.util.StringUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; /** * @author mycat */ public final class SelectVariables { private static final Logger LOGGER = LoggerFactory.getLogger(SelectVariables.class); public static void execute(ServerConnection c, String sql) { String subSql= sql.substring(sql.indexOf("SELECT")+6); List splitVar= Splitter.on(",").omitEmptyStrings().trimResults().splitToList(subSql) ; splitVar=convert(splitVar); int FIELD_COUNT = splitVar.size(); ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; int i = 0; byte packetId = 0; header.packetId = ++packetId; for (int i1 = 0, splitVarSize = splitVar.size(); i1 < splitVarSize; i1++) { String s = splitVar.get(i1); fields[i] = PacketUtil.getField(s, Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; } ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } EOFPacket eof = new EOFPacket(); eof.packetId = ++packetId; // write eof buffer = eof.write(buffer, c,true); // write rows //byte packetId = eof.packetId; RowDataPacket row = new RowDataPacket(FIELD_COUNT); for (int i1 = 0, splitVarSize = splitVar.size(); i1 < splitVarSize; i1++) { String s = splitVar.get(i1); String value= variables.get(s) ==null?"":variables.get(s) ; row.add(value.getBytes()); } row.packetId = ++packetId; buffer = row.write(buffer, c,true); // write lastEof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // write buffer c.write(buffer); } private static List convert(List in) { List out=new ArrayList<>(); for (String s : in) { int asIndex=s.toUpperCase().indexOf(" AS "); if(asIndex!=-1) { out.add(s.substring(asIndex+4)) ; } } if(out.isEmpty()) { return in; } else { return out; } } private static final Map variables = new HashMap(); static { variables.put("@@character_set_client", "utf8"); variables.put("@@character_set_connection", "utf8"); variables.put("@@character_set_results", "utf8"); variables.put("@@character_set_server", "utf8"); variables.put("@@init_connect", ""); variables.put("@@interactive_timeout", "172800"); variables.put("@@license", "GPL"); variables.put("@@lower_case_table_names", "1"); variables.put("@@max_allowed_packet", "16777216"); variables.put("@@net_buffer_length", "16384"); variables.put("@@net_write_timeout", "60"); variables.put("@@query_cache_size", "0"); variables.put("@@query_cache_type", "OFF"); variables.put("@@sql_mode", "STRICT_TRANS_TABLES"); variables.put("@@system_time_zone", "CST"); variables.put("@@time_zone", "SYSTEM"); variables.put("@@tx_isolation", "REPEATABLE-READ"); variables.put("@@wait_timeout", "172800"); variables.put("@@session.auto_increment_increment", "1"); variables.put("character_set_client", "utf8"); variables.put("character_set_connection", "utf8"); variables.put("character_set_results", "utf8"); variables.put("character_set_server", "utf8"); variables.put("init_connect", ""); variables.put("interactive_timeout", "172800"); variables.put("license", "GPL"); variables.put("lower_case_table_names", "1"); variables.put("max_allowed_packet", "16777216"); variables.put("net_buffer_length", "16384"); variables.put("net_write_timeout", "60"); variables.put("query_cache_size", "0"); variables.put("query_cache_type", "OFF"); variables.put("sql_mode", "STRICT_TRANS_TABLES"); variables.put("system_time_zone", "CST"); variables.put("time_zone", "SYSTEM"); variables.put("tx_isolation", "REPEATABLE-READ"); variables.put("wait_timeout", "172800"); variables.put("auto_increment_increment", "1"); } } ================================================ FILE: src/main/java/io/mycat/server/response/SelectVersion.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import java.nio.ByteBuffer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.config.Versions; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.ServerConnection; /** * @author mycat */ public class SelectVersion { private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("VERSION()", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void response(ServerConnection c) { ByteBuffer buffer = c.allocate(); buffer = header.write(buffer, c,true); for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } buffer = eof.write(buffer, c,true); byte packetId = eof.packetId; RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(Versions.SERVER_VERSION); row.packetId = ++packetId; buffer = row.write(buffer, c,true); EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); c.write(buffer); } } ================================================ FILE: src/main/java/io/mycat/server/response/SelectVersionComment.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import java.nio.ByteBuffer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.ServerConnection; /** * @author mycat */ public class SelectVersionComment { private static final byte[] VERSION_COMMENT = "MyCat Server (OpenCloudDB)".getBytes(); private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("@@VERSION_COMMENT", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void response(ServerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(VERSION_COMMENT); row.packetId = ++packetId; buffer = row.write(buffer, c,true); // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } } ================================================ FILE: src/main/java/io/mycat/server/response/SessionIncrement.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import java.nio.ByteBuffer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.ServerConnection; import io.mycat.util.LongUtil; /** * @author mycat */ public class SessionIncrement { private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("@@session.auto_increment_increment", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void response(ServerConnection c) { ByteBuffer buffer = c.allocate(); buffer = header.write(buffer, c,true); for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } buffer = eof.write(buffer, c,true); byte packetId = eof.packetId; RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(LongUtil.toBytes(1)); row.packetId = ++packetId; buffer = row.write(buffer, c,true); EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); c.write(buffer); } } ================================================ FILE: src/main/java/io/mycat/server/response/SessionIsolation.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import java.nio.ByteBuffer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.ServerConnection; import io.mycat.util.StringUtil; /** * @author mycat */ public class SessionIsolation { private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("@@session.tx_isolation", Fields.FIELD_TYPE_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void response(ServerConnection c) { ByteBuffer buffer = c.allocate(); buffer = header.write(buffer, c,true); for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } buffer = eof.write(buffer, c,true); byte packetId = eof.packetId; RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode("REPEATABLE-READ",c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); c.write(buffer); } } ================================================ FILE: src/main/java/io/mycat/server/response/ShowCobarCluster.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import java.nio.ByteBuffer; import java.util.LinkedList; import java.util.List; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Alarms; import io.mycat.config.Fields; import io.mycat.config.MycatCluster; import io.mycat.config.MycatConfig; import io.mycat.config.MycatNode; import io.mycat.config.model.MycatNodeConfig; import io.mycat.config.model.SchemaConfig; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.ServerConnection; import io.mycat.util.IntegerUtil; import io.mycat.util.StringUtil; /** * @author mycat */ public class ShowCobarCluster { private static final Logger alarm = LoggerFactory.getLogger("alarm"); private static final int FIELD_COUNT = 2; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("HOST", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("WEIGHT", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void response(ServerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write field for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; for (RowDataPacket row : getRows(c)) { row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } private static List getRows(ServerConnection c) { List rows = new LinkedList(); MycatConfig config = MycatServer.getInstance().getConfig(); MycatCluster cluster = config.getCluster(); Map schemas = config.getSchemas(); SchemaConfig schema = (c.getSchema() == null) ? null : schemas.get(c.getSchema()); // 如果没有指定schema或者schema为null，则使用全部集群。 if (schema == null) { Map nodes = cluster.getNodes(); for (MycatNode n : nodes.values()) { if (n != null && n.isOnline()) { rows.add(getRow(n, c.getCharset())); } } } else { Map nodes = cluster.getNodes(); for (MycatNode n : nodes.values()) { if (n != null && n.isOnline()) { rows.add(getRow(n, c.getCharset())); } } } if (rows.size() == 0) { alarm.error(Alarms.CLUSTER_EMPTY + c.toString()); } return rows; } private static RowDataPacket getRow(MycatNode node, String charset) { MycatNodeConfig conf = node.getConfig(); RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(conf.getHost(), charset)); row.add(IntegerUtil.toBytes(conf.getWeight())); return row; } } ================================================ FILE: src/main/java/io/mycat/server/response/ShowCobarStatus.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import java.nio.ByteBuffer; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.ErrorPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.ServerConnection; /** * 加入了offline状态推送，用于心跳语句。 * * @author mycat * @author mycat */ public class ShowCobarStatus { private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); private static final RowDataPacket status = new RowDataPacket(FIELD_COUNT); private static final EOFPacket lastEof = new EOFPacket(); private static final ErrorPacket error = PacketUtil.getShutdown(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("STATUS", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; status.add("ON".getBytes()); status.packetId = ++packetId; lastEof.packetId = ++packetId; } public static void response(ServerConnection c) { if (MycatServer.getInstance().isOnline()) { ByteBuffer buffer = c.allocate(); buffer = header.write(buffer, c,true); for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } buffer = eof.write(buffer, c,true); buffer = status.write(buffer, c,true); buffer = lastEof.write(buffer, c,true); c.write(buffer); } else { error.write(c); } } } ================================================ FILE: src/main/java/io/mycat/server/response/ShowDatabases.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import java.nio.ByteBuffer; import java.util.Map; import java.util.Set; import java.util.TreeSet; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.config.MycatConfig; import io.mycat.config.model.UserConfig; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.ServerConnection; import io.mycat.util.StringUtil; /** * @author mycat */ public class ShowDatabases { private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("DATABASE", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void response(ServerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; MycatConfig conf = MycatServer.getInstance().getConfig(); Map users = conf.getUsers(); UserConfig user = users == null ? null : users.get(c.getUser()); if (user != null) { TreeSet schemaSet = new TreeSet(); Set schemaList = user.getSchemas(); if (schemaList == null || schemaList.size() == 0) { schemaSet.addAll(conf.getSchemas().keySet()); } else { for (String schema : schemaList) { schemaSet.add(schema); } } for (String name : schemaSet) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(name, c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } } ================================================ FILE: src/main/java/io/mycat/server/response/ShowFullTables.java ================================================ package io.mycat.server.response; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; import com.google.common.base.Strings; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.ErrorCode; import io.mycat.config.Fields; import io.mycat.config.MycatConfig; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.UserConfig; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.ServerConnection; import io.mycat.server.parser.ServerParse; import io.mycat.server.util.SchemaUtil; import io.mycat.util.StringUtil; /** * show tables impl * @author yanglixue * */ public class ShowFullTables { private static final int FIELD_COUNT = 2; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); private static final String SCHEMA_KEY = "schemaName"; //private static final String LIKE_KEY = "like"; //private static final Pattern pattern = Pattern.compile("^\\s*(SHOW)\\s++(FULL)*\\s*(TABLES)(\\s+(FROM)\\s+([a-zA-Z_0-9]+))?(\\s+(LIKE\\s+'(.*)'))?\\s*",Pattern.CASE_INSENSITIVE); /** * response method. * @param c */ public static void response(ServerConnection c,String stmt,int type) { String showSchema = SchemaUtil.parseShowTableSchema(stmt); if (showSchema == null) { showSchema = c.getSchema(); } SchemaConfig schema = MycatServer.getInstance().getConfig().getSchemas().get(showSchema); if(schema != null) { //不分库的schema，show tables从后端 mysql中查 String node = schema.getDataNode(); if(!Strings.isNullOrEmpty(node)) { c.execute(stmt, ServerParse.SHOW); return; } else { reponseLogicTable(c, stmt); return; } } else { c.writeErrMessage(ErrorCode.ER_NO_DB_ERROR, "Cannot find the corresponding logic schema of Mycat"); return; } } private static void reponseLogicTable(ServerConnection c, String stmt) { // 分库的schema，直接从SchemaConfig中获取所有表名 Map parm = buildFields(c,stmt); Set tableSet = getTableSet(c, parm); int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("Tables in " + parm.get(SCHEMA_KEY), Fields.FIELD_TYPE_VAR_STRING); fields[i].packetId = ++packetId; fields[i+1] = PacketUtil.getField("Table_type " , Fields.FIELD_TYPE_VAR_STRING); fields[i+1].packetId = ++packetId; eof.packetId = ++packetId; ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows packetId = eof.packetId; for (String name : tableSet) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(name, c.getCharset())); row.add(StringUtil.encode("BASE TABLE", c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } public static Set getTableSet(ServerConnection c, String stmt) { Map parm = buildFields(c, stmt); return getTableSet(c, parm); } private static Set getTableSet(ServerConnection c, Map parm) { TreeSet tableSet = new TreeSet(); MycatConfig conf = MycatServer.getInstance().getConfig(); Map users = conf.getUsers(); UserConfig user = users == null ? null : users.get(c.getUser()); if (user != null) { Map schemas = conf.getSchemas(); for (String name:schemas.keySet()){ if (null !=parm.get(SCHEMA_KEY) && parm.get(SCHEMA_KEY).toUpperCase().equals(name.toUpperCase()) ){ if(null==parm.get("LIKE_KEY")){ tableSet.addAll(schemas.get(name).getTables().keySet()); }else{ String p = "^" + parm.get("LIKE_KEY").replaceAll("%", ".*"); Pattern pattern = Pattern.compile(p,Pattern.CASE_INSENSITIVE); Matcher ma ; for (String tname : schemas.get(name).getTables().keySet()){ ma=pattern.matcher(tname); if(ma.matches()){ tableSet.add(tname); } } } } }; } return tableSet; } /** * build fields * @param c * @param stmt */ private static Map buildFields(ServerConnection c,String stmt) { Map map = new HashMap(); String fields [] =SchemaUtil.parseShowTable(stmt); if (null !=fields[3] && (!"".equals(fields[3])) && (!"null".equals(fields[3]))){ map.put(SCHEMA_KEY, fields[3]); } if ((fields[5] ==null || !"table_type".equals(fields[5])) && null !=fields[8] && (!"".equals(fields[8])) && (!"null".equals(fields[8]))){ map.put("LIKE_KEY", fields[8]); } if(null==map.get(SCHEMA_KEY)){ map.put(SCHEMA_KEY, c.getSchema()); } return map; } } ================================================ FILE: src/main/java/io/mycat/server/response/ShowMyCATCluster.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import java.nio.ByteBuffer; import java.util.LinkedList; import java.util.List; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Alarms; import io.mycat.config.Fields; import io.mycat.config.MycatCluster; import io.mycat.config.MycatConfig; import io.mycat.config.MycatNode; import io.mycat.config.model.MycatNodeConfig; import io.mycat.config.model.SchemaConfig; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.ServerConnection; import io.mycat.util.IntegerUtil; import io.mycat.util.StringUtil; /** * @author mycat */ public class ShowMyCATCluster { private static final Logger alarm = LoggerFactory.getLogger("alarm"); private static final int FIELD_COUNT = 2; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("HOST", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; fields[i] = PacketUtil.getField("WEIGHT", Fields.FIELD_TYPE_LONG); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; } public static void response(ServerConnection c) { ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write field for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows byte packetId = eof.packetId; for (RowDataPacket row : getRows(c)) { row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } private static List getRows(ServerConnection c) { List rows = new LinkedList(); MycatConfig config = MycatServer.getInstance().getConfig(); MycatCluster cluster = config.getCluster(); Map schemas = config.getSchemas(); SchemaConfig schema = (c.getSchema() == null) ? null : schemas.get(c.getSchema()); // 如果没有指定schema或者schema为null，则使用全部集群。 if (schema == null) { Map nodes = cluster.getNodes(); for (MycatNode n : nodes.values()) { if (n != null && n.isOnline()) { rows.add(getRow(n, c.getCharset())); } } } else { Map nodes = cluster.getNodes(); for (MycatNode n : nodes.values()) { if (n != null && n.isOnline()) { rows.add(getRow(n, c.getCharset())); } } } if (rows.size() == 0) { alarm.error(Alarms.CLUSTER_EMPTY + c.toString()); } return rows; } private static RowDataPacket getRow(MycatNode node, String charset) { MycatNodeConfig conf = node.getConfig(); RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(conf.getHost(), charset)); row.add(IntegerUtil.toBytes(conf.getWeight())); return row; } } ================================================ FILE: src/main/java/io/mycat/server/response/ShowMyCatStatus.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.server.response; import java.nio.ByteBuffer; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.Fields; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.ErrorPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.ServerConnection; /** * 加入了offline状态推送，用于心跳语句。 * * @author mycat * @author mycat */ public class ShowMyCatStatus { private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); private static final RowDataPacket status = new RowDataPacket(FIELD_COUNT); private static final EOFPacket lastEof = new EOFPacket(); private static final ErrorPacket error = PacketUtil.getShutdown(); static { int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("STATUS", Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; status.add("ON".getBytes()); status.packetId = ++packetId; lastEof.packetId = ++packetId; } public static void response(ServerConnection c) { if (MycatServer.getInstance().isOnline()) { ByteBuffer buffer = c.allocate(); buffer = header.write(buffer, c,true); for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } buffer = eof.write(buffer, c,true); buffer = status.write(buffer, c,true); buffer = lastEof.write(buffer, c,true); c.write(buffer); } else { error.write(c); } } } ================================================ FILE: src/main/java/io/mycat/server/response/ShowTables.java ================================================ package io.mycat.server.response; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; import com.google.common.base.Strings; import io.mycat.MycatServer; import io.mycat.backend.mysql.PacketUtil; import io.mycat.config.ErrorCode; import io.mycat.config.Fields; import io.mycat.config.MycatConfig; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.UserConfig; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.ServerConnection; import io.mycat.server.parser.ServerParse; import io.mycat.server.util.SchemaUtil; import io.mycat.util.StringUtil; /** * show tables impl * @author yanglixue * */ public class ShowTables { private static final int FIELD_COUNT = 1; private static final ResultSetHeaderPacket header = PacketUtil.getHeader(FIELD_COUNT); private static final FieldPacket[] fields = new FieldPacket[FIELD_COUNT]; private static final EOFPacket eof = new EOFPacket(); private static final String SCHEMA_KEY = "schemaName"; private static final String LIKE_KEY = "like"; private static final Pattern pattern = Pattern.compile("^\\s*(SHOW)\\s+(TABLES)(\\s+(FROM)\\s+([a-zA-Z_0-9]+))?(\\s+(LIKE\\s+'(.*)'))?\\s*",Pattern.CASE_INSENSITIVE); /** * response method. * @param c */ public static void response(ServerConnection c,String stmt,int type) { String showSchemal= SchemaUtil.parseShowTableSchema(stmt) ; String cSchema =showSchemal==null? c.getSchema():showSchemal; SchemaConfig schema = MycatServer.getInstance().getConfig().getSchemas().get(cSchema); if(schema != null) { //不分库的schema，show tables从后端 mysql中查 String node = schema.getDataNode(); if(!Strings.isNullOrEmpty(node)) { c.execute(stmt, ServerParse.SHOW); return; } else { reponseLogicTable(c, stmt); return; } } else { c.writeErrMessage(ErrorCode.ER_NO_DB_ERROR,"No database selected"); return; } } private static void reponseLogicTable(ServerConnection c, String stmt) { // 分库的schema，直接从SchemaConfig中获取所有表名 Map parm = buildFields(c,stmt); java.util.Set tableSet = getTableSet(c, parm); int i = 0; byte packetId = 0; header.packetId = ++packetId; fields[i] = PacketUtil.getField("Tables in " + parm.get(SCHEMA_KEY), Fields.FIELD_TYPE_VAR_STRING); fields[i++].packetId = ++packetId; eof.packetId = ++packetId; ByteBuffer buffer = c.allocate(); // write header buffer = header.write(buffer, c,true); // write fields for (FieldPacket field : fields) { buffer = field.write(buffer, c,true); } // write eof buffer = eof.write(buffer, c,true); // write rows packetId = eof.packetId; for (String name : tableSet) { RowDataPacket row = new RowDataPacket(FIELD_COUNT); row.add(StringUtil.encode(name.toLowerCase(), c.getCharset())); row.packetId = ++packetId; buffer = row.write(buffer, c,true); } // write last eof EOFPacket lastEof = new EOFPacket(); lastEof.packetId = ++packetId; buffer = lastEof.write(buffer, c,true); // post write c.write(buffer); } public static Set getTableSet(ServerConnection c, String stmt) { Map parm = buildFields(c, stmt); return getTableSet(c, parm); } private static Set getTableSet(ServerConnection c, Map parm) { TreeSet tableSet = new TreeSet(); MycatConfig conf = MycatServer.getInstance().getConfig(); Map users = conf.getUsers(); UserConfig user = users == null ? null : users.get(c.getUser()); if (user != null) { Map schemas = conf.getSchemas(); for (String name:schemas.keySet()){ if (null !=parm.get(SCHEMA_KEY) && parm.get(SCHEMA_KEY).toUpperCase().equals(name.toUpperCase()) ){ if(null==parm.get("LIKE_KEY")){ tableSet.addAll(schemas.get(name).getTables().keySet()); }else{ String p = "^" + parm.get("LIKE_KEY").replaceAll("%", ".*"); Pattern pattern = Pattern.compile(p,Pattern.CASE_INSENSITIVE); Matcher ma ; for (String tname : schemas.get(name).getTables().keySet()){ ma=pattern.matcher(tname); if(ma.matches()){ tableSet.add(tname); } } } } }; } return tableSet; } /** * build fields * @param c * @param stmt */ private static Map buildFields(ServerConnection c,String stmt) { Map map = new HashMap(); Matcher ma = pattern.matcher(stmt); if(ma.find()){ String schemaName=ma.group(5); if (null !=schemaName && (!"".equals(schemaName)) && (!"null".equals(schemaName))){ map.put(SCHEMA_KEY, schemaName); } String like = ma.group(8); if (null !=like && (!"".equals(like)) && (!"null".equals(like))){ map.put("LIKE_KEY", like); } } if(null==map.get(SCHEMA_KEY)){ map.put(SCHEMA_KEY, c.getSchema()); } return map; } } ================================================ FILE: src/main/java/io/mycat/server/sqlcmd/CommitCommand.java ================================================ package io.mycat.server.sqlcmd; import io.mycat.backend.BackendConnection; import io.mycat.net.mysql.ErrorPacket; import io.mycat.server.NonBlockingSession; public class CommitCommand implements SQLCtrlCommand { @Override public void sendCommand(NonBlockingSession session, BackendConnection con) { con.commit(); } @Override public void errorResponse(NonBlockingSession session, byte[] err, int total, int failed) { ErrorPacket errPkg = new ErrorPacket(); errPkg.read(err); String errInfo = "total " + total + " failed " + failed + " detail:" + new String(errPkg.message); session.getSource().setTxInterrupt(errInfo); errPkg.write(session.getSource()); } @Override public void okResponse(NonBlockingSession session, byte[] ok) { session.getSource().write(ok); } @Override public boolean releaseConOnErr() { // need rollback when err return false; } @Override public boolean relaseConOnOK() { return true; } @Override public boolean isAutoClearSessionCons() { // need rollback when err return false; } } ================================================ FILE: src/main/java/io/mycat/server/sqlcmd/SQLCmdConstant.java ================================================ package io.mycat.server.sqlcmd; public class SQLCmdConstant { public static final CommitCommand COMMIT_CMD = new CommitCommand(); private SQLCmdConstant() { } } ================================================ FILE: src/main/java/io/mycat/server/sqlcmd/SQLCtrlCommand.java ================================================ package io.mycat.server.sqlcmd; import io.mycat.backend.BackendConnection; import io.mycat.server.NonBlockingSession; /** * sql command like set xxxx ,only return OK /Err Pacakage,can't return restult * set * * @author wuzhih * */ public interface SQLCtrlCommand { boolean isAutoClearSessionCons(); boolean releaseConOnErr(); boolean relaseConOnOK(); void sendCommand(NonBlockingSession session, BackendConnection con); /** * 收到错误数据包的响应处理 */ void errorResponse(NonBlockingSession session,byte[] err,int total,int failed); /** * 收到OK数据包的响应处理 */ void okResponse(NonBlockingSession session, byte[] ok); } ================================================ FILE: src/main/java/io/mycat/server/util/SchemaUtil.java ================================================ package io.mycat.server.util; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.ast.statement.SQLShowColumnsStatement; import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; import com.alibaba.druid.sql.parser.SQLStatementParser; import io.mycat.MycatServer; import io.mycat.config.model.SchemaConfig; import io.mycat.route.parser.druid.MycatSchemaStatVisitor; import io.mycat.route.parser.druid.MycatStatementParser; import io.mycat.server.parser.ServerParse; /** * Created by magicdoom on 2016/1/26. */ public class SchemaUtil { public static SchemaInfo parseSchema(String sql) { try{ MycatStatementParser parser = new MycatStatementParser(sql); return parseTables(parser.parseStatement(), new MycatSchemaStatVisitor()); }catch (Exception ignore){ return null; } } public static String detectDefaultDb(String sql, int type) { String db = null; Map schemaConfigMap = MycatServer.getInstance().getConfig() .getSchemas(); if (ServerParse.SELECT == type) { SchemaUtil.SchemaInfo schemaInfo = SchemaUtil.parseSchema(sql); if ((schemaInfo == null || schemaInfo.table == null) && !schemaConfigMap.isEmpty()) { db = schemaConfigMap.entrySet().iterator().next().getKey(); } if (schemaInfo != null && schemaInfo.schema != null) { if (schemaConfigMap.containsKey(schemaInfo.schema)) { db = schemaInfo.schema; /** * 对 MySQL 自带的元数据库 information_schema 进行返回 */ } else if ("information_schema".equalsIgnoreCase(schemaInfo.schema)) { db = "information_schema"; } } } else if (ServerParse.INSERT == type || ServerParse.UPDATE == type || ServerParse.DELETE == type || ServerParse.DDL == type) { SchemaUtil.SchemaInfo schemaInfo = SchemaUtil.parseSchema(sql); if (schemaInfo != null && schemaInfo.schema != null && schemaConfigMap.containsKey(schemaInfo.schema)) { db = schemaInfo.schema; } } else if(ServerParse.COMMAND == type) { int index = sql.indexOf(ServerParse.COM_FIELD_LIST_FLAG); String table = sql.substring(index + 17); db = getSchema(schemaConfigMap, table); } else if ((ServerParse.SHOW == type || ServerParse.USE == type || ServerParse.EXPLAIN == type || ServerParse.SET == type || ServerParse.HELP == type || ServerParse.DESCRIBE == type) && !schemaConfigMap.isEmpty()) { try { //当前存在多个schema的时候使用原来的逻辑会出现bug，改为根据mycat schema配置中的对应关系获取 SQLStatementParser parser = new MySqlStatementParser(sql); SQLShowColumnsStatement stmt = (SQLShowColumnsStatement) parser.parseStatement(); db = getSchema(schemaConfigMap, stmt.getTable().getSimpleName()); } catch (Exception e) { //e.printStackTrace(); //兼容mysql gui 不填默认database db = schemaConfigMap.entrySet().iterator().next().getKey(); } } return db; } /** 根据mycat schema配置中的对应关系获取对应的db */ public static String getSchema(Map schemaConfigMap, String table) { String db = null; for (Map.Entry entry : schemaConfigMap.entrySet()) { if(entry.getValue().getTables().containsKey(table.toUpperCase())) { db = entry.getKey(); break; } } return db; } public static String parseShowTableSchema(String sql) { // Matcher ma = SHOW_FULL_TABLE_PATTERN.matcher(sql); // if (ma.matches() && ma.groupCount() >= 5) { // return ma.group(5); // } String fields[] =parseShowTable(sql); return fields[3]; } /** * 解析show full table from|in schema-name where table_type|tables_in_xxx like 'xxx' * * @param sql * @return fields' array. * 
     *          [0] is matched, 1 is matcher , other is not matcher
     *          [1] is full, 代表当前是show full tabe ，如果为空则表示show table
     *          [2] from or in
     *          [3] schema-name
     *          [4] where
     *          [5] is 'table_type' or 'tables_in_xxxx'
     *          [6] is 'like' or '=' or ... other operator
     *          [7] 
     *          [8] is where match condition str, etc. table-name or table_type like 'BASE TABLE'
     *
*/ public static String[] parseShowTable(String sql) { Matcher matcher = SHOW_FULL_TABLE_PATTERN.matcher(sql); // matcher.matches(); String[] fields = new String[9]; if (matcher.find()) {// show tables fields[0] = "1"; // group(1) is 'full' fields[1] = matcher.group(1); if(fields[1] != null) { fields[1] = fields[1].trim(); } // group(2) is 'from or in' fields[2] = matcher.group(2); if (fields[2] == null || fields[2].length() <= 0) { fields[3] = null; if(matcher.group(3) !=null && "LIKE".equals(matcher.group(3))) { fields[4] = null; } else { fields[4] = "WHERE"; } } else { fields[2] = fields[2].trim(); // group(3) is schema-name fields[3] = matcher.group(3); //group(4) is 'where' fields[4] = matcher.group(4); if(fields[4] != null) { fields[4] = fields[4].trim(); } } //group(5) is 'table_type' or 'tables_in_xxxx' fields[5] = matcher.group(5); //group(6) is 'like' or '=' or ... other operator if(fields[4] == null) { fields[6] = "LIKE"; } else { fields[6] = matcher.group(6); if(fields[6] !=null) { fields[6] = fields[6].trim(); } } //skip fields[7] = matcher.group(7); // //group(8) is where match condition str, etc. table-name or table_type like 'BASE TABLE' fields[8] = matcher.group(8); } return fields; } private static SchemaInfo parseTables(SQLStatement stmt, MycatSchemaStatVisitor schemaStatVisitor) { stmt.accept(schemaStatVisitor); String key = schemaStatVisitor.getCurrentTable(); if (key != null && key.contains("`")) { key = key.replaceAll("`", ""); } if (key != null) { SchemaInfo schemaInfo = new SchemaInfo(); int pos = key.indexOf("."); if (pos > 0) { schemaInfo.schema = key.substring(0, pos); schemaInfo.table = key.substring(pos + 1); } else { schemaInfo.table = key; } return schemaInfo; } return null; } public static class SchemaInfo { public String table; public String schema; @Override public String toString() { final StringBuffer sb = new StringBuffer("SchemaInfo{"); sb.append("table='").append(table).append('\''); sb.append(", schema='").append(schema).append('\''); sb.append('}'); return sb.toString(); } } //sample：SHOW FULL TABLES FROM information_schema WHERE Tables_in_information_schema LIKE 'KEY_COLUMN_USAGE' //注意sql中like后面会有单引号 private static Pattern SHOW_FULL_TABLE_PATTERN = Pattern.compile( "^\\s*show\\s+(full\\s+)?tables \\s*(in |from )?\\s*(\\w+)*\\s*(where )?\\s*(table_type|tables_in_\\w+)*\\s*(\\= |like )?\\s*('([\\w%\\s]+)')*\\s*(;)*\\s*", Pattern.CASE_INSENSITIVE); public static void main(String[] args) { String sql = "SELECT name, type FROM `mysql`.`proc` as xxxx WHERE Db='base'"; // System.out.println(parseSchema(sql)); sql = "insert into aaa.test(id) values(1)"; // System.out.println(parseSchema(sql)); sql = "update updatebase.test set xx=1 "; //System.out.println(parseSchema(sql)); sql = "CREATE TABLE IF not EXISTS `test` (\n" + " `id` bigint(20) NOT NULL AUTO_INCREMENT,\n" + " `sid` bigint(20) DEFAULT NULL,\n" + " `name` varchar(45) DEFAULT NULL,\n" + " `value` varchar(45) DEFAULT NULL,\n" + " `_slot` int(11) DEFAULT NULL COMMENT '自动迁移算法slot,禁止修改',\n" + " PRIMARY KEY (`id`)\n" + ") ENGINE=InnoDB AUTO_INCREMENT=805781256930734081 DEFAULT CHARSET=utf8"; System.out.println(parseSchema(sql)); String pat3 = "SHOW FULL TABLES FROM information_schema WHERE Tables_in_information_schema LIKE 'KEY_COLUMN_USAGE'"; Matcher ma = SHOW_FULL_TABLE_PATTERN.matcher(pat3); if (ma.matches()) { System.out.println(ma.groupCount()); System.out.println(ma.group(5)); } } } ================================================ FILE: src/main/java/io/mycat/sqlengine/AllJobFinishedListener.java ================================================ package io.mycat.sqlengine; /** * called when all jobs in EngineCxt finished * @author wuzhih * */ public interface AllJobFinishedListener { void onAllJobFinished(EngineCtx ctx); } ================================================ FILE: src/main/java/io/mycat/sqlengine/BatchSQLJob.java ================================================ package io.mycat.sqlengine; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import io.mycat.MycatServer; import io.mycat.config.model.SystemConfig; public class BatchSQLJob { private ConcurrentHashMap runningJobs = new ConcurrentHashMap(); private ConcurrentLinkedQueue waitingJobs = new ConcurrentLinkedQueue(); private volatile boolean noMoreJobInput = false; /* * * parallExecute: 是否可以并行执行 * */ public void addJob(SQLJob newJob, boolean parallExecute) { SystemConfig system = MycatServer.getInstance().getConfig().getSystem(); parallExecute = parallExecute && (system.getParallExecute() == 1); if (parallExecute) { runJob(newJob); } else { waitingJobs.offer(newJob); if (runningJobs.isEmpty()) { SQLJob job = waitingJobs.poll(); if (job != null) { runJob(job); } } } } //设置批量任务已经不会在添加任务了。 public void setNoMoreJobInput(boolean noMoreJobInput) { this.noMoreJobInput = noMoreJobInput; } //执行任务 private void runJob(SQLJob newJob) { // EngineCtx.LOGGER.info("run job " + newJob); runningJobs.put(newJob.getId(), newJob); MycatServer.getInstance().getBusinessExecutor().execute(newJob); } //单个的任务执行完毕。 等待任务列表中有任务， 继续执行下一个任务。 //返回： 是否所有的任务执行完毕。 public boolean jobFinished(SQLJob sqlJob) { if (EngineCtx.LOGGER.isDebugEnabled()) { EngineCtx.LOGGER.info("job finished " + sqlJob); } runningJobs.remove(sqlJob.getId()); SQLJob job = waitingJobs.poll(); if (job != null) { runJob(job); return false; } else { if (noMoreJobInput) { return runningJobs.isEmpty() && waitingJobs.isEmpty(); } else { return false; } } } } ================================================ FILE: src/main/java/io/mycat/sqlengine/EngineCtx.java ================================================ package io.mycat.sqlengine; import java.nio.ByteBuffer; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.ResultSetHeaderPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.NonBlockingSession; import io.mycat.server.ServerConnection; // //任务的进行的调用， //向mysqlClient 写数据。 public class EngineCtx { public static final Logger LOGGER = LoggerFactory.getLogger(EngineCtx.class); private final BatchSQLJob bachJob; private AtomicInteger jobId = new AtomicInteger(0); AtomicInteger packetId = new AtomicInteger(0); private final NonBlockingSession session; private AtomicBoolean finished = new AtomicBoolean(false); private AllJobFinishedListener allJobFinishedListener; private AtomicBoolean headerWrited = new AtomicBoolean(); private final ReentrantLock writeLock = new ReentrantLock(); private volatile boolean hasError = false; // private volatile RouteResultset rrs; private volatile boolean isStreamOutputResult = true; //是否流式输出。 public EngineCtx(NonBlockingSession session) { this.bachJob = new BatchSQLJob(); this.session = session; } public boolean getIsStreamOutputResult(){ return isStreamOutputResult; } public void setIsStreamOutputResult(boolean isStreamOutputResult){ this. isStreamOutputResult = isStreamOutputResult; } public byte incPackageId() { return (byte) packetId.incrementAndGet(); } /* * 将sql 发送到所有的dataNodes分片 * 顺序执行 * */ public void executeNativeSQLSequnceJob(String[] dataNodes, String sql, SQLJobHandler jobHandler) { for (String dataNode : dataNodes) { SQLJob job = new SQLJob(jobId.incrementAndGet(), sql, dataNode, jobHandler, this); bachJob.addJob(job, false); } } public ReentrantLock getWriteLock() { return writeLock; } /* * 所有任务完成的回调 * */ public void setAllJobFinishedListener( AllJobFinishedListener allJobFinishedListener) { this.allJobFinishedListener = allJobFinishedListener; } /* * 将sql 发送到所有的dataNodes分片 * 可以并行执行 * */ public void executeNativeSQLParallJob(String[] dataNodes, String sql, SQLJobHandler jobHandler) { for (String dataNode : dataNodes) { SQLJob job = new SQLJob(jobId.incrementAndGet(), sql, dataNode, jobHandler, this); bachJob.addJob(job, true); } } /** * set no more jobs created */ public void endJobInput() { bachJob.setNoMoreJobInput(true); } //a 表和 b表的字段的信息合并在一块。 //向mysql client 输出。 public void writeHeader(List afields, List bfields) { if (headerWrited.compareAndSet(false, true)) { try { writeLock.lock(); // write new header ResultSetHeaderPacket headerPkg = new ResultSetHeaderPacket(); headerPkg.fieldCount = afields.size() +bfields.size()-1; headerPkg.packetId = incPackageId(); LOGGER.debug("packge id " + headerPkg.packetId); ServerConnection sc = session.getSource(); ByteBuffer buf = headerPkg.write(sc.allocate(), sc, true); // wirte a fields for (byte[] field : afields) { field[3] = incPackageId(); buf = sc.writeToBuffer(field, buf); } // write b field for (int i=1;i afields) { if (headerWrited.compareAndSet(false, true)) { try { writeLock.lock(); // write new header ResultSetHeaderPacket headerPkg = new ResultSetHeaderPacket(); headerPkg.fieldCount = afields.size();// -1; headerPkg.packetId = incPackageId(); LOGGER.debug("packge id " + headerPkg.packetId); ServerConnection sc = session.getSource(); ByteBuffer buf = headerPkg.write(sc.allocate(), sc, true); // wirte a fields for (byte[] field : afields) { field[3] = incPackageId(); buf = sc.writeToBuffer(field, buf); } // write field eof EOFPacket eofPckg = new EOFPacket(); eofPckg.packetId = incPackageId(); buf = eofPckg.write(buf, sc, true); sc.write(buf); //LOGGER.info("header outputed ,packgId:" + eofPckg.packetId); } finally { writeLock.unlock(); } } } public void writeRow(RowDataPacket rowDataPkg) { ServerConnection sc = session.getSource(); try { writeLock.lock(); rowDataPkg.packetId = incPackageId(); // 输出完整的 记录到客户端 ByteBuffer buf = rowDataPkg.write(sc.allocate(), sc, true); sc.write(buf); //LOGGER.info("write row ,packgId:" + rowDataPkg.packetId); } finally { writeLock.unlock(); } } public void writeEof() { ServerConnection sc = session.getSource(); EOFPacket eofPckg = new EOFPacket(); eofPckg.packetId = incPackageId(); ByteBuffer buf = eofPckg.write(sc.allocate(), sc, false); sc.write(buf); LOGGER.info("write eof ,packgId:" + eofPckg.packetId); } public NonBlockingSession getSession() { return session; } //单个sqlJob任务完成之后调用的。 //全部任务完成之后 回调allJobFinishedListener 这个函数。 public void onJobFinished(SQLJob sqlJob) { boolean allFinished = bachJob.jobFinished(sqlJob); if (allFinished && finished.compareAndSet(false, true)) { if(!hasError){ LOGGER.info("all job finished for front connection: " + session.getSource()); allJobFinishedListener.onAllJobFinished(this); }else{ LOGGER.info("all job finished with error for front connection: " + session.getSource()); } } } public boolean isHasError() { return hasError; } public void setHasError(boolean hasError) { this.hasError = hasError; } // public void setRouteResultset(RouteResultset rrs){ // this.rrs = rrs; // } } ================================================ FILE: src/main/java/io/mycat/sqlengine/MultiRowSQLQueryResultHandler.java ================================================ package io.mycat.sqlengine; import java.util.LinkedList; import java.util.List; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * 当SQLJob的结果有多行时，利用该处理器进行处理 * @author digdeep@126.com */ public class MultiRowSQLQueryResultHandler extends OneRawSQLQueryResultHandler{ private static final Logger LOGGER = LoggerFactory .getLogger(MultiRowSQLQueryResultHandler.class); // 获得结果之后，利用该对象进行回调进行通知和处理结果 private final SQLQueryResultListener>>> callback; private List> resultRows = new LinkedList<>(); // 保存结果行 public MultiRowSQLQueryResultHandler(String[] fetchCols, SQLQueryResultListener>>> callback) { super(fetchCols, null); this.callback = callback; } @Override public boolean onRowData(String dataNode, byte[] rowData) { super.onRowData(dataNode, rowData); resultRows.add(getResult()); return false; } @Override public void finished(String dataNode, boolean failed, String errorMsg) { SQLQueryResult>> queryResult = new SQLQueryResult>>(this.resultRows, !failed); queryResult.setErrMsg(errorMsg); if(callback != null) this.callback.onResult(queryResult); // callback 是构造函数传进来，在得到结果是进行回调 else LOGGER.warn(" callback is null "); } } ================================================ FILE: src/main/java/io/mycat/sqlengine/OneRawSQLQueryResultHandler.java ================================================ package io.mycat.sqlengine; import java.util.HashMap; import java.util.List; import java.util.Map; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.RowDataPacket; public class OneRawSQLQueryResultHandler implements SQLJobHandler { private Map fetchColPosMap; private final SQLQueryResultListener>> callback; private final String[] fetchCols; private int fieldCount = 0; private Map result = new HashMap(); public OneRawSQLQueryResultHandler(String[] fetchCols, SQLQueryResultListener>> callBack) { this.fetchCols = fetchCols; this.callback = callBack; } private String mark; public void onHeader(String dataNode, byte[] header, List fields) { fieldCount = fields.size(); fetchColPosMap = new HashMap(); for (String watchFd : fetchCols) { for (int i = 0; i < fieldCount; i++) { byte[] field = fields.get(i); FieldPacket fieldPkg = new FieldPacket(); fieldPkg.read(field); String fieldName = new String(fieldPkg.name); if (watchFd.equalsIgnoreCase(fieldName)) { fetchColPosMap.put(fieldName, i); } } } } @Override public boolean onRowData(String dataNode, byte[] rowData) { RowDataPacket rowDataPkg = new RowDataPacket(fieldCount); rowDataPkg.read(rowData); String variableName = ""; String variableValue = ""; //fieldcount为2可能是select x也可能是show create table命令 if(fieldCount==2 && (fetchColPosMap.get("Variable_name")!=null || fetchColPosMap.get("Value")!=null)){ Integer ind = fetchColPosMap.get("Variable_name"); if (ind != null) { byte[] columnData = rowDataPkg.fieldValues.get(ind); String columnVal = columnData!=null?new String(columnData):null; variableName = columnVal; } ind = fetchColPosMap.get("Value"); if (ind != null) { byte[] columnData = rowDataPkg.fieldValues.get(ind); String columnVal = columnData!=null?new String(columnData):null; variableValue = columnVal; } result.put(variableName, variableValue); }else{ for (String fetchCol : fetchCols) { Integer ind = fetchColPosMap.get(fetchCol); if (ind != null) { byte[] columnData = rowDataPkg.fieldValues.get(ind); String columnVal = columnData!=null?new String(columnData):null; result.put(fetchCol, columnVal); } else { LOGGER.warn("cant't find column in sql query result " + fetchCol); } } } return false; } @Override public void finished(String dataNode, boolean failed, String errorMsg) { SQLQueryResult> queryRestl=new SQLQueryResult>(this.result,!failed, dataNode,errorMsg); this.callback.onResult(queryRestl); } public String getMark() { return mark; } public void setMark(String mark) { this.mark = mark; } // 子类 MultiRowSQLQueryResultHandler 需要使用 protected Map getResult() { return result; } } ================================================ FILE: src/main/java/io/mycat/sqlengine/SQLJob.java ================================================ package io.mycat.sqlengine; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; import io.mycat.backend.BackendConnection; import io.mycat.backend.datasource.PhysicalDBNode; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.mysql.nio.handler.ResponseHandler; import io.mycat.config.MycatConfig; import io.mycat.net.mysql.ErrorPacket; import io.mycat.route.RouteResultsetNode; import io.mycat.server.ServerConnection; import io.mycat.server.parser.ServerParse; /** * asyn execute in EngineCtx or standalone (EngineCtx=null) * * @author wuzhih * */ public class SQLJob implements ResponseHandler, Runnable { private static final Logger LOGGER = LoggerFactory.getLogger(SQLJob.class); private final String sql; private final String dataNodeOrDatabase; private BackendConnection connection; private final SQLJobHandler jobHandler; private final EngineCtx ctx; private final PhysicalDatasource ds; private final int id; private volatile boolean finished; public SQLJob(int id, String sql, String dataNode, SQLJobHandler jobHandler, EngineCtx ctx) { super(); this.id = id; this.sql = sql; this.dataNodeOrDatabase = dataNode; this.jobHandler = jobHandler; this.ctx = ctx; this.ds = null; } public SQLJob(String sql, String databaseName, SQLJobHandler jobHandler, PhysicalDatasource ds) { super(); this.id = 0; this.sql = sql; this.dataNodeOrDatabase = databaseName; this.jobHandler = jobHandler; this.ctx = null; this.ds = ds; } public void run() { try { if (ds == null) { RouteResultsetNode node = new RouteResultsetNode( dataNodeOrDatabase, ServerParse.SELECT, sql); // create new connection MycatConfig conf = MycatServer.getInstance().getConfig(); PhysicalDBNode dn = conf.getDataNodes().get(node.getName()); dn.getConnection(dn.getDatabase(), true, node, this, node); } else { ds.getConnection(dataNodeOrDatabase, true, this, null); } } catch (Exception e) { LOGGER.info("can't get connection for sql ,error:" ,e); doFinished(true,e.getMessage()); } } public void teminate(String reason) { LOGGER.info("terminate this job reason:" + reason + " con:" + connection + " sql " + this.sql); if (connection != null) { connection.close(reason); } } @Override public void connectionAcquired(final BackendConnection conn) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("con query sql:" + sql + " to con:" + conn); } conn.setResponseHandler(this); try { if(ctx != null) { ServerConnection sc = ctx.getSession().getSource(); //conn.setCharsetIndex(sc.getCharsetIndex()); conn.query(sql ,sc.getCharsetIndex()); }else { conn.query(sql ); } connection = conn; } catch (Exception e) {// (UnsupportedEncodingException e) { doFinished(true,e.getMessage()); } } public boolean isFinished() { return finished; } private void doFinished(boolean failed,String errorMsg) { finished = true; jobHandler.finished(dataNodeOrDatabase, failed,errorMsg ); if (ctx != null) { if(failed){ ctx.setHasError(true); } ctx.onJobFinished(this); } } @Override public void connectionError(Throwable e, BackendConnection conn) { LOGGER.info("can't get connection for sql :" + sql); doFinished(true,e.getMessage()); } @Override public void errorResponse(byte[] err, BackendConnection conn) { ErrorPacket errPg = new ErrorPacket(); errPg.read(err); String errMsg = "error response errno:" + errPg.errno + ", " + new String(errPg.message) + " from of sql :" + sql + " at con:" + conn; // @see https://dev.mysql.com/doc/refman/5.6/en/error-messages-server.html // ER_SPECIFIC_ACCESS_DENIED_ERROR if ( errPg.errno == 1227 ) { LOGGER.warn( errMsg ); } else { LOGGER.info( errMsg ); } doFinished(true,errMsg); conn.release(); } @Override public void okResponse(byte[] ok, BackendConnection conn) { // conn.syncAndExcute(); //modify by zwy 这边 涉及到use database的返回，不能直接释放连接 需要继续处理包 boolean executeResponse = conn.syncAndExcute(); if(executeResponse){ doFinished(false,null); conn.release(); } else { LOGGER.debug("syn response {}" ,conn); } } @Override public void fieldEofResponse(byte[] header, List fields, byte[] eof, BackendConnection conn) { jobHandler.onHeader(dataNodeOrDatabase, header, fields); } @Override public void rowResponse(byte[] row, BackendConnection conn) { boolean finsihed = jobHandler.onRowData(dataNodeOrDatabase, row); if (finsihed) { doFinished(false,null); conn.close("not needed by user proc"); } } @Override public void rowEofResponse(byte[] eof, BackendConnection conn) { doFinished(false,null); conn.release(); } @Override public void writeQueueAvailable() { } @Override public void connectionClose(BackendConnection conn, String reason) { doFinished(true,reason); } public int getId() { return id; } @Override public String toString() { return "SQLJob [ id=" + id + ",dataNodeOrDatabase=" + dataNodeOrDatabase + ",sql=" + sql + ", jobHandler=" + jobHandler + "]"; } } ================================================ FILE: src/main/java/io/mycat/sqlengine/SQLJobHandler.java ================================================ package io.mycat.sqlengine; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public interface SQLJobHandler { public static final Logger LOGGER = LoggerFactory.getLogger(SQLJobHandler.class); public void onHeader(String dataNode, byte[] header, List fields); public boolean onRowData(String dataNode, byte[] rowData); public void finished(String dataNode, boolean failed, String errorMsg); } ================================================ FILE: src/main/java/io/mycat/sqlengine/SQLQueryResult.java ================================================ package io.mycat.sqlengine; public class SQLQueryResult { private final T result; private final boolean success; private final String dataNode; // dataNode or database name private String tableName; private String errMsg; public SQLQueryResult(T result, boolean success) { super(); this.result = result; this.success = success; this.dataNode = null; } public SQLQueryResult(T result, boolean success, String dataNode,String errMsg) { super(); this.result = result; this.success = success; this.dataNode= dataNode; this.errMsg=errMsg; } public String getErrMsg() { return errMsg; } public void setErrMsg(String errMsg) { this.errMsg = errMsg; } public T getResult() { return result; } public boolean isSuccess() { return success; } public String getDataNode() { return dataNode; } public String getTableName() { return tableName; } public void setTableName(String tableName) { this.tableName = tableName; } } ================================================ FILE: src/main/java/io/mycat/sqlengine/SQLQueryResultListener.java ================================================ package io.mycat.sqlengine; public interface SQLQueryResultListener { public void onResult(T result); } ================================================ FILE: src/main/java/io/mycat/sqlengine/mpp/AbstractDataNodeMerge.java ================================================ package io.mycat.sqlengine.mpp; import io.mycat.MycatServer; import io.mycat.backend.mysql.nio.handler.MultiNodeQueryHandler; import io.mycat.net.mysql.RowDataPacket; import io.mycat.route.RouteResultset; import io.mycat.server.NonBlockingSession; import io.mycat.util.StringUtil; import org.apache.log4j.Logger; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; /** * Created by zagnix on 2016/7/6. */ public abstract class AbstractDataNodeMerge implements Runnable{ private static Logger LOGGER = Logger.getLogger(AbstractDataNodeMerge.class); /** *row 有多少col */ protected int fieldCount; /** * 本次select的路由缓存集 */ protected final RouteResultset rrs; /** * 夸分片处理handler */ protected MultiNodeQueryHandler multiQueryHandler = null; /** * 分片结束包 */ public PackWraper END_FLAG_PACK = new PackWraper(); /** * 是否执行流式结果集输出 */ protected boolean isStreamOutputResult = false; /** * rowData缓存队列 */ protected BlockingQueue packs = new LinkedBlockingQueue(); /** * 标志业务线程是否启动了？ */ protected final AtomicBoolean running = new AtomicBoolean(false); public AbstractDataNodeMerge(MultiNodeQueryHandler handler,RouteResultset rrs){ this.rrs = rrs; this.multiQueryHandler = handler; } public boolean isStreamOutputResult() { return isStreamOutputResult; } public void setStreamOutputResult(boolean streamOutputResult) { isStreamOutputResult = streamOutputResult; } /** * Add a row pack, and may be wake up a business thread to work if not running. * @param pack row pack * @return true wake up a business thread, otherwise false * * @author Uncle-pan * @since 2016-03-23 */ protected final boolean addPack(final PackWraper pack){ packs.add(pack); if(running.get()){ return false; } final MycatServer server = MycatServer.getInstance(); server.getBusinessExecutor().execute(this); return true; } /** * 处理新进来每个row数据，通过PackWraper进行封装， * 投递到队列中进行后续处理即可。 * process new record (mysql binary data),if data can output to client * ,return true * * @param dataNode * DN's name (data from this dataNode) * @param rowData * raw data */ public boolean onNewRecord(String dataNode, byte[] rowData) { final PackWraper data = new PackWraper(); data.dataNode = dataNode; data.rowData = rowData; addPack(data); return false; } /** * 将Map对应的col字段集，返回row中对应的index数组 * @param columns * @param toIndexMap * @return */ protected static int[] toColumnIndex(String[] columns, Map toIndexMap) { int[] result = new int[columns.length]; ColMeta curColMeta; for (int i = 0; i < columns.length; i++) { curColMeta = toIndexMap.get(StringUtil.removeBackquote(columns[i]).toUpperCase()); if (curColMeta == null) { throw new IllegalArgumentException( "all columns in group by clause should be in the selected column list.!" + columns[i]); } result[i] = curColMeta.colIndex; } return result; } @Override public abstract void run(); public abstract void onRowMetaData(Map columToIndx, int fieldCount) throws IOException; public void outputMergeResult(NonBlockingSession session, byte[] eof) { addPack(END_FLAG_PACK); } public RouteResultset getRrs() { return this.rrs; } /** * 做最后的结果集输出 * @return (最多i*(offset+size)行数据) */ public abstract List getResults(byte[] eof); public abstract void clear(); } ================================================ FILE: src/main/java/io/mycat/sqlengine/mpp/ColMeta.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.sqlengine.mpp; import java.io.Serializable; public class ColMeta implements Serializable{ public static final int COL_TYPE_DECIMAL = 0; public static final int COL_TYPE_INT = 1; public static final int COL_TYPE_SHORT = 2; public static final int COL_TYPE_LONG = 3; public static final int COL_TYPE_FLOAT = 4; public static final int COL_TYPE_DOUBLE = 5; public static final int COL_TYPE_NULL = 6; public static final int COL_TYPE_TIMSTAMP = 7; public static final int COL_TYPE_LONGLONG = 8; public static final int COL_TYPE_INT24 = 9; public static final int COL_TYPE_DATE = 0x0a; public static final int COL_TYPE_DATETIME=0X0C; public static final int COL_TYPE_TIME = 0x0b; public static final int COL_TYPE_YEAR = 0x0d; public static final int COL_TYPE_NEWDATE = 0x0e; public static final int COL_TYPE_VACHAR = 0x0f; public static final int COL_TYPE_BIT = 0x10; public static final int COL_TYPE_NEWDECIMAL = 0xf6; public static final int COL_TYPE_ENUM = 0xf7; public static final int COL_TYPE_SET = 0xf8; public static final int COL_TYPE_TINY_BLOB = 0xf9; public static final int COL_TYPE_TINY_TYPE_MEDIUM_BLOB = 0xfa; public static final int COL_TYPE_TINY_TYPE_LONG_BLOB = 0xfb; public static final int COL_TYPE_BLOB = 0xfc; public static final int COL_TYPE_VAR_STRING = 0xfd; public static final int COL_TYPE_STRING = 0xfe; public static final int COL_TYPE_GEOMETRY = 0xff; public int colIndex; public final int colType; public int decimals; public int avgSumIndex; public int avgCountIndex; public ColMeta(int colIndex, int colType) { super(); this.colIndex = colIndex; this.colType = colType; } public ColMeta(int avgSumIndex,int avgCountIndex, int colType) { super(); this.avgSumIndex = avgSumIndex; this.avgCountIndex=avgCountIndex; this.colType = colType; } public int getColIndex() { return colIndex; } public void setColIndex(int colIndex) { this.colIndex = colIndex; } public int getColType() { return colType; } @Override public String toString() { return "ColMeta [colIndex=" + colIndex + ", colType=" + colType + "]"; } } ================================================ FILE: src/main/java/io/mycat/sqlengine/mpp/ColumnRoutePair.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.sqlengine.mpp; /** * column ->node index * * @author wuzhih * */ public class ColumnRoutePair { public final String colValue; public final RangeValue rangeValue; public Integer nodeId; public int slot=-2; public int getSlot() { return slot; } public void setSlot(int slot) { this.slot = slot; } public ColumnRoutePair(String colValue) { super(); this.colValue = colValue; this.rangeValue = null; } public ColumnRoutePair(RangeValue rangeValue) { super(); this.rangeValue = rangeValue; this.colValue = null; } public Integer getNodeId() { return nodeId; } public void setNodeId(Integer nodeId) { this.nodeId = nodeId; } @Override public int hashCode() { final int prime = 31; int result = 1; result = prime * result + ((colValue == null) ? 0 : colValue.hashCode()); result = prime * result + ((rangeValue == null) ? 0 : rangeValue.hashCode()); result = prime * result + ((nodeId == null) ? 0 : nodeId.hashCode()); return result; } @Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (getClass() != obj.getClass()) { return false; } ColumnRoutePair other = (ColumnRoutePair) obj; if (colValue == null) { if (other.colValue != null) { return false; } } else if (!colValue.equals(other.colValue)) { return false; } if (rangeValue == null) { if (other.rangeValue != null) { return false; } } else if (!rangeValue.equals(other.rangeValue)) { return false; } if (nodeId == null) { if (other.nodeId != null) { return false; } } else if (!nodeId.equals(other.nodeId)) { return false; } return true; } @Override public String toString() { return "ColumnRoutePair [colValue=" + colValue + ", nodeId=" + nodeId + "]"; } } ================================================ FILE: src/main/java/io/mycat/sqlengine/mpp/DataMergeService.java ================================================ package io.mycat.sqlengine.mpp; /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ import com.alibaba.druid.sql.SQLUtils; import io.mycat.MycatServer; import io.mycat.backend.mysql.BufferUtil; import io.mycat.backend.mysql.nio.handler.MultiNodeQueryHandler; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.route.RouteResultset; import io.mycat.route.RouteResultsetNode; import io.mycat.server.ServerConnection; import io.mycat.sqlengine.mpp.tmp.RowDataSorter; import io.mycat.util.StringUtil; import org.apache.log4j.Logger; import java.nio.ByteBuffer; import java.util.*; import java.util.concurrent.ConcurrentHashMap; /** * Data merge service handle data Min,Max,AVG group 、order by 、limit * * @author wuzhih /modify by coder_czp/2015/11/2 * * Fixbug: mycat sql timeout and hang problem. * @author Uncle-pan * @since 2016-03-23 * */ public class DataMergeService extends AbstractDataNodeMerge { private RowDataSorter sorter; private RowDataPacketGrouper grouper; private Map> result = new HashMap>(); private static Logger LOGGER = Logger.getLogger(DataMergeService.class); private ConcurrentHashMap canDiscard = new ConcurrentHashMap(); public DataMergeService(MultiNodeQueryHandler handler, RouteResultset rrs) { super(handler,rrs); for (RouteResultsetNode node : rrs.getNodes()) { result.put(node.getName(), new LinkedList()); } } /** * @param columToIndx * @param fieldCount */ public void onRowMetaData(Map columToIndx, int fieldCount) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("field metadata keys:" + columToIndx.keySet()); LOGGER.debug("field metadata values:" + columToIndx.values()); } int[] groupColumnIndexs = null; this.fieldCount = fieldCount; if (rrs.getGroupByCols() != null) { groupColumnIndexs = toColumnIndex(rrs.getGroupByCols(), columToIndx); } if (rrs.getHavingCols() != null) { ColMeta colMeta = columToIndx.get(rrs.getHavingCols().getLeft() .toUpperCase()); if (colMeta != null) { rrs.getHavingCols().setColMeta(colMeta); } } if (rrs.isHasAggrColumn()) { List mergCols = new LinkedList(); Map mergeColsMap = rrs.getMergeCols(); if (mergeColsMap != null) { for (Map.Entry mergEntry : mergeColsMap .entrySet()) { String colName = mergEntry.getKey().toUpperCase(); int type = mergEntry.getValue(); if (MergeCol.MERGE_AVG == type) { ColMeta sumColMeta = columToIndx.get(colName + "SUM"); ColMeta countColMeta = columToIndx.get(colName + "COUNT"); if (sumColMeta != null && countColMeta != null) { ColMeta colMeta = new ColMeta(sumColMeta.colIndex, countColMeta.colIndex, sumColMeta.getColType()); colMeta.decimals = sumColMeta.decimals; // 保存精度 mergCols.add(new MergeCol(colMeta, mergEntry .getValue())); } } else { ColMeta colMeta = columToIndx.get(SQLUtils.normalize(colName)); mergCols.add(new MergeCol(colMeta, mergEntry.getValue())); } } } // add no alias merg column for (Map.Entry fieldEntry : columToIndx.entrySet()) { String colName = fieldEntry.getKey(); int result = MergeCol.tryParseAggCol(colName); if (result != MergeCol.MERGE_UNSUPPORT && result != MergeCol.MERGE_NOMERGE) { mergCols.add(new MergeCol(fieldEntry.getValue(), result)); } } grouper = new RowDataPacketGrouper(groupColumnIndexs, mergCols.toArray(new MergeCol[mergCols.size()]), rrs.getHavingCols()); } if (rrs.getOrderByCols() != null) { LinkedHashMap orders = rrs.getOrderByCols(); OrderCol[] orderCols = new OrderCol[orders.size()]; int i = 0; for (Map.Entry entry : orders.entrySet()) { String key = StringUtil.removeBackquote(entry.getKey() .toUpperCase()); ColMeta colMeta = columToIndx.get(key); if (colMeta == null) { throw new IllegalArgumentException( "all columns in order by clause should be in the selected column list!" + entry.getKey()); } orderCols[i++] = new OrderCol(colMeta, entry.getValue()); } RowDataSorter tmp = new RowDataSorter(orderCols); tmp.setLimit(rrs.getLimitStart(), rrs.getLimitSize()); sorter = tmp; } if (MycatServer.getInstance(). getConfig().getSystem(). getUseStreamOutput() == 1 && grouper == null && sorter == null) { setStreamOutputResult(true); }else { setStreamOutputResult(false); } } /** * release resources */ public void clear() { result.clear(); grouper = null; sorter = null; } @Override public void run() { // sort-or-group: no need for us to using multi-threads, because //both sorter and group are synchronized!! // @author Uncle-pan // @since 2016-03-23 if(!running.compareAndSet(false, true)){ return; } // eof handler has been placed to "if (pack == END_FLAG_PACK){}" in for-statement // @author Uncle-pan // @since 2016-03-23 boolean nulpack = false; try{ // loop-on-packs for (; ; ) { final PackWraper pack = packs.poll(); // async: handling row pack queue, this business thread should exit when no pack // @author Uncle-pan // @since 2016-03-23 if(pack == null){ nulpack = true; break; } // eof: handling eof pack and exit if (pack == END_FLAG_PACK) { final int warningCount = 0; final EOFPacket eofp = new EOFPacket(); final ByteBuffer eof = ByteBuffer.allocate(9); BufferUtil.writeUB3(eof, eofp.calcPacketSize()); eof.put(eofp.packetId); eof.put(eofp.fieldCount); BufferUtil.writeUB2(eof, warningCount); BufferUtil.writeUB2(eof, eofp.status); final ServerConnection source = multiQueryHandler.getSession().getSource(); final byte[] array = eof.array(); multiQueryHandler.outputMergeResult(source, array, getResults(array)); break; } // merge: sort-or-group, or simple add final RowDataPacket row = new RowDataPacket(fieldCount); row.read(pack.rowData); if (grouper != null) { grouper.addRow(row); } else if (sorter != null) { if (!sorter.addRow(row)) { canDiscard.put(pack.dataNode,true); } } else { result.get(pack.dataNode).add(row); } }// rof }catch(final Exception e){ multiQueryHandler.handleDataProcessException(e); }finally{ running.set(false); } // try to check packs, it's possible that adding a pack after polling a null pack //and before this time pointer!! // @author Uncle-pan // @since 2016-03-23 if(nulpack && !packs.isEmpty()){ this.run(); } } /** * return merged data * @return (最多i*(offset+size)行数据) */ public List getResults(byte[] eof) { List tmpResult = null; if (this.grouper != null) { tmpResult = grouper.getResult(); grouper = null; } if (sorter != null) { if (tmpResult != null) { Iterator itor = tmpResult.iterator(); while (itor.hasNext()) { sorter.addRow(itor.next()); itor.remove(); } } tmpResult = sorter.getSortedResult(); sorter = null; } //no grouper and sorter if(tmpResult == null){ tmpResult = new LinkedList<>(); /** * 每次移除dataNode,防止一个dataNode重复发送多次结果集 */ for (RouteResultsetNode node : rrs.getNodes()) { LinkedList remove = result.remove(node.getName()); if (remove != null){ tmpResult.addAll(remove); } } } if (LOGGER.isDebugEnabled()) { LOGGER.debug("prepare mpp merge result for " + rrs.getStatement()); } return tmpResult; } } ================================================ FILE: src/main/java/io/mycat/sqlengine/mpp/DataNodeMergeManager.java ================================================ package io.mycat.sqlengine.mpp; import io.mycat.MycatServer; import io.mycat.backend.mysql.BufferUtil; import io.mycat.backend.mysql.MySQLMessage; import io.mycat.backend.mysql.nio.handler.MultiNodeQueryHandler; import io.mycat.memory.MyCatMemory; import io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager; import io.mycat.memory.unsafe.memory.mm.MemoryManager; import io.mycat.memory.unsafe.row.BufferHolder; import io.mycat.memory.unsafe.row.StructType; import io.mycat.memory.unsafe.row.UnsafeRow; import io.mycat.memory.unsafe.row.UnsafeRowWriter; import io.mycat.memory.unsafe.utils.MycatPropertyConf; import io.mycat.memory.unsafe.utils.sort.PrefixComparator; import io.mycat.memory.unsafe.utils.sort.PrefixComparators; import io.mycat.memory.unsafe.utils.sort.RowPrefixComputer; import io.mycat.memory.unsafe.utils.sort.UnsafeExternalRowSorter; import io.mycat.net.mysql.EOFPacket; import io.mycat.net.mysql.RowDataPacket; import io.mycat.route.RouteResultset; import io.mycat.server.ServerConnection; import io.mycat.util.StringUtil; import org.apache.log4j.Logger; import java.io.IOException; import java.nio.ByteBuffer; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; /** * Created by zagnix on 2016/6/21. */ public class DataNodeMergeManager extends AbstractDataNodeMerge { private static Logger LOGGER = Logger.getLogger(DataNodeMergeManager.class); /** * key为datanode的分片节点名字 * value为对应的排序器 * 目前，没有使用！ */ private ConcurrentHashMap unsafeRows = new ConcurrentHashMap(); /** * 全局sorter，排序器 */ private UnsafeExternalRowSorter globalSorter = null; /** * UnsafeRowGrouper */ private UnsafeRowGrouper unsafeRowGrouper = null; /** * 全局merge，排序器 */ private UnsafeExternalRowSorter globalMergeResult = null; /** * sorter需要的上下文环境 */ private final MyCatMemory myCatMemory; private final MemoryManager memoryManager; private final MycatPropertyConf conf; /** * Limit N，M */ private final int limitStart; private final int limitSize; private int[] mergeColsIndex; private boolean hasEndFlag = false; private AtomicBoolean isMiddleResultDone; public DataNodeMergeManager(MultiNodeQueryHandler handler, RouteResultset rrs,AtomicBoolean isMiddleResultDone) { super(handler,rrs); this.isMiddleResultDone = isMiddleResultDone; this.myCatMemory = MycatServer.getInstance().getMyCatMemory(); this.memoryManager = myCatMemory.getResultMergeMemoryManager(); this.conf = myCatMemory.getConf(); this.limitStart = rrs.getLimitStart(); this.limitSize = rrs.getLimitSize(); } public void onRowMetaData(Map columToIndx, int fieldCount) throws IOException { if (LOGGER.isDebugEnabled()) { LOGGER.debug("field metadata keys:" + columToIndx != null ? columToIndx.keySet() : "null"); LOGGER.debug("field metadata values:" + columToIndx != null ? columToIndx.values() : "null"); } OrderCol[] orderCols = null; StructType schema = null; UnsafeExternalRowSorter.PrefixComputer prefixComputer = null; PrefixComparator prefixComparator = null; DataNodeMemoryManager dataNodeMemoryManager = null; UnsafeExternalRowSorter sorter = null; int[] groupColumnIndexs = null; this.fieldCount = fieldCount; if (rrs.getGroupByCols() != null) { groupColumnIndexs = toColumnIndex(rrs.getGroupByCols(), columToIndx); if (LOGGER.isDebugEnabled()) { for (int i = 0; i mergCols = new LinkedList(); Map mergeColsMap = rrs.getMergeCols(); if (mergeColsMap != null) { if (LOGGER.isDebugEnabled() && rrs.getMergeCols() != null) { LOGGER.debug("isHasAggrColumn:" + rrs.getMergeCols().toString()); } for (Map.Entry mergEntry : mergeColsMap .entrySet()) { String colName = mergEntry.getKey().toUpperCase(); int type = mergEntry.getValue(); if (MergeCol.MERGE_AVG == type) { ColMeta sumColMeta = columToIndx.get(colName + "SUM"); ColMeta countColMeta = columToIndx.get(colName + "COUNT"); if (sumColMeta != null && countColMeta != null) { ColMeta colMeta = new ColMeta(sumColMeta.colIndex, countColMeta.colIndex, sumColMeta.getColType()); mergCols.add(new MergeCol(colMeta, mergEntry .getValue())); } } else { ColMeta colMeta = columToIndx.get(colName); mergCols.add(new MergeCol(colMeta, mergEntry.getValue())); } } } // add no alias merg column for (Map.Entry fieldEntry : columToIndx.entrySet()) { String colName = fieldEntry.getKey(); int result = MergeCol.tryParseAggCol(colName); if (result != MergeCol.MERGE_UNSUPPORT && result != MergeCol.MERGE_NOMERGE) { mergCols.add(new MergeCol(fieldEntry.getValue(), result)); } } /** * Group操作 */ MergeCol[] mergColsArrays = mergCols.toArray(new MergeCol[mergCols.size()]); unsafeRowGrouper = new UnsafeRowGrouper(columToIndx,rrs.getGroupByCols(), mergColsArrays, rrs.getHavingCols()); if(mergColsArrays!=null&&mergColsArrays.length>0){ mergeColsIndex = new int[mergColsArrays.length]; for(int i = 0;i orders = rrs.getOrderByCols(); orderCols = new OrderCol[orders.size()]; int i = 0; for (Map.Entry entry : orders.entrySet()) { String key = StringUtil.removeBackquote(entry.getKey() .toUpperCase()); ColMeta colMeta = columToIndx.get(key); if (colMeta == null) { throw new IllegalArgumentException( "all columns in order by clause should be in the selected column list!" + entry.getKey()); } orderCols[i++] = new OrderCol(colMeta, entry.getValue()); } /** * 构造全局排序器 */ schema = new StructType(columToIndx,fieldCount); schema.setOrderCols(orderCols); prefixComputer = new RowPrefixComputer(schema); // if(orderCols.length>0 // && orderCols[0].getOrderType() // == OrderCol.COL_ORDER_TYPE_ASC){ // prefixComparator = PrefixComparators.LONG; // }else { // prefixComparator = PrefixComparators.LONG_DESC; // } prefixComparator = getPrefixComparator(orderCols); dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager,Thread.currentThread().getId()); /** * 默认排序，只是将数据连续存储到内存中即可。 */ globalSorter = new UnsafeExternalRowSorter( dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize","32k"), false/**是否使用基数排序*/, true/**排序*/); } if(conf.getBoolean("mycat.stream.output.result",false) && globalSorter == null && unsafeRowGrouper == null){ setStreamOutputResult(true); }else { /** * 1.schema */ schema = new StructType(columToIndx,fieldCount); schema.setOrderCols(orderCols); /** * 2 .PrefixComputer */ prefixComputer = new RowPrefixComputer(schema); /** * 3 .PrefixComparator 默认是ASC，可以选择DESC */ prefixComparator = PrefixComparators.LONG; dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId()); globalMergeResult = new UnsafeExternalRowSorter( dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize", "32k"), false,/**是否使用基数排序*/ false/**不排序*/); } } private PrefixComparator getPrefixComparator(OrderCol[] orderCols) { PrefixComparator prefixComparator = null; OrderCol firstOrderCol = orderCols[0]; int orderType = firstOrderCol.getOrderType(); int colType = firstOrderCol.colMeta.colType; switch (colType) { case ColMeta.COL_TYPE_INT: case ColMeta.COL_TYPE_LONG: case ColMeta.COL_TYPE_INT24: case ColMeta.COL_TYPE_SHORT: case ColMeta.COL_TYPE_LONGLONG: prefixComparator = (orderType == OrderCol.COL_ORDER_TYPE_ASC ? PrefixComparators.LONG : PrefixComparators.LONG_DESC); break; case ColMeta.COL_TYPE_FLOAT: case ColMeta.COL_TYPE_DOUBLE: case ColMeta.COL_TYPE_DECIMAL: case ColMeta.COL_TYPE_NEWDECIMAL: prefixComparator = (orderType == OrderCol.COL_ORDER_TYPE_ASC ? PrefixComparators.DOUBLE : PrefixComparators.DOUBLE_DESC); break; case ColMeta.COL_TYPE_DATE: case ColMeta.COL_TYPE_TIMSTAMP: case ColMeta.COL_TYPE_TIME: case ColMeta.COL_TYPE_YEAR: case ColMeta.COL_TYPE_DATETIME: case ColMeta.COL_TYPE_NEWDATE: case ColMeta.COL_TYPE_BIT: case ColMeta.COL_TYPE_VAR_STRING: case ColMeta.COL_TYPE_STRING: case ColMeta.COL_TYPE_ENUM: case ColMeta.COL_TYPE_SET: prefixComparator = (orderType == OrderCol.COL_ORDER_TYPE_ASC ? PrefixComparators.BINARY : PrefixComparators.BINARY_DESC); break; default: prefixComparator = (orderType == OrderCol.COL_ORDER_TYPE_ASC ? PrefixComparators.LONG : PrefixComparators.LONG_DESC); break; } return prefixComparator; } @Override public List getResults(byte[] eof) { return null; } private UnsafeRow unsafeRow = null; private BufferHolder bufferHolder = null; private UnsafeRowWriter unsafeRowWriter = null; private int Index = 0; private volatile int clearStatus = ClearStatusEnum.INIT; @Override public void run() { if (!running.compareAndSet(false, true)) { return; } boolean nulpack = false; try { for (; ; ) { if(clearStatus == ClearStatusEnum.PREPARE_CLEAR || clearStatus == ClearStatusEnum.CLEARED) { break; } final PackWraper pack = packs.poll(); if (pack == null) { nulpack = true; break; } if (pack == END_FLAG_PACK) { hasEndFlag = true; if(packs.peek()!=null){ packs.add(pack); continue; } /** * 最后一个节点datenode发送了row eof packet说明了整个 * 分片数据全部接收完成，进而将结果集全部发给你Mycat 客户端 */ final int warningCount = 0; final EOFPacket eofp = new EOFPacket(); final ByteBuffer eof = ByteBuffer.allocate(9); BufferUtil.writeUB3(eof, eofp.calcPacketSize()); eof.put(eofp.packetId); eof.put(eofp.fieldCount); BufferUtil.writeUB2(eof,warningCount); BufferUtil.writeUB2(eof,eofp.status); final ServerConnection source = multiQueryHandler.getSession().getSource(); final byte[] array = eof.array(); Iterator iters = null; if (unsafeRowGrouper != null){ /** * group by里面需要排序情况 */ if (globalSorter != null){ iters = unsafeRowGrouper.getResult(globalSorter); }else { iters = unsafeRowGrouper.getResult(globalMergeResult); } }else if(globalSorter != null){ iters = globalSorter.sort(); }else if (!isStreamOutputResult){ iters = globalMergeResult.sort(); } if(iters != null){ multiQueryHandler.outputMergeResult(source,array,iters,isMiddleResultDone); } break; } unsafeRow = new UnsafeRow(fieldCount); bufferHolder = new BufferHolder(unsafeRow,0); unsafeRowWriter = new UnsafeRowWriter(bufferHolder,fieldCount); bufferHolder.reset(); /** *构造一行row，将对应的col填充. */ MySQLMessage mm = new MySQLMessage(pack.rowData); unsafeRowWriter.grow((mm.getRowLength(fieldCount) + 1 ) / 2); mm.readUB3(); mm.read(); int nullnum = 0; for (int i = 0; i < fieldCount; i++) { byte[] colValue = mm.readBytesWithLength(); if (colValue != null) unsafeRowWriter.write(i,colValue); else { if(mergeColsIndex!=null&&mergeColsIndex.length>0){ if(Arrays.binarySearch(mergeColsIndex, i)<0){ nullnum++; } } unsafeRow.setNullAt(i); } } if(mergeColsIndex!=null&&mergeColsIndex.length>0){ if(nullnum == (fieldCount - mergeColsIndex.length)){ if(!hasEndFlag){ packs.add(pack); continue; } } } unsafeRow.setTotalSize(bufferHolder.totalSize()); if(unsafeRowGrouper != null){ unsafeRowGrouper.addRow(unsafeRow); }else if (globalSorter != null){ globalSorter.insertRow(unsafeRow); }else { globalMergeResult.insertRow(unsafeRow); } unsafeRow = null; bufferHolder = null; unsafeRowWriter = null; } } catch (final Exception e) { e.printStackTrace(); multiQueryHandler.handleDataProcessException(e); } finally { synchronized (this) { running.set(false); if(clearStatus == ClearStatusEnum.PREPARE_CLEAR){ clear(); return ; } } if (nulpack && !packs.isEmpty()) { this.run(); } } } /** * 释放DataNodeMergeManager所申请的资源 */ public void clear() { if(clearStatus == ClearStatusEnum.INIT) { synchronized (this){ if(clearStatus == ClearStatusEnum.INIT && running.get() == true ) { clearStatus = ClearStatusEnum.PREPARE_CLEAR; } } } boolean flag = false; synchronized (this) { if(clearStatus == ClearStatusEnum.CLEARED || running.get() == true){ return; } clearStatus = ClearStatusEnum.CLEARED; flag = true; } if(!flag){ return ; } unsafeRows.clear(); if (unsafeRowGrouper != null) { unsafeRowGrouper.free(); unsafeRowGrouper = null; } if(globalSorter != null){ globalSorter.cleanupResources(); globalSorter = null; } if (globalMergeResult != null){ globalMergeResult.cleanupResources(); globalMergeResult = null; } } } class ClearStatusEnum { public static int INIT = -1; public static int PREPARE_CLEAR = 1; public static int CLEARED = 2; } ================================================ FILE: src/main/java/io/mycat/sqlengine/mpp/HavingCols.java ================================================ package io.mycat.sqlengine.mpp; import java.io.Serializable; /** * Created by v1.lion on 2015/6/10. */ public class HavingCols implements Serializable { String left; String right; String operator; public ColMeta colMeta; public HavingCols(String left, String right, String operator) { this.left = left; this.right = right; this.operator = operator; } public String getLeft() { return left; } public void setLeft(String left) { this.left = left; } public String getRight() { return right; } public void setRight(String right) { this.right = right; } public String getOperator() { return operator; } public void setOperator(String operator) { this.operator = operator; } public ColMeta getColMeta() { return colMeta; } public void setColMeta(ColMeta colMeta) { this.colMeta = colMeta; } } ================================================ FILE: src/main/java/io/mycat/sqlengine/mpp/LoadData.java ================================================ package io.mycat.sqlengine.mpp; import java.io.Serializable; import java.util.List; /** * Created by magicdoom on 2015/3/30. */ public class LoadData implements Serializable { public static final String loadDataHint="/*loaddata*/"; private boolean isLocal; private List data; private String fileName; private String charset; private String lineTerminatedBy; private String fieldTerminatedBy; private String enclose; private String escape; public String getEscape() { return escape; } public void setEscape(String escape) { this.escape = escape; } public boolean isLocal() { return isLocal; } public void setLocal(boolean isLocal) { this.isLocal = isLocal; } public List getData() { return data; } public void setData(List data) { this.data = data; } public String getFileName() { return fileName; } public void setFileName(String fileName) { this.fileName = fileName; } public String getCharset() { return charset; } public void setCharset(String charset) { this.charset = charset; } public String getLineTerminatedBy() { return lineTerminatedBy; } public void setLineTerminatedBy(String lineTerminatedBy) { this.lineTerminatedBy = lineTerminatedBy; } public String getFieldTerminatedBy() { return fieldTerminatedBy; } public void setFieldTerminatedBy(String fieldTerminatedBy) { this.fieldTerminatedBy = fieldTerminatedBy; } public String getEnclose() { return enclose; } public void setEnclose(String enclose) { this.enclose = enclose; } } ================================================ FILE: src/main/java/io/mycat/sqlengine/mpp/MergeCol.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.sqlengine.mpp; public class MergeCol { public static final int MERGE_COUNT = 1; public static final int MERGE_SUM = 2; public static final int MERGE_MIN = 3; public static final int MERGE_MAX = 4; public static final int MERGE_AVG= 5; public static final int MERGE_UNSUPPORT = -1; public static final int MERGE_NOMERGE = -2; public final int mergeType; public final ColMeta colMeta; public MergeCol(ColMeta colMeta, int mergeType) { super(); this.colMeta = colMeta; this.mergeType = mergeType; } public static int getMergeType(String mergeType) { String upper=mergeType.toUpperCase(); if (upper.startsWith("COUNT")) { return MERGE_COUNT; } else if (upper.startsWith("SUM")) { return MERGE_SUM; } else if (upper.startsWith("MIN")) { return MERGE_MIN; } else if (upper.startsWith("MAX")) { return MERGE_MAX; } else if (upper.startsWith("AVG")) { return MERGE_AVG; } else { return MERGE_UNSUPPORT; } } public static int tryParseAggCol(String column) { // MIN(*),MAX(*),COUNT(*),SUM if (column.length() < 6) { return -1; } column = column.toUpperCase(); if (column.startsWith("COUNT(")) { return MERGE_COUNT; } else if (column.startsWith("SUM(")) { return MERGE_SUM; } else if (column.startsWith("MIN(")) { return MERGE_MIN; } else if (column.startsWith("MAX(")) { return MERGE_MAX; } else if (column.startsWith("AVG(")) { return MERGE_AVG; } else { return MERGE_NOMERGE; } } } ================================================ FILE: src/main/java/io/mycat/sqlengine/mpp/OrderCol.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.sqlengine.mpp; public class OrderCol { public final int orderType; public final ColMeta colMeta; public static final int COL_ORDER_TYPE_ASC = 0; // ASC public static final int COL_ORDER_TYPE_DESC = 1; // DESC public OrderCol(ColMeta colMeta, int orderType) { super(); this.colMeta = colMeta; this.orderType = orderType; } public int getOrderType() { return orderType; } public ColMeta getColMeta() { return colMeta; } } ================================================ FILE: src/main/java/io/mycat/sqlengine/mpp/PackWraper.java ================================================ package io.mycat.sqlengine.mpp; /** * Created by zagnix on 2016/7/6. */ /** * 一行数据是从哪个节点来的。 * 通过dataNode查找对应的sorter， * 将数据放到对应的datanode的sorter， * 进行排序. */ public final class PackWraper { public byte[] rowData; public String dataNode; } ================================================ FILE: src/main/java/io/mycat/sqlengine/mpp/RangRowDataPacketSorter.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.sqlengine.mpp; import io.mycat.net.mysql.RowDataPacket; import io.mycat.sqlengine.mpp.tmp.RowDataSorter; public class RangRowDataPacketSorter extends RowDataSorter { public RangRowDataPacketSorter(OrderCol[] orderCols) { super(orderCols); } public boolean ascDesc(int byColumnIndex) { if (this.orderCols[byColumnIndex].orderType == OrderCol.COL_ORDER_TYPE_ASC) {// 升序 return true; } return false; } public int compareRowData(RowDataPacket l, RowDataPacket r, int byColumnIndex) { byte[] left = l.fieldValues.get(this.orderCols[byColumnIndex].colMeta.colIndex); byte[] right = r.fieldValues.get(this.orderCols[byColumnIndex].colMeta.colIndex); return compareObject(left, right, this.orderCols[byColumnIndex]); } } ================================================ FILE: src/main/java/io/mycat/sqlengine/mpp/RangeValue.java ================================================ package io.mycat.sqlengine.mpp; public class RangeValue { /* * 左值不包含 右值包含 */ public static final Integer NE = 0; public static final Integer EE = 1; public static final Integer EN = 2; public static final Integer NN = 3; public Object beginValue; public Object endValue; public Integer rangeType; public RangeValue(Object beginValue, Object endValue, Integer rangeType) { super(); this.beginValue = beginValue; this.endValue = endValue; this.rangeType = rangeType; } @Override public int hashCode(){ int hash = 0; hash = beginValue.hashCode(); hash = hash*31+endValue.hashCode(); hash = hash*31+rangeType; return hash; } @Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (getClass() != obj.getClass()) { return false; } RangeValue other = (RangeValue) obj; if( beginValue == null ){ if(other.beginValue != null){ return false; } }else if( !beginValue.equals(other.beginValue) ){ return false; } if( endValue == null ){ if(other.endValue != null){ return false; } }else if( !endValue.equals(other.endValue) ){ return false; } if( rangeType == null ){ if(other.rangeType != null){ return false; } }else if( !rangeType.equals(other.rangeType) ){ return false; } return true; } } ================================================ FILE: src/main/java/io/mycat/sqlengine/mpp/RowDataPacketGrouper.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.sqlengine.mpp; import java.math.BigDecimal; import java.math.RoundingMode; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.TreeSet; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.ByteUtil; import io.mycat.util.CompareUtil; import io.mycat.util.LongUtil; /** * implement group function select a,count(*),sum(*) from A group by a * * @author wuzhih * */ public class RowDataPacketGrouper { private List result = Collections.synchronizedList(new ArrayList()); private final MergeCol[] mergCols; private int[] mergeColsIndex; private final int[] groupColumnIndexs; private boolean ishanlderFirstRow = false; //结果集汇聚时,是否已处理第一条记录. private boolean isMergAvg=false; private HavingCols havingCols; public RowDataPacketGrouper(int[] groupColumnIndexs, MergeCol[] mergCols,HavingCols havingCols) { super(); this.groupColumnIndexs = groupColumnIndexs; this.mergCols = mergCols; this.havingCols = havingCols; if(mergCols!=null&&mergCols.length>0){ mergeColsIndex = new int[mergCols.length]; for(int i = 0;i getResult() { if(!isMergAvg) { for (RowDataPacket row : result) { mergAvg(row); } isMergAvg=true; } if(havingCols != null){ filterHaving(); } return result; } private void filterHaving(){ if (havingCols.getColMeta() == null || result == null) { return; } Iterator it = result.iterator(); byte[] right = havingCols.getRight().getBytes( StandardCharsets.UTF_8); int index = havingCols.getColMeta().getColIndex(); int colType = havingCols.getColMeta().getColType(); // Added by winbill. 20160312. while (it.hasNext()){ RowDataPacket rowDataPacket = it.next(); switch (havingCols.getOperator()) { case "=": /* Add parameter of colType, Modified by winbill. 20160312. */ if (eq(rowDataPacket.fieldValues.get(index),right,colType)) { it.remove(); } break; case ">": /* Add parameter of colType, Modified by winbill. 20160312. */ if (gt(rowDataPacket.fieldValues.get(index),right,colType)) { it.remove(); } break; case "<": /* Add parameter of colType, Modified by winbill. 20160312. */ if (lt(rowDataPacket.fieldValues.get(index),right,colType)) { it.remove(); } break; case ">=": /* Add parameter of colType, Modified by winbill. 20160312. */ if (gt(rowDataPacket.fieldValues.get(index),right,colType) && eq(rowDataPacket.fieldValues.get(index),right,colType)) { it.remove(); } break; case "<=": /* Add parameter of colType, Modified by winbill. 20160312. */ if (lt(rowDataPacket.fieldValues.get(index),right,colType) && eq(rowDataPacket.fieldValues.get(index),right,colType)) { it.remove(); } break; case "!=": /* Add parameter of colType, Modified by winbill. 20160312. */ if (neq(rowDataPacket.fieldValues.get(index),right,colType)) { it.remove(); } break; } } } /* * Using new compare function instead of compareNumberByte * Modified by winbill. 20160312. */ private boolean lt(byte[] l, byte[] r, final int colType) { // return -1 != ByteUtil.compareNumberByte(l, r); return -1 != RowDataPacketGrouper.compareObject(l, r, colType); } private boolean gt(byte[] l, byte[] r, final int colType) { // return 1 != ByteUtil.compareNumberByte(l, r, havingCol); return 1 != RowDataPacketGrouper.compareObject(l, r, colType); } private boolean eq(byte[] l, byte[] r, final int colType) { // return 0 != ByteUtil.compareNumberByte(l, r, havingCol); return 0 != RowDataPacketGrouper.compareObject(l, r, colType); } private boolean neq(byte[] l, byte[] r, final int colType) { // return 0 == ByteUtil.compareNumberByte(l, r, havingCol); return 0 == RowDataPacketGrouper.compareObject(l, r, colType); } /* * Compare with the value of having column * winbill. 20160312. */ public static final int compareObject(byte[] left,byte[] right, final int colType) { switch (colType) { case ColMeta.COL_TYPE_SHORT: case ColMeta.COL_TYPE_INT: case ColMeta.COL_TYPE_INT24: case ColMeta.COL_TYPE_LONG: return CompareUtil.compareInt(ByteUtil.getInt(left), ByteUtil.getInt(right)); case ColMeta.COL_TYPE_LONGLONG: return CompareUtil.compareLong(ByteUtil.getLong(left), ByteUtil.getLong(right)); case ColMeta.COL_TYPE_FLOAT: case ColMeta.COL_TYPE_DOUBLE: case ColMeta.COL_TYPE_DECIMAL: case ColMeta.COL_TYPE_NEWDECIMAL: return CompareUtil.compareDouble(ByteUtil.getDouble(left), ByteUtil.getDouble(right)); case ColMeta.COL_TYPE_DATE: case ColMeta.COL_TYPE_TIMSTAMP: case ColMeta.COL_TYPE_TIME: case ColMeta.COL_TYPE_YEAR: case ColMeta.COL_TYPE_DATETIME: case ColMeta.COL_TYPE_NEWDATE: case ColMeta.COL_TYPE_BIT: case ColMeta.COL_TYPE_VAR_STRING: case ColMeta.COL_TYPE_STRING: // ENUM和SET类型都是字符串，按字符串处理 case ColMeta.COL_TYPE_ENUM: case ColMeta.COL_TYPE_SET: return ByteUtil.compareNumberByte(left, right); // BLOB相关类型和GEOMETRY类型不支持排序，略掉 } return 0; } public void addRow(RowDataPacket rowDataPkg) { for (RowDataPacket row : result) { if (sameGropuColums(rowDataPkg, row)) { aggregateRow(row, rowDataPkg); return; } } // not aggreated ,insert new result.add(rowDataPkg); } private void aggregateRow(RowDataPacket toRow, RowDataPacket newRow) { if (mergCols == null) { return; } /* * 这里进行一次判断, 在跨分片聚合的情况下,如果有一个没有记录的分片，最先返回,可能返回有null 的情况. */ if(!ishanlderFirstRow&&mergeColsIndex!=null&&mergeColsIndex.length>0){ List values = toRow.fieldValues; for(int i=0;i=0){ continue; } if(values.get(i)==null){ values.set(i, newRow.fieldValues.get(i)); } } ishanlderFirstRow = true; } for (MergeCol merg : mergCols) { if(merg.mergeType!=MergeCol.MERGE_AVG) { byte[] result = mertFields( toRow.fieldValues.get(merg.colMeta.colIndex), newRow.fieldValues.get(merg.colMeta.colIndex), merg.colMeta.colType, merg.mergeType); if (result != null) { toRow.fieldValues.set(merg.colMeta.colIndex, result); } } } } private void mergAvg(RowDataPacket toRow) { if (mergCols == null) { return; } TreeSet rmIndexSet = new TreeSet(); for (MergeCol merg : mergCols) { if(merg.mergeType==MergeCol.MERGE_AVG) { byte[] result = mertFields( toRow.fieldValues.get(merg.colMeta.avgSumIndex), toRow.fieldValues.get(merg.colMeta.avgCountIndex), merg.colMeta.colType, merg.mergeType); if (result != null) { toRow.fieldValues.set(merg.colMeta.avgSumIndex, result); // toRow.fieldValues.remove(merg.colMeta.avgCountIndex) ; // toRow.fieldCount=toRow.fieldCount-1; rmIndexSet.add(merg.colMeta.avgCountIndex); } } } // remove by index from large to small, to make sure each element deleted correctly for(int index : rmIndexSet.descendingSet()) { toRow.fieldValues.remove(index); toRow.fieldCount = toRow.fieldCount - 1; } } private byte[] mertFields(byte[] bs, byte[] bs2, int colType, int mergeType) { // System.out.println("mergeType:"+ mergeType+" colType "+colType+ // " field:"+Arrays.toString(bs)+ " -> "+Arrays.toString(bs2)); if(bs2==null || bs2.length==0) { return bs; }else if(bs==null || bs.length==0) { return bs2; } switch (mergeType) { case MergeCol.MERGE_SUM: if (colType == ColMeta.COL_TYPE_DOUBLE || colType == ColMeta.COL_TYPE_FLOAT) { Double vale = ByteUtil.getDouble(bs) + ByteUtil.getDouble(bs2); return vale.toString().getBytes(); // return String.valueOf(vale).getBytes(); } else if(colType == ColMeta.COL_TYPE_NEWDECIMAL || colType == ColMeta.COL_TYPE_DECIMAL) { BigDecimal d1 = new BigDecimal(new String(bs)); d1 = d1.add(new BigDecimal(new String(bs2))); return String.valueOf(d1).getBytes(); } // continue to count case case MergeCol.MERGE_COUNT: { long s1 = Long.parseLong(new String(bs)); long s2 = Long.parseLong(new String(bs2)); long total = s1 + s2; return LongUtil.toBytes(total); } case MergeCol.MERGE_MAX: { // System.out.println("value:"+ // ByteUtil.getNumber(bs).doubleValue()); // System.out.println("value2:"+ // ByteUtil.getNumber(bs2).doubleValue()); // int compare = CompareUtil.compareDouble(ByteUtil.getNumber(bs) // .doubleValue(), ByteUtil.getNumber(bs2).doubleValue()); // return ByteUtil.compareNumberByte(bs, bs2); int compare = ByteUtil.compareNumberByte(bs, bs2); return (compare > 0) ? bs : bs2; } case MergeCol.MERGE_MIN: { // int compare = CompareUtil.compareDouble(ByteUtil.getNumber(bs) // .doubleValue(), ByteUtil.getNumber(bs2).doubleValue()); // int compare = ByteUtil.compareNumberArray(bs, bs2); //return (compare > 0) ? bs2 : bs; int compare = ByteUtil.compareNumberByte(bs, bs2); return (compare > 0) ? bs2 : bs; // return ByteUtil.compareNumberArray2(bs, bs2, 2); } case MergeCol.MERGE_AVG: { if (colType == ColMeta.COL_TYPE_DOUBLE || colType == ColMeta.COL_TYPE_FLOAT) { double aDouble = ByteUtil.getDouble(bs); long s2 = Long.parseLong(new String(bs2)); Double vale = aDouble / s2; return vale.toString().getBytes(); } else if(colType == ColMeta.COL_TYPE_NEWDECIMAL || colType == ColMeta.COL_TYPE_DECIMAL) { BigDecimal sum = new BigDecimal(new String(bs)); // mysql avg 处理精度为 sum结果的精度扩展4, 采用四舍五入 BigDecimal avg = sum.divide(new BigDecimal(new String(bs2)), sum.scale() + 4, RoundingMode.HALF_UP); return avg.toString().getBytes(); } } default: return null; } } // private static final private boolean sameGropuColums(RowDataPacket newRow, RowDataPacket existRow) { if (groupColumnIndexs == null) {// select count(*) from aaa , or group // column return true; } for (int i = 0; i < groupColumnIndexs.length; i++) { if (!Arrays.equals(newRow.fieldValues.get(groupColumnIndexs[i]), existRow.fieldValues.get(groupColumnIndexs[i]))) { return false; } } return true; } } ================================================ FILE: src/main/java/io/mycat/sqlengine/mpp/RowDataPacketSorter.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.sqlengine.mpp; import java.util.Collection; import java.util.Collections; import java.util.concurrent.ConcurrentLinkedQueue; import io.mycat.memory.unsafe.utils.BytesTools; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.net.mysql.RowDataPacket; import io.mycat.util.ByteUtil; public class RowDataPacketSorter { private static final Logger LOGGER = LoggerFactory.getLogger(RowDataPacketSorter.class); protected final OrderCol[] orderCols; private Collection sorted = new ConcurrentLinkedQueue(); private RowDataPacket[] array, resultTemp; private int p1, pr, p2; public RowDataPacketSorter(OrderCol[] orderCols) { super(); this.orderCols = orderCols; } public boolean addRow(RowDataPacket row) { return this.sorted.add(row); } public Collection getSortedResult() { try { this.mergeSort(sorted.toArray(new RowDataPacket[sorted.size()])); } catch (Exception e) { LOGGER.error("getSortedResultError",e); } if (array != null) { Collections.addAll(this.sorted, array); } return sorted; } private RowDataPacket[] mergeSort(RowDataPacket[] result) throws Exception { this.sorted.clear(); array = result; if (result == null || result.length < 2 || this.orderCols == null || orderCols.length < 1) { return result; } mergeR(0, result.length - 1); return array; } private void mergeR(int startIndex, int endIndex) { if (startIndex < endIndex) { int mid = (startIndex + endIndex) / 2; mergeR(startIndex, mid); mergeR(mid + 1, endIndex); merge(startIndex, mid, endIndex); } } private void merge(int startIndex, int midIndex, int endIndex) { resultTemp = new RowDataPacket[(endIndex - startIndex + 1)]; pr = 0; p1 = startIndex; p2 = midIndex + 1; while (p1 <= midIndex || p2 <= endIndex) { if (p1 == midIndex + 1) { while (p2 <= endIndex) { resultTemp[pr++] = array[p2++]; } } else if (p2 == endIndex + 1) { while (p1 <= midIndex) { resultTemp[pr++] = array[p1++]; } } else { compare(0); } } for (p1 = startIndex, p2 = 0; p1 <= endIndex; p1++, p2++) { array[p1] = resultTemp[p2]; } } /** * 递归按照排序字段进行排序 * * @param byColumnIndex */ private void compare(int byColumnIndex) { if (byColumnIndex == this.orderCols.length) { if (this.orderCols[byColumnIndex - 1].orderType == OrderCol.COL_ORDER_TYPE_ASC) { resultTemp[pr++] = array[p1++]; } else { resultTemp[pr++] = array[p2++]; } return; } byte[] left = array[p1].fieldValues.get(this.orderCols[byColumnIndex].colMeta.colIndex); byte[] right = array[p2].fieldValues.get(this.orderCols[byColumnIndex].colMeta.colIndex); if (compareObject(left, right, this.orderCols[byColumnIndex]) <= 0) { if (compareObject(left, right, this.orderCols[byColumnIndex]) < 0) { if (this.orderCols[byColumnIndex].orderType == OrderCol.COL_ORDER_TYPE_ASC) {// 升序 resultTemp[pr++] = array[p1++]; } else { resultTemp[pr++] = array[p2++]; } } else {// 如果当前字段相等，则按照下一个字段排序 compare(byColumnIndex + 1); } } else { if (this.orderCols[byColumnIndex].orderType == OrderCol.COL_ORDER_TYPE_ASC) {// 升序 resultTemp[pr++] = array[p2++]; } else { resultTemp[pr++] = array[p1++]; } } } public static final int compareObject(Object l, Object r, OrderCol orderCol) { return compareObject(( byte[])l, (byte[])r, orderCol); } public static final int compareObject(byte[] left,byte[] right, OrderCol orderCol) { int colType = orderCol.getColMeta().getColType(); switch (colType) { case ColMeta.COL_TYPE_DECIMAL: case ColMeta.COL_TYPE_FLOAT: case ColMeta.COL_TYPE_DOUBLE: case ColMeta.COL_TYPE_NEWDECIMAL: // 因为mysql的日期也是数字字符串方式表达，因此可以跟整数等一起对待 return ByteUtil.compareDouble(left, right); case ColMeta.COL_TYPE_INT: case ColMeta.COL_TYPE_SHORT: case ColMeta.COL_TYPE_LONG: case ColMeta.COL_TYPE_LONGLONG: case ColMeta.COL_TYPE_INT24: case ColMeta.COL_TYPE_DATE: case ColMeta.COL_TYPE_TIMSTAMP: case ColMeta.COL_TYPE_TIME: case ColMeta.COL_TYPE_YEAR: case ColMeta.COL_TYPE_NEWDATE: case ColMeta.COL_TYPE_BIT: // return BytesTools.compareTo(left,right); return ByteUtil.compareNumberByte(left, right); case ColMeta.COL_TYPE_VAR_STRING: case ColMeta.COL_TYPE_STRING: // ENUM和SET类型都是字符串，按字符串处理 case ColMeta.COL_TYPE_ENUM: case ColMeta.COL_TYPE_SET: //MySQL与SQL Server的DateTime格式不同 需按字符串处理 case ColMeta.COL_TYPE_DATETIME: return BytesTools.compareTo(left,right); // BLOB相关类型和GEOMETRY类型不支持排序，略掉 } return 0; } } ================================================ FILE: src/main/java/io/mycat/sqlengine/mpp/UnsafeRowGrouper.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.sqlengine.mpp; import io.mycat.MycatServer; import io.mycat.memory.MyCatMemory; import io.mycat.memory.unsafe.KVIterator; import io.mycat.memory.unsafe.map.UnsafeFixedWidthAggregationMap; import io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager; import io.mycat.memory.unsafe.memory.mm.MemoryManager; import io.mycat.memory.unsafe.row.BufferHolder; import io.mycat.memory.unsafe.row.StructType; import io.mycat.memory.unsafe.row.UnsafeRow; import io.mycat.memory.unsafe.row.UnsafeRowWriter; import io.mycat.memory.unsafe.utils.BytesTools; import io.mycat.memory.unsafe.utils.MycatPropertyConf; import io.mycat.memory.unsafe.utils.sort.UnsafeExternalRowSorter; import io.mycat.util.ByteUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.math.BigDecimal; import java.math.RoundingMode; import java.nio.charset.StandardCharsets; import java.text.NumberFormat; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; /** * Created by zagnix on 2016/6/26. * * implement group function select a,count(*),sum(*) from A group by a * */ public class UnsafeRowGrouper { private static final Logger logger = LoggerFactory.getLogger(UnsafeRowGrouper.class); private UnsafeFixedWidthAggregationMap aggregationMap = null; private final Map columToIndx; private final MergeCol[] mergCols; private String[] sortColumnsByIndex = null; private final String[] columns; private boolean isMergAvg=false; private HavingCols havingCols; private UnsafeRow groupKey = null; private UnsafeRow valueKey = null; private BufferHolder bufferHolder = null; private UnsafeRowWriter unsafeRowWriter = null; private final int groupKeyfieldCount; private final int valuefieldCount; private StructType groupKeySchema ; private StructType aggBufferSchema; private UnsafeRow emptyAggregationBuffer; private final MyCatMemory myCatMemory; private final MemoryManager memoryManager; private final MycatPropertyConf conf; public UnsafeRowGrouper(Map columToIndx,String[] columns, MergeCol[] mergCols, HavingCols havingCols) { super(); assert columns!=null; assert columToIndx!=null; assert mergCols !=null; this.columToIndx = columToIndx; this.columns = columns; this.mergCols = mergCols; this.havingCols = havingCols; this.sortColumnsByIndex = columns !=null ? toSortColumnsByIndex(columns,columToIndx):null; this.groupKeyfieldCount = columns != null?columns.length:0; this.valuefieldCount = columToIndx != null?columToIndx.size():0; this.myCatMemory = MycatServer.getInstance().getMyCatMemory(); this.memoryManager = myCatMemory.getResultMergeMemoryManager(); this.conf = myCatMemory.getConf(); logger.debug("columToIndx :" + (columToIndx != null ? columToIndx.toString():"null")); initGroupKey(); initEmptyValueKey(); DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager,Thread.currentThread().getId()); aggregationMap = new UnsafeFixedWidthAggregationMap( emptyAggregationBuffer, aggBufferSchema, groupKeySchema, dataNodeMemoryManager, 1024, conf.getSizeAsBytes("mycat.buffer.pageSize", "32k"), false); } private String[] toSortColumnsByIndex(String[] columns, Map columToIndx) { Map map = new HashMap(); ColMeta curColMeta; for (int i = 0; i < columns.length; i++) { curColMeta = columToIndx.get(columns[i].toUpperCase()); if (curColMeta == null) { throw new IllegalArgumentException( "all columns in group by clause should be in the selected column list.!" + columns[i]); } map.put(columns[i],curColMeta.colIndex); } String[] sortColumnsByIndex = new String[map.size()]; List> entryList = new ArrayList< Map.Entry>( map.entrySet()); Collections.sort(entryList, new Comparator>() { @Override public int compare(Map.Entry o1, Map.Entry o2) { return o1.getValue().compareTo(o2.getValue()); } }); Iterator> iter = entryList.iterator(); Map.Entry tmpEntry = null; int index = 0; while (iter.hasNext()) { tmpEntry = iter.next(); sortColumnsByIndex[index++] = tmpEntry.getKey(); } return sortColumnsByIndex; } private void initGroupKey(){ /** * 构造groupKey */ Map groupcolMetaMap = new HashMap(this.groupKeyfieldCount); groupKey = new UnsafeRow(this.groupKeyfieldCount); bufferHolder = new BufferHolder(groupKey,0); unsafeRowWriter = new UnsafeRowWriter(bufferHolder,this.groupKeyfieldCount); bufferHolder.reset(); ColMeta curColMeta = null; for (int i = 0; i < this.groupKeyfieldCount; i++) { curColMeta = this.columToIndx.get(sortColumnsByIndex[i].toUpperCase()); groupcolMetaMap.put(sortColumnsByIndex[i],curColMeta); switch (curColMeta.colType) { case ColMeta.COL_TYPE_BIT: groupKey.setByte(i, (byte) 0); break; case ColMeta.COL_TYPE_INT: case ColMeta.COL_TYPE_INT24: case ColMeta.COL_TYPE_LONG: groupKey.setInt(i, 0); break; case ColMeta.COL_TYPE_SHORT: groupKey.setShort(i, (short) 0); break; case ColMeta.COL_TYPE_FLOAT: groupKey.setFloat(i, 0); break; case ColMeta.COL_TYPE_DOUBLE: groupKey.setDouble(i, 0); break; case ColMeta.COL_TYPE_NEWDECIMAL: // groupKey.setDouble(i, 0); unsafeRowWriter.write(i, new BigDecimal(0L)); break; case ColMeta.COL_TYPE_LONGLONG: groupKey.setLong(i, 0); break; default: unsafeRowWriter.write(i, "init".getBytes()); break; } } groupKey.setTotalSize(bufferHolder.totalSize()); groupKeySchema = new StructType(groupcolMetaMap,this.groupKeyfieldCount); groupKeySchema.setOrderCols(null); } private void initEmptyValueKey(){ /** * 构造valuerow */ emptyAggregationBuffer = new UnsafeRow(this.valuefieldCount); bufferHolder = new BufferHolder(emptyAggregationBuffer,0); unsafeRowWriter = new UnsafeRowWriter(bufferHolder,this.valuefieldCount); bufferHolder.reset(); ColMeta curColMeta = null; for (Map.Entry fieldEntry : columToIndx.entrySet()) { curColMeta = fieldEntry.getValue(); switch (curColMeta.colType) { case ColMeta.COL_TYPE_BIT: emptyAggregationBuffer.setByte(curColMeta.colIndex, (byte) 0); break; case ColMeta.COL_TYPE_INT: case ColMeta.COL_TYPE_INT24: case ColMeta.COL_TYPE_LONG: emptyAggregationBuffer.setInt(curColMeta.colIndex, 0); break; case ColMeta.COL_TYPE_SHORT: emptyAggregationBuffer.setShort(curColMeta.colIndex, (short) 0); break; case ColMeta.COL_TYPE_LONGLONG: emptyAggregationBuffer.setLong(curColMeta.colIndex, 0); break; case ColMeta.COL_TYPE_FLOAT: emptyAggregationBuffer.setFloat(curColMeta.colIndex, 0); break; case ColMeta.COL_TYPE_DOUBLE: emptyAggregationBuffer.setDouble(curColMeta.colIndex, 0); break; case ColMeta.COL_TYPE_NEWDECIMAL: // emptyAggregationBuffer.setDouble(curColMeta.colIndex, 0); unsafeRowWriter.write(curColMeta.colIndex, new BigDecimal(0L)); break; default: unsafeRowWriter.write(curColMeta.colIndex, "init".getBytes()); break; } } emptyAggregationBuffer.setTotalSize(bufferHolder.totalSize()); aggBufferSchema = new StructType(columToIndx,this.valuefieldCount); aggBufferSchema.setOrderCols(null); } public Iterator getResult(@Nonnull UnsafeExternalRowSorter sorter) throws IOException { KVIterator iter = aggregationMap.iterator(); /** * 求平均值 */ if (isMergeAvg() && !isMergAvg){ try { while (iter.next()){ mergAvg(iter.getValue()); } } catch (IOException e) { logger.error(e.getMessage()); } isMergAvg = true; processAvgFieldPrecision(); } /** * group having */ if (havingCols !=null){ filterHaving(sorter); }else{ /** * KVIterator ==>Iterator */ insertValue(sorter); } return sorter.sort(); } /** * 处理AVG列精度 */ private void processAvgFieldPrecision() { for (MergeCol mergCol : mergCols) { if (mergCol.mergeType != MergeCol.MERGE_AVG) { continue ; } for (String key : columToIndx.keySet()) { ColMeta colMeta = columToIndx.get(key); // AVG列的小数点精度默认取SUM小数点精度, 计算和返回的小数点精度应该扩展4 if (colMeta.colIndex == mergCol.colMeta.avgSumIndex) { colMeta.decimals += 4; break ; } } } } /** * 判断列是否为AVG列 * @param columnName * @return */ private boolean isAvgField(String columnName) { Pattern pattern = Pattern.compile("AVG([1-9]\\d*|0)SUM"); Matcher matcher = pattern.matcher(columnName); return matcher.find(); } public UnsafeRow getAllBinaryRow(UnsafeRow row) throws UnsupportedEncodingException { UnsafeRow value = new UnsafeRow( this.valuefieldCount); bufferHolder = new BufferHolder(value,0); unsafeRowWriter = new UnsafeRowWriter(bufferHolder, this.valuefieldCount); bufferHolder.reset(); ColMeta curColMeta = null; for (Map.Entry fieldEntry : columToIndx.entrySet()) { curColMeta = fieldEntry.getValue(); if (!row.isNullAt(curColMeta.colIndex)) { switch (curColMeta.colType) { case ColMeta.COL_TYPE_BIT: unsafeRowWriter.write(curColMeta.colIndex, row.getByte(curColMeta.colIndex)); break; case ColMeta.COL_TYPE_INT: case ColMeta.COL_TYPE_LONG: case ColMeta.COL_TYPE_INT24: unsafeRowWriter.write(curColMeta.colIndex, BytesTools.int2Bytes(row.getInt(curColMeta.colIndex))); break; case ColMeta.COL_TYPE_SHORT: unsafeRowWriter.write(curColMeta.colIndex, BytesTools.short2Bytes(row.getShort(curColMeta.colIndex))); break; case ColMeta.COL_TYPE_LONGLONG: unsafeRowWriter.write(curColMeta.colIndex, BytesTools.long2Bytes(row.getLong(curColMeta.colIndex))); break; case ColMeta.COL_TYPE_FLOAT: unsafeRowWriter.write(curColMeta.colIndex, BytesTools.float2Bytes(row.getFloat(curColMeta.colIndex))); break; case ColMeta.COL_TYPE_DOUBLE: unsafeRowWriter.write(curColMeta.colIndex, BytesTools.double2Bytes(row.getDouble(curColMeta.colIndex))); break; case ColMeta.COL_TYPE_NEWDECIMAL: int scale = curColMeta.decimals; BigDecimal decimalVal = row.getDecimal(curColMeta.colIndex, scale); unsafeRowWriter.write(curColMeta.colIndex, decimalVal.toString().getBytes()); break; default: unsafeRowWriter.write(curColMeta.colIndex, row.getBinary(curColMeta.colIndex)); break; } }else { unsafeRowWriter.setNullAt(curColMeta.colIndex); } } value.setTotalSize(bufferHolder.totalSize()); return value; } private void insertValue(@Nonnull UnsafeExternalRowSorter sorter){ KVIterator it = aggregationMap.iterator(); try { while (it.next()){ UnsafeRow row = getAllBinaryRow(it.getValue()); sorter.insertRow(row); } } catch (IOException e) { logger.error("group insertValue err: " + e.getMessage()); free(); } } private void filterHaving(@Nonnull UnsafeExternalRowSorter sorter){ if (havingCols.getColMeta() == null || aggregationMap == null) { return; } KVIterator it = aggregationMap.iterator(); byte[] right = havingCols.getRight().getBytes(StandardCharsets.UTF_8); int index = havingCols.getColMeta().getColIndex(); try { while (it.next()){ UnsafeRow row = getAllBinaryRow(it.getValue()); switch (havingCols.getOperator()) { case "=": if (eq(row.getBinary(index),right)) { sorter.insertRow(row); } break; case ">": if (gt(row.getBinary(index),right)) { sorter.insertRow(row); } break; case "<": if (lt(row.getBinary(index),right)) { sorter.insertRow(row); } break; case ">=": if (gt(row.getBinary(index),right) || eq(row.getBinary(index),right)) { sorter.insertRow(row); } break; case "<=": if (lt(row.getBinary(index),right) || eq(row.getBinary(index),right)) { sorter.insertRow(row); } break; case "!=": if (neq(row.getBinary(index),right)) { sorter.insertRow(row); } break; } } } catch (IOException e) { logger.error(e.getMessage()); } } private boolean lt(byte[] l, byte[] r) { return -1 >= ByteUtil.compareNumberByte(l, r); } private boolean gt(byte[] l, byte[] r) { return 1 <= ByteUtil.compareNumberByte(l, r); } private boolean eq(byte[] l, byte[] r) { return 0 == ByteUtil.compareNumberByte(l, r); } private boolean neq(byte[] l, byte[] r) { return 0 != ByteUtil.compareNumberByte(l, r); } /** * 构造groupKey */ private UnsafeRow getGroupKey(UnsafeRow row) throws UnsupportedEncodingException { UnsafeRow key = null; if(this.sortColumnsByIndex == null){ /** * 针对没有group by关键字 * select count(*) from table; */ key = new UnsafeRow(this.groupKeyfieldCount+1); bufferHolder = new BufferHolder(key,0); unsafeRowWriter = new UnsafeRowWriter(bufferHolder,this.groupKeyfieldCount+1); bufferHolder.reset(); unsafeRowWriter.write(0,"same".getBytes()); key.setTotalSize(bufferHolder.totalSize()); return key; } key = new UnsafeRow(this.groupKeyfieldCount); bufferHolder = new BufferHolder(key,0); unsafeRowWriter = new UnsafeRowWriter(bufferHolder,this.groupKeyfieldCount); bufferHolder.reset(); ColMeta curColMeta = null; for (int i = 0; i < this.groupKeyfieldCount;i++) { curColMeta = this.columToIndx.get(sortColumnsByIndex[i].toUpperCase()); if(!row.isNullAt(curColMeta.colIndex)){ switch(curColMeta.colType){ case ColMeta.COL_TYPE_BIT: key.setByte(i,row.getByte(curColMeta.colIndex)); case ColMeta.COL_TYPE_INT: case ColMeta.COL_TYPE_LONG: case ColMeta.COL_TYPE_INT24: key.setInt(i, BytesTools.getInt(row.getBinary(curColMeta.colIndex))); break; case ColMeta.COL_TYPE_SHORT: key.setShort(i, BytesTools.getShort(row.getBinary(curColMeta.colIndex))); break; case ColMeta.COL_TYPE_FLOAT: key.setFloat(i, BytesTools.getFloat(row.getBinary(curColMeta.colIndex))); break; case ColMeta.COL_TYPE_DOUBLE: key.setDouble(i, BytesTools.getDouble(row.getBinary(curColMeta.colIndex))); break; case ColMeta.COL_TYPE_NEWDECIMAL: // key.setDouble(i, // BytesTools.getDouble(row.getBinary(curColMeta.colIndex))); unsafeRowWriter.write(i, new BigDecimal(new String(row.getBinary(curColMeta.colIndex)))); break; case ColMeta.COL_TYPE_LONGLONG: key.setLong(i, BytesTools.getLong(row.getBinary(curColMeta.colIndex))); break; default: unsafeRowWriter.write(i, row.getBinary(curColMeta.colIndex)); break; } }else { key.setNullAt(i); } } key.setTotalSize(bufferHolder.totalSize()); return key; } /** * 构造value */ private UnsafeRow getValue(UnsafeRow row) throws UnsupportedEncodingException { UnsafeRow value = new UnsafeRow(this.valuefieldCount); bufferHolder = new BufferHolder(value,0); unsafeRowWriter = new UnsafeRowWriter(bufferHolder,this.valuefieldCount); bufferHolder.reset(); ColMeta curColMeta = null; for (Map.Entry fieldEntry : columToIndx.entrySet()) { curColMeta = fieldEntry.getValue(); if(!row.isNullAt(curColMeta.colIndex)) { switch (curColMeta.colType) { case ColMeta.COL_TYPE_BIT: value.setByte(curColMeta.colIndex, row.getByte(curColMeta.colIndex)); break; case ColMeta.COL_TYPE_INT: case ColMeta.COL_TYPE_LONG: case ColMeta.COL_TYPE_INT24: value.setInt(curColMeta.colIndex, BytesTools.getInt(row.getBinary(curColMeta.colIndex))); break; case ColMeta.COL_TYPE_SHORT: value.setShort(curColMeta.colIndex, BytesTools.getShort(row.getBinary(curColMeta.colIndex))); break; case ColMeta.COL_TYPE_LONGLONG: value.setLong(curColMeta.colIndex, BytesTools.getLong(row.getBinary(curColMeta.colIndex))); break; case ColMeta.COL_TYPE_FLOAT: value.setFloat(curColMeta.colIndex, BytesTools.getFloat(row.getBinary(curColMeta.colIndex))); break; case ColMeta.COL_TYPE_DOUBLE: value.setDouble(curColMeta.colIndex, BytesTools.getDouble(row.getBinary(curColMeta.colIndex))); break; case ColMeta.COL_TYPE_NEWDECIMAL: // value.setDouble(curColMeta.colIndex, BytesTools.getDouble(row.getBinary(curColMeta.colIndex))); unsafeRowWriter.write(curColMeta.colIndex, new BigDecimal(new String(row.getBinary(curColMeta.colIndex)))); break; default: unsafeRowWriter.write(curColMeta.colIndex, row.getBinary(curColMeta.colIndex)); break; } }else { switch(curColMeta.colType) { case ColMeta.COL_TYPE_NEWDECIMAL: BigDecimal nullDecimal = null; unsafeRowWriter.write(curColMeta.colIndex, nullDecimal); break; default: value.setNullAt(curColMeta.colIndex); break; } } } value.setTotalSize(bufferHolder.totalSize()); return value; } public void addRow(UnsafeRow rowDataPkg) throws UnsupportedEncodingException { UnsafeRow key = getGroupKey(rowDataPkg); UnsafeRow value = getValue(rowDataPkg); if(aggregationMap.find(key)){ UnsafeRow rs = aggregationMap.getAggregationBuffer(key); aggregateRow(key,rs,value); }else { aggregationMap.put(key,value); } return; } private boolean isMergeAvg(){ if (mergCols == null) { return false; } for (MergeCol merg : mergCols) { if(merg.mergeType == MergeCol.MERGE_AVG) { return true; } } return false; } private void aggregateRow(UnsafeRow key,UnsafeRow toRow, UnsafeRow newRow) throws UnsupportedEncodingException { if (mergCols == null) { return; } for (MergeCol merg : mergCols) { if(merg.mergeType != MergeCol.MERGE_AVG && merg.colMeta !=null) { byte[] result = null; byte[] left = null; byte[] right = null; int type = merg.colMeta.colType; int index = merg.colMeta.colIndex; switch(type){ case ColMeta.COL_TYPE_INT: case ColMeta.COL_TYPE_LONG: case ColMeta.COL_TYPE_INT24: if (!toRow.isNullAt(index)) { left = BytesTools.int2Bytes(toRow.getInt(index)); } if (!newRow.isNullAt(index)) { right = BytesTools.int2Bytes(newRow.getInt(index)); } break; case ColMeta.COL_TYPE_SHORT: left = BytesTools.short2Bytes(toRow.getShort(index)); right =BytesTools.short2Bytes(newRow.getShort(index)); break; case ColMeta.COL_TYPE_LONGLONG: left = BytesTools.long2Bytes(toRow.getLong(index)); right = BytesTools.long2Bytes(newRow.getLong(index)); break; case ColMeta.COL_TYPE_FLOAT: left = BytesTools.float2Bytes(toRow.getFloat(index)); right = BytesTools.float2Bytes(newRow.getFloat(index)); break; case ColMeta.COL_TYPE_DOUBLE: left = BytesTools.double2Bytes(toRow.getDouble(index)); right = BytesTools.double2Bytes(newRow.getDouble(index)); break; case ColMeta.COL_TYPE_NEWDECIMAL: // left = BytesTools.double2Bytes(toRow.getDouble(index)); // right = BytesTools.double2Bytes(newRow.getDouble(index)); int scale = merg.colMeta.decimals; BigDecimal decimalLeft = toRow.getDecimal(index, scale); BigDecimal decimalRight = newRow.getDecimal(index, scale); left = decimalLeft == null ? null : decimalLeft.toString().getBytes(); right = decimalRight == null ? null : decimalRight.toString().getBytes(); break; case ColMeta.COL_TYPE_DATE: case ColMeta.COL_TYPE_TIMSTAMP: case ColMeta.COL_TYPE_TIME: case ColMeta.COL_TYPE_YEAR: case ColMeta.COL_TYPE_DATETIME: case ColMeta.COL_TYPE_NEWDATE: case ColMeta.COL_TYPE_BIT: case ColMeta.COL_TYPE_VAR_STRING: case ColMeta.COL_TYPE_STRING: case ColMeta.COL_TYPE_ENUM: case ColMeta.COL_TYPE_SET: left = toRow.getBinary(index); right = newRow.getBinary(index); break; default: break; } result = mertFields(left,right,type,merg.mergeType); if (result != null) { switch(type){ case ColMeta.COL_TYPE_BIT: toRow.setByte(index,result[0]); case ColMeta.COL_TYPE_INT: case ColMeta.COL_TYPE_LONG: case ColMeta.COL_TYPE_INT24: toRow.setInt(index,BytesTools.getInt(result)); break; case ColMeta.COL_TYPE_SHORT: toRow.setShort(index,BytesTools.getShort(result)); break; case ColMeta.COL_TYPE_LONGLONG: toRow.setLong(index,BytesTools.getLong(result)); break; case ColMeta.COL_TYPE_FLOAT: toRow.setFloat(index,BytesTools.getFloat(result)); break; case ColMeta.COL_TYPE_DOUBLE: toRow.setDouble(index,BytesTools.getDouble(result)); break; case ColMeta.COL_TYPE_NEWDECIMAL: // toRow.setDouble(index,BytesTools.getDouble(result)); toRow.updateDecimal(index, new BigDecimal(new String(result))); break; /** *TODO UnsafeFixedWidthAggregationMap 中存放 * UnsafeRow时，非数值类型的列不可更改其值， * 为了统一处理聚合函数这块 * 做max或者min聚合时候，目前解决方法 * 先free原来 UnsafeFixedWidthAggregationMap对象。 * 然后重新创建一个UnsafeFixedWidthAggregationMap对象 * 然后存放最新的max或者min值作为下次比较。 **/ case ColMeta.COL_TYPE_DATE: case ColMeta.COL_TYPE_TIMSTAMP: case ColMeta.COL_TYPE_TIME: case ColMeta.COL_TYPE_YEAR: case ColMeta.COL_TYPE_DATETIME: case ColMeta.COL_TYPE_NEWDATE: case ColMeta.COL_TYPE_VAR_STRING: case ColMeta.COL_TYPE_STRING: case ColMeta.COL_TYPE_ENUM: case ColMeta.COL_TYPE_SET: aggregationMap.free(); DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager,Thread.currentThread().getId()); aggregationMap = new UnsafeFixedWidthAggregationMap( emptyAggregationBuffer, aggBufferSchema, groupKeySchema, dataNodeMemoryManager, 1024, conf.getSizeAsBytes("mycat.buffer.pageSize", "32k"), false); UnsafeRow unsafeRow = new UnsafeRow(toRow.numFields()); bufferHolder = new BufferHolder(unsafeRow, 0); unsafeRowWriter = new UnsafeRowWriter(bufferHolder, toRow.numFields()); bufferHolder.reset(); for (int i = 0; i < toRow.numFields(); i++) { if (i == index) { unsafeRowWriter.write(i,result); } else if (!toRow.isNullAt(i)) { unsafeRowWriter.write(i, toRow.getBinary(i)); } else if (toRow.isNullAt(i)){ unsafeRow.setNullAt(i); } } unsafeRow.setTotalSize(bufferHolder.totalSize()); aggregationMap.put(key, unsafeRow); toRow = unsafeRow; break; default: break; } } } } } private void mergAvg(UnsafeRow toRow) throws UnsupportedEncodingException { if (mergCols == null) { return; } for (MergeCol merg : mergCols) { if(merg.mergeType==MergeCol.MERGE_AVG) { byte[] result = null; byte[] avgSum = null; byte[] avgCount = null; int type = merg.colMeta.colType; int avgSumIndex = merg.colMeta.avgSumIndex; int avgCountIndex = merg.colMeta.avgCountIndex; switch(type){ case ColMeta.COL_TYPE_BIT: avgSum = BytesTools.toBytes(toRow.getByte(avgSumIndex)); avgCount = BytesTools.toBytes(toRow.getLong(avgCountIndex)); break; case ColMeta.COL_TYPE_INT: case ColMeta.COL_TYPE_LONG: case ColMeta.COL_TYPE_INT24: avgSum = BytesTools.int2Bytes(toRow.getInt(avgSumIndex)); avgCount = BytesTools.long2Bytes(toRow.getLong(avgCountIndex)); break; case ColMeta.COL_TYPE_SHORT: avgSum =BytesTools.short2Bytes(toRow.getShort(avgSumIndex)); avgCount = BytesTools.long2Bytes(toRow.getLong(avgCountIndex)); break; case ColMeta.COL_TYPE_LONGLONG: avgSum = BytesTools.long2Bytes(toRow.getLong(avgSumIndex)); avgCount = BytesTools.long2Bytes(toRow.getLong(avgCountIndex)); break; case ColMeta.COL_TYPE_FLOAT: avgSum = BytesTools.float2Bytes(toRow.getFloat(avgSumIndex)); avgCount = BytesTools.long2Bytes(toRow.getLong(avgCountIndex)); break; case ColMeta.COL_TYPE_DOUBLE: avgSum = BytesTools.double2Bytes(toRow.getDouble(avgSumIndex)); avgCount = BytesTools.long2Bytes(toRow.getLong(avgCountIndex)); break; case ColMeta.COL_TYPE_NEWDECIMAL: // avgSum = BytesTools.double2Bytes(toRow.getDouble(avgSumIndex)); // avgCount = BytesTools.long2Bytes(toRow.getLong(avgCountIndex)); int scale = merg.colMeta.decimals; BigDecimal sumDecimal = toRow.getDecimal(avgSumIndex, scale); avgSum = sumDecimal == null ? null : sumDecimal.toString().getBytes(); avgCount = BytesTools.long2Bytes(toRow.getLong(avgCountIndex)); break; default: break; } result = mertFields(avgSum,avgCount,merg.colMeta.colType,merg.mergeType); if (result != null) { switch(type){ case ColMeta.COL_TYPE_BIT: toRow.setByte(avgSumIndex,result[0]); break; case ColMeta.COL_TYPE_INT: case ColMeta.COL_TYPE_LONG: case ColMeta.COL_TYPE_INT24: toRow.setInt(avgSumIndex,BytesTools.getInt(result)); break; case ColMeta.COL_TYPE_SHORT: toRow.setShort(avgSumIndex,BytesTools.getShort(result)); break; case ColMeta.COL_TYPE_LONGLONG: toRow.setLong(avgSumIndex,BytesTools.getLong(result)); break; case ColMeta.COL_TYPE_FLOAT: toRow.setFloat(avgSumIndex,BytesTools.getFloat(result)); break; case ColMeta.COL_TYPE_DOUBLE: toRow.setDouble(avgSumIndex,ByteUtil.getDouble(result)); break; case ColMeta.COL_TYPE_NEWDECIMAL: // toRow.setDouble(avgSumIndex,ByteUtil.getDouble(result)); toRow.updateDecimal(avgSumIndex, new BigDecimal(new String(result))); break; default: break; } } } } } private byte[] mertFields(byte[] bs, byte[] bs2, int colType, int mergeType) throws UnsupportedEncodingException { if(bs2==null || bs2.length==0) { return bs; }else if(bs==null || bs.length==0) { return bs2; } switch (mergeType) { case MergeCol.MERGE_SUM: if (colType == ColMeta.COL_TYPE_DOUBLE || colType == ColMeta.COL_TYPE_FLOAT){ double value = BytesTools.getDouble(bs) + BytesTools.getDouble(bs2); return BytesTools.double2Bytes(value); } else if(colType == ColMeta.COL_TYPE_NEWDECIMAL || colType == ColMeta.COL_TYPE_DECIMAL) { BigDecimal decimal = new BigDecimal(new String(bs)); decimal = decimal.add(new BigDecimal(new String(bs2))); return decimal.toString().getBytes(); } case MergeCol.MERGE_COUNT: { long s1 = BytesTools.getLong(bs); long s2 = BytesTools.getLong(bs2); long total = s1 + s2; return BytesTools.long2Bytes(total); } case MergeCol.MERGE_MAX: { int compare = ByteUtil.compareNumberByte(bs,bs2); return (compare > 0) ? bs : bs2; } case MergeCol.MERGE_MIN: { int compare = ByteUtil.compareNumberByte(bs,bs2); return (compare > 0) ? bs2 : bs; } case MergeCol.MERGE_AVG: { /** * 元素总个数 */ long count = BytesTools.getLong(bs2); if (colType == ColMeta.COL_TYPE_DOUBLE || colType == ColMeta.COL_TYPE_FLOAT) { /** * 数值总和 */ double sum = BytesTools.getDouble(bs); double value = sum / count; return BytesTools.double2Bytes(value); } else if(colType == ColMeta.COL_TYPE_NEWDECIMAL || colType == ColMeta.COL_TYPE_DECIMAL){ BigDecimal sum = new BigDecimal(new String(bs)); // AVG计算时候小数点精度扩展4, 并且四舍五入 BigDecimal avg = sum.divide(new BigDecimal(count), sum.scale() + 4, RoundingMode.HALF_UP); return avg.toString().getBytes(); } } default: return null; } } public void free(){ if(aggregationMap != null) aggregationMap.free(); } } ================================================ FILE: src/main/java/io/mycat/sqlengine/mpp/model/NodeRowDataPacket.java ================================================ package io.mycat.sqlengine.mpp.model; import java.util.ArrayList; import java.util.List; import io.mycat.net.mysql.RowDataPacket; import io.mycat.route.RouteResultsetNode; public class NodeRowDataPacket { private RouteResultsetNode node; private long trimTotal = 0; private int trimSize = 0; private List trimRangRDPacketList = new ArrayList(); private List rangRDPacketList = new ArrayList(); public NodeRowDataPacket(RouteResultsetNode node, int trimSize) { this.node = node; this.trimSize = trimSize; } public void newRang() { RangRowDataPacket rangPacket = new RangRowDataPacket(); rangRDPacketList.add(rangPacket); } public long loadTotal() { return this.loadTrimTotal() + this.loadNotTrimTotal(); } public long loadTrimTotal() { this.trimTotal = 0; for (RangRowDataPacket packet : trimRangRDPacketList) { if (packet.isTrim()) { this.trimTotal += packet.allSize(); } } return this.trimTotal; } public long loadNotTrimTotal() { long total = 0; for (RangRowDataPacket packet : rangRDPacketList) { total += packet.allSize(); } return total; } public void moveToTrim() { RangRowDataPacket head = this.loadHeadPacket(); if (head != null && this.rangRDPacketList.remove(head)) { this.trimRangRDPacketList.add(head); if (head.allSize() == this.trimSize) { head.leftHeadTail(); } } } public void moveHeadTail3ToTrim() { if (this.rangRDPacketList.size() >= 3) { int m = 0; while ((m = this.rangRDPacketList.size()) > 3) { RangRowDataPacket packet = this.rangRDPacketList.remove(0); if (packet.allSize() == this.trimSize) { packet.leftHeadTail(); } addTrimWithCombine(packet); } } } private void addTrimWithCombine(RangRowDataPacket packet) { if (packet.allSize() == this.trimSize) { if (this.trimRangRDPacketList.isEmpty()) { this.trimRangRDPacketList.add(packet); } else { int last = this.trimRangRDPacketList.size() - 1; RangRowDataPacket lastPacket = this.trimRangRDPacketList.get(last); if (lastPacket.isTrim()) { lastPacket.combine(packet); } else { //异常 } } } } public void moveAllToTrim() { int m = 0; while ((m = this.rangRDPacketList.size()) > 0) { RangRowDataPacket packet = this.rangRDPacketList.remove(0); if (packet.getRowDataPacketList().size() == this.trimSize) { packet.leftHeadTail(); } addTrimWithCombine(packet); } } public void addPacket(RowDataPacket packet) { RangRowDataPacket rangPacket = rangRDPacketList.get(rangRDPacketList.size() - 1); rangPacket.appendPacket(packet); } public RouteResultsetNode getNode() { return node; } public List loadData() { List result = new ArrayList(); for (RangRowDataPacket packet : rangRDPacketList) { result.addAll(packet.getRowDataPacketList()); } for (RangRowDataPacket packet : trimRangRDPacketList) { if (!packet.isTrim()) { result.addAll(packet.getRowDataPacketList()); } } return result; } public RangRowDataPacket loadHeadPacket() { if (rangRDPacketList != null && !rangRDPacketList.isEmpty()) { return rangRDPacketList.get(0); } return null; } public RangRowDataPacket loadTailPacket() { return this.loadTailPacket(1); } public RangRowDataPacket loadTailPacket(int tailIndex) { int size = rangRDPacketList.size() - tailIndex; if (size >= 0) { return rangRDPacketList.get(size); } return null; } } ================================================ FILE: src/main/java/io/mycat/sqlengine/mpp/model/RangRowDataPacket.java ================================================ package io.mycat.sqlengine.mpp.model; import java.util.ArrayList; import java.util.List; import io.mycat.net.mysql.RowDataPacket; public class RangRowDataPacket { public static final int DATA_TYPE_ALL = 100; public static final int DATA_TYPE_TRIM = 200; private int dataType = DATA_TYPE_ALL; private List rowDataPacketList = new ArrayList(); private int trimCount = 0; public int getTrimCount() { return trimCount; } public void appendPacket(List packetList) { this.rowDataPacketList.addAll(packetList); } public void appendPacket(RowDataPacket packet) { this.rowDataPacketList.add(packet); } public boolean isTrim() { return dataType == DATA_TYPE_TRIM ? true : false; } public void leftHeadTail() { while (this.rowDataPacketList.size() > 2) { this.rowDataPacketList.remove(1); trimCount++; } dataType = DATA_TYPE_TRIM; } public int allSize() { if (dataType == DATA_TYPE_TRIM) { return trimCount + rowDataPacketList.size(); } else { return rowDataPacketList.size(); } } public void combine(RangRowDataPacket rowData) { if (dataType == DATA_TYPE_TRIM) { if (this.rowDataPacketList.isEmpty()) { this.trimCount = rowData.getTrimCount(); this.rowDataPacketList.addAll(rowData.getRowDataPacketList()); } else { if (rowData.allSize() == 0) { return; } this.rowDataPacketList.remove(1); if (rowData.allSize() == 1) { this.trimCount += 1 + rowData.getTrimCount(); this.rowDataPacketList.add(rowData.getHead()); } else if (rowData.allSize() >= 2) { this.trimCount += 2 + rowData.getTrimCount(); this.rowDataPacketList.add(rowData.getTail()); } } } } public List getRowDataPacketList() { return rowDataPacketList; } public RowDataPacket getHead() { if (this.rowDataPacketList.isEmpty()) { return null; } return this.rowDataPacketList.get(0); } public RowDataPacket getTail() { if (this.rowDataPacketList.size() < 2) { return null; } return this.rowDataPacketList.get(this.rowDataPacketList.size()-1); } } ================================================ FILE: src/main/java/io/mycat/statistic/CommandCount.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.statistic; /** * @author mycat */ public class CommandCount { private long initDB; private long query; private long stmtPrepare; private long stmtSendLongData; private long stmtReset; private long stmtExecute; private long stmtClose; private long ping; private long kill; private long quit; private long heartbeat; private long setOption; private long other; public CommandCount(){ } public void doInitDB() { ++initDB; } public long initDBCount() { return initDB; } public void doQuery() { ++query; } public long queryCount() { return query; } public void doStmtPrepare() { ++stmtPrepare; } public long stmtPrepareCount() { return stmtPrepare; } public void doStmtSendLongData() { ++stmtSendLongData; } public long stmtSendLongDataCount() { return stmtSendLongData; } public void doStmtReset() { ++stmtReset; } public long stmtResetCount() { return stmtReset; } public void doStmtExecute() { ++stmtExecute; } public long stmtExecuteCount() { return stmtExecute; } public void doStmtClose() { ++stmtClose; } public long stmtCloseCount() { return stmtClose; } public void doPing() { ++ping; } public long pingCount() { return ping; } public void doKill() { ++kill; } public long killCount() { return kill; } public void doQuit() { ++quit; } public long quitCount() { return quit; } public void doOther() { ++other; } public long heartbeat() { return heartbeat; } public void doHeartbeat() { ++heartbeat; } public void doSetOption() { ++setOption; } public long otherCount() { return other; } } ================================================ FILE: src/main/java/io/mycat/statistic/DataSourceSyncRecorder.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.statistic; import java.text.SimpleDateFormat; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.config.model.DataHostConfig; import io.mycat.util.TimeUtil; /** * 记录最近3个时段的平均响应时间，默认1，10，30分钟。 * * @author songwie */ public class DataSourceSyncRecorder { private Map records; private final List asynRecords;//value,time private static final Logger LOGGER = LoggerFactory.getLogger("DataSourceSyncRecorder"); private static final long SWAP_TIME = 24 * 60 * 60 * 1000L; //日期处理 private static final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); private int switchType = 2; public DataSourceSyncRecorder() { this.records = new HashMap() ; this.asynRecords = new LinkedList(); } public String get() { return records.toString(); } public void set(Map resultResult,int switchType) { try{ long time = TimeUtil.currentTimeMillis(); this.switchType = switchType; remove(time); if (resultResult!=null && !resultResult.isEmpty()) { this.records = resultResult; if(switchType==DataHostConfig.SYN_STATUS_SWITCH_DS){ //slave String sencords = resultResult.get("Seconds_Behind_Master"); long Seconds_Behind_Master = -1; if(sencords!=null){ Seconds_Behind_Master = Long.parseLong(sencords); } this.asynRecords.add(new Record(TimeUtil.currentTimeMillis(),Seconds_Behind_Master)); } if(switchType==DataHostConfig.CLUSTER_STATUS_SWITCH_DS){//cluster double wsrep_local_recv_queue_avg = Double.valueOf(resultResult.get("wsrep_local_recv_queue_avg")); this.asynRecords.add(new Record(TimeUtil.currentTimeMillis(),wsrep_local_recv_queue_avg)); } return; } }catch(Exception e){ LOGGER.error("record DataSourceSyncRecorder error " + e.getMessage()); } } /** * 删除超过统计时间段的数据 */ private void remove(long time) { final List recordsAll = this.asynRecords; while (recordsAll.size() > 0) { Record record = recordsAll.get(0); if (time >= record.time + SWAP_TIME) { recordsAll.remove(0); } else { break; } } } public int getSwitchType() { return this.switchType; } public void setSwitchType(int switchType) { this.switchType = switchType; } public Map getRecords() { return this.records; } public List getAsynRecords() { return this.asynRecords; } public static SimpleDateFormat getSdf() { return sdf; } /** * @author mycat */ public static class Record { private Object value; private long time; Record(long time, Object value) { this.time = time; this.value = value; } public Object getValue() { return this.value; } public void setValue(Object value) { this.value = value; } public long getTime() { return this.time; } public void setTime(long time) { this.time = time; } } } ================================================ FILE: src/main/java/io/mycat/statistic/HeartbeatRecorder.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.statistic; import java.util.List; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.util.TimeUtil; /** * 记录最近3个时段的平均响应时间，默认1，10，30分钟。 * * @author mycat */ public class HeartbeatRecorder { private static final int MAX_RECORD_SIZE = 256; private static final long AVG1_TIME = 60 * 1000L; private static final long AVG2_TIME = 10 * 60 * 1000L; private static final long AVG3_TIME = 30 * 60 * 1000L; private static final long SWAP_TIME = 24 * 60 * 60 * 1000L; private long avg1; private long avg2; private long avg3; private final Queue records; private final Queue recordsAll; private static final Logger LOGGER = LoggerFactory.getLogger("DataSourceSyncRecorder"); public HeartbeatRecorder() { this.records = new ConcurrentLinkedQueue(); this.recordsAll = new ConcurrentLinkedQueue(); } public String get() { return new StringBuilder().append(avg1).append(',').append(avg2).append(',').append(avg3).toString(); } public void set(long value) { try{ long time = TimeUtil.currentTimeMillis(); if (value < 0) { recordsAll.offer(new Record(0, time)); return; } remove(time); int size = records.size(); if (size == 0) { records.offer(new Record(value, time)); avg1 = avg2 = avg3 = value; return; } if (size >= MAX_RECORD_SIZE) { records.poll(); } records.offer(new Record(value, time)); recordsAll.offer(new Record(value, time)); calculate(time); }catch(Exception e){ LOGGER.error("record HeartbeatRecorder error " ,e); } } /** * 删除超过统计时间段的数据 */ private void remove(long time) { final Queue records = this.records; while (records.size() > 0) { Record record = records.peek(); if (time >= record.time + AVG3_TIME) { records.poll(); } else { break; } } final Queue recordsAll = this.recordsAll; while (recordsAll.size() > 0) { Record record = recordsAll.peek(); if (time >= record.time + SWAP_TIME) { recordsAll.poll(); } else { break; } } } /** * 计算记录的统计数据 */ private void calculate(long time) { long v1 = 0L, v2 = 0L, v3 = 0L; int c1 = 0, c2 = 0, c3 = 0; for (Record record : records) { long t = time - record.time; if (t <= AVG1_TIME) { v1 += record.value; ++c1; } if (t <= AVG2_TIME) { v2 += record.value; ++c2; } if (t <= AVG3_TIME) { v3 += record.value; ++c3; } } avg1 = (v1 / c1); avg2 = (v2 / c2); avg3 = (v3 / c3); } public Queue getRecordsAll() { return this.recordsAll; } /** * @author mycat */ public static class Record { private long value; private long time; Record(long value, long time) { this.value = value; this.time = time; } public long getValue() { return this.value; } public void setValue(long value) { this.value = value; } public long getTime() { return this.time; } public void setTime(long time) { this.time = time; } } } ================================================ FILE: src/main/java/io/mycat/statistic/SQLRecord.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.statistic; /** * @author mycat */ public final class SQLRecord implements Comparable { public String host; public String schema; public String statement; public long startTime; public long executeTime; public String dataNode; public int dataNodeIndex; @Override public int compareTo(SQLRecord o) { //执行时间从大到小 long para = o.executeTime - executeTime; //开始时间从大到小 return (int) (para == 0 ? (o.startTime - startTime) : para ); } @Override public boolean equals(Object arg0) { return super.equals(arg0); } @Override public int hashCode() { // TODO Auto-generated method stub return super.hashCode(); } } ================================================ FILE: src/main/java/io/mycat/statistic/SQLRecorder.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.statistic; import java.util.*; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.locks.ReentrantLock; /** * SQL统计排序记录器 * * @author mycat */ public final class SQLRecorder { private final int count; SortedSet records; public SQLRecorder(int count) { this.count = count; this.records = new ConcurrentSkipListSet<>(); } public List getRecords() { List keyList = new ArrayList(records); return keyList; } public void add(SQLRecord record) { records.add(record); } public void clear() { records.clear(); } public void recycle(){ if(records.size() > count){ SortedSet records2 = new ConcurrentSkipListSet<>(); List keyList = new ArrayList(records); int i = 0; for(SQLRecord key : keyList){ if(i == count) { break; } records2.add(key); i++; } records = records2; } } } ================================================ FILE: src/main/java/io/mycat/statistic/stat/Histogram.java ================================================ package io.mycat.statistic.stat; import java.util.concurrent.atomic.AtomicLongArray; public class Histogram { private final long[] ranges; private final AtomicLongArray rangeCounters; public Histogram(long... ranges){ this.ranges = ranges; this.rangeCounters = new AtomicLongArray(ranges.length); } public void reset() { for (int i = 0; i < rangeCounters.length(); i++) { rangeCounters.set(i, 0); } } public void record(long range) { int index = rangeCounters.length(); for (int i = 0; i < ranges.length; i++) { if (range == ranges[i]) { index = i; break; } } rangeCounters.incrementAndGet(index); } public long get(int index) { return rangeCounters.get(index); } public long[] toArray() { long[] array = new long[rangeCounters.length()]; for (int i = 0; i < rangeCounters.length(); i++) { array[i] = rangeCounters.get(i); } return array; } public long[] toArrayAndReset() { long[] array = new long[rangeCounters.length()]; for (int i = 0; i < rangeCounters.length(); i++) { array[i] = rangeCounters.getAndSet(i, 0); } return array; } public long[] getRanges() { return ranges; } public long getValue(int index) { return rangeCounters.get(index); } public long getSum() { long sum = 0; for (int i = 0; i < rangeCounters.length(); ++i) { sum += rangeCounters.get(i); } return sum; } public String toString() { StringBuilder buf = new StringBuilder(); buf.append('['); for (int i = 0; i < rangeCounters.length(); i++) { if (i != 0) { buf.append(", "); } buf.append(rangeCounters.get(i)); } buf.append(']'); return buf.toString(); } } ================================================ FILE: src/main/java/io/mycat/statistic/stat/HostStatAnalyzer.java ================================================ package io.mycat.statistic.stat; /** * 前端SQL客户端主机 的访问统计 * * @author zhuam * */ public class HostStatAnalyzer implements QueryResultListener { @Override public void onQueryResult(QueryResult query) { } } ================================================ FILE: src/main/java/io/mycat/statistic/stat/QueryConditionAnalyzer.java ================================================ package io.mycat.statistic.stat; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; import com.alibaba.druid.stat.TableStat.Condition; import io.mycat.route.parser.druid.MycatSchemaStatVisitor; import io.mycat.server.parser.ServerParse; /** * 特定 SQL 查询条件的统计分析 * -------------------------------------------------- * * 例: * SELECT * FROM v1user Where userName=? AND cityName =? * SELECT * FROM v1user Where userName=? * SELECT * FROM v1user Where userName=? AND age > 20 * * SELECT * FROM v1user Where userName = "张三" AND cityName = "北京"; * SELECT * FROM v1user Where userName = "李四" * SELECT * FROM v1user Where userName = "张三" AND age > 20 * * 现在我们希望知道DB 中 业务比较关注的 userName 有哪些，次数是多少, 怎么处理哩，如下 * * 设置： 表名&条件列 ( v1user&userName ) 即可，取消请设置 NULL * * @author zhuam * */ public class QueryConditionAnalyzer implements QueryResultListener { private final static long MAX_QUERY_MAP_SIZE = 100000; private static final Logger LOGGER = LoggerFactory.getLogger(QueryConditionAnalyzer.class); private String tableName = null; private String columnName = null; // column value -> count // private final HashMap map = new HashMap(); private final Map map = new ConcurrentHashMap<>(); private ReentrantLock lock = new ReentrantLock(); private SQLParser sqlParser = new SQLParser(); private final static QueryConditionAnalyzer instance = new QueryConditionAnalyzer(); private QueryConditionAnalyzer() {} public static QueryConditionAnalyzer getInstance() { return instance; } @Override public void onQueryResult(QueryResult queryResult) { // this.lock.lock(); // try { int sqlType = queryResult.getSqlType(); String sql = queryResult.getSql(); switch(sqlType) { case ServerParse.SELECT: List values = sqlParser.parseConditionValues(sql, this.tableName, this.columnName); if ( values != null ) { if ( this.map.size() < MAX_QUERY_MAP_SIZE ) { for(Object value : values) { AtomicLong count = this.map.get(value); if (count == null) { count = new AtomicLong(1L); } else { count.getAndIncrement(); } this.map.put(value, count); } } else { LOGGER.debug(" this map is too large size "); } } } // } finally { // this.lock.unlock(); // } } public boolean setCf(String cf) { boolean isOk = false; this.lock.lock(); try { if ( !"NULL".equalsIgnoreCase(cf) ) { String[] table_column = cf.split("&"); if ( table_column != null && table_column.length == 2 ) { this.tableName = table_column[0]; this.columnName = table_column[1]; this.map.clear(); isOk = true; } } else { this.tableName = null; this.columnName = null; this.map.clear(); isOk = true; } } finally { this.lock.unlock(); } return isOk; } public String getKey() { return this.tableName + "." + this.columnName; } public List> getValues() { List> list = new ArrayList>(map.entrySet()); return list; } // SQL 解析 class SQLParser { /** * 去掉库名、去掉`` * @param tableName * @return */ private String fixName(String tableName) { if ( tableName != null ) { tableName = tableName.replace("`", ""); int dotIdx = tableName.indexOf("."); if ( dotIdx > 0 ) { tableName = tableName.substring(1 + dotIdx).trim(); } } return tableName; } /** * 解析 SQL 获取指定表及条件列的值 * * @param sql * @param tableName * @param colnumName * @return */ public List parseConditionValues(String sql, String tableName, String colnumName) { List values = null; if ( sql != null && tableName != null && columnName != null ) { values = new ArrayList(); MySqlStatementParser parser = new MySqlStatementParser(sql); SQLStatement stmt = parser.parseStatement(); MycatSchemaStatVisitor visitor = new MycatSchemaStatVisitor(); stmt.accept(visitor); String currentTable = visitor.getCurrentTable(); if ( tableName.equalsIgnoreCase( currentTable ) ) { List conditions = visitor.getConditions(); for(Condition condition: conditions) { String ccN = condition.getColumn().getName(); ccN = fixName(ccN); if ( colnumName.equalsIgnoreCase( ccN ) ) { List ccVL = condition.getValues(); values.addAll( ccVL ); } } } } return values; } } /* ----------------------------------------------------------------- public static void main(String arg[]) { String sql = "SELECT `fnum`, `forg`, `fdst`, `airline`, `ftype` , `ports_of_call`, " + "`scheduled_deptime`, `scheduled_arrtime`, `actual_deptime`, `actual_arrtime`, " + "`flight_status_code` FROM dynamic " + "WHERE `fnum` = 'CA123' AND `forg` = 'PEK' AND `fdst` = 'SHA' " + "AND `scheduled_deptime` BETWEEN 1212121 AND 232323233 " + "AND `fservice` = 'J' AND `fcategory` = 1 " + "AND `share_execute_flag` = 1 ORDER BY scheduled_deptime"; QueryResult qr = new QueryResult("zhuam", ServerParse.SELECT, sql, 0); QueryConditionAnalyzer analyzer = QueryConditionAnalyzer.getInstance(); analyzer.setTableColumnFilter("dynamic&fnum"); analyzer.onQuery(qr); List> list = analyzer.getValues(); System.out.println( list ); } */ } ================================================ FILE: src/main/java/io/mycat/statistic/stat/QueryResult.java ================================================ package io.mycat.statistic.stat; /** * SQL 执行结果 * * @author zhuam * */ public class QueryResult { private String schema; // 逻辑schema private String user; //用户 private int sqlType; //SQL类型 private String sql; //SQL private long sqlRows; //SQL 返回或影响的结果集长度 private long netInBytes; //NET IN 字节数 private long netOutBytes; //NET OUT 字节数 private long startTime; //开始时间 private long endTime; //结束时间 private int resultSize; //结果集大小 private String host; public QueryResult(String schema, String user, int sqlType, String sql, long sqlRows, long netInBytes, long netOutBytes, long startTime, long endTime , int resultSize, String host) { super(); this.schema = schema == null ? "" : schema; this.user = user; this.sqlType = sqlType; this.sql = sql; this.sqlRows = sqlRows; this.netInBytes = netInBytes; this.netOutBytes = netOutBytes; this.startTime = startTime; this.endTime = endTime; this.resultSize=resultSize; this.host = host; } public String getUser() { return user; } public int getSqlType() { return sqlType; } public String getSql() { return sql; } public long getSqlRows() { return sqlRows; } public long getNetInBytes() { return netInBytes; } public long getNetOutBytes() { return netOutBytes; } public long getStartTime() { return startTime; } public long getEndTime() { return endTime; } public int getResultSize() { return resultSize; } public String getHost() { return host; } public String getSchema() { return schema; } } ================================================ FILE: src/main/java/io/mycat/statistic/stat/QueryResultDispatcher.java ================================================ package io.mycat.statistic.stat; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.MycatServer; /** * SQL执行后的派发 QueryResult 事件 * * @author zhuam * */ public class QueryResultDispatcher { private static final Logger LOGGER = LoggerFactory.getLogger(QueryResultDispatcher.class); private static List listeners = new CopyOnWriteArrayList(); // 初始化强制加载 static { listeners.add( UserStatAnalyzer.getInstance() ); listeners.add( TableStatAnalyzer.getInstance() ); listeners.add( QueryConditionAnalyzer.getInstance() ); } public static void addListener(QueryResultListener listener) { if (listener == null) { throw new NullPointerException(); } listeners.add(listener); } public static void removeListener(QueryResultListener listener) { listeners.remove(listener); } public static void removeAllListener() { listeners.clear(); } public static void dispatchQuery(final QueryResult queryResult) { // 是否派发 QueryResult 事件 int useSqlStat = MycatServer.getInstance().getConfig().getSystem().getUseSqlStat(); if ( useSqlStat == 0 ) { return; } //TODO：异步分发，待进一步调优 MycatServer.getInstance().getBusinessExecutor().execute(new Runnable() { public void run() { for(QueryResultListener listener: listeners) { try { listener.onQueryResult( queryResult ); } catch(Exception e) { LOGGER.error("error:",e); } } } }); } } ================================================ FILE: src/main/java/io/mycat/statistic/stat/QueryResultListener.java ================================================ package io.mycat.statistic.stat; public interface QueryResultListener { public void onQueryResult(QueryResult queryResult); } ================================================ FILE: src/main/java/io/mycat/statistic/stat/SqlFrequency.java ================================================ package io.mycat.statistic.stat; import java.util.concurrent.atomic.AtomicLong; public class SqlFrequency implements Comparable{ private String sql; private AtomicLong count = new AtomicLong(0); private long lastTime = 0; private long executeTime = 0; private long allExecuteTime = 0; private long maxTime = 0; private long avgTime = 0; private long minTime = 0; private String host; public String getSql() { return sql; } public void setSql(String sql) { this.sql = sql; } public long getCount() { return this.count.get(); } public void incCount() { this.count.getAndIncrement(); } public long getLastTime() { return lastTime; } public void setLastTime(long lastTime) { this.lastTime = lastTime; } public long getExecuteTime() { return executeTime; } public long getMaxTime() { return maxTime; } public long getMinTime() { return minTime; } public long getAvgTime() { return avgTime; } public void setExecuteTime(long execTime) { if (execTime > this.maxTime) { this.maxTime = execTime; } if (this.minTime == 0) { this.minTime = execTime; } if (execTime > 0 && execTime < this.minTime) { this.minTime = execTime; } this.allExecuteTime+=execTime; if (count.get() > 0) { this.avgTime = this.allExecuteTime / this.count.get(); } this.executeTime = execTime; } @Override public int compareTo(SqlFrequency o) { long para = o.count.get() - count.get(); long para2 = o.lastTime - lastTime; return para == 0L ? (int)(para2 == 0L ? o.allExecuteTime - allExecuteTime : para2) : (int)para ; } @Override public boolean equals(Object obj) { if(obj instanceof SqlFrequency) { return this.compareTo((SqlFrequency)obj) == 0; } else { return super.equals(obj); } } public String getHost() { return host; } public void setHost(String host) { this.host = host; } } ================================================ FILE: src/main/java/io/mycat/statistic/stat/SqlResultSet.java ================================================ package io.mycat.statistic.stat; /** * 结果集记录模型 */ public class SqlResultSet { private String sql; private int resultSetSize = 0; private int count; public String getSql() { return sql; } public void setSql(String sql) { this.sql = sql; } public int getResultSetSize() { return resultSetSize; } public void setResultSetSize(int resultSetSize) { this.resultSetSize = resultSetSize; } public int getCount() { return count; } public void count() { this.count++; } } ================================================ FILE: src/main/java/io/mycat/statistic/stat/SqlResultSizeRecorder.java ================================================ package io.mycat.statistic.stat; import java.util.concurrent.ConcurrentHashMap; import com.alibaba.druid.DbType; import com.alibaba.druid.sql.visitor.ParameterizedOutputVisitorUtils; /** * 大结果集 SQL * */ public class SqlResultSizeRecorder { private ConcurrentHashMap sqlResultSetMap = new ConcurrentHashMap(); public void addSql(String sql,int resultSetSize ){ SqlResultSet sqlResultSet; SqlParser sqlParserHigh = new SqlParser(); sql=sqlParserHigh.mergeSql(sql); if(this.sqlResultSetMap.containsKey(sql)){ sqlResultSet =this.sqlResultSetMap.get(sql); sqlResultSet.count(); sqlResultSet.setSql(sql); //System.out.println(sql); sqlResultSet.setResultSetSize(resultSetSize); }else{ sqlResultSet = new SqlResultSet(); sqlResultSet.setResultSetSize(resultSetSize); sqlResultSet.setSql(sql); this.sqlResultSetMap.put(sql, sqlResultSet); } } /** * 获取 SQL 大结果集记录 */ public ConcurrentHashMap getSqlResultSet() { return sqlResultSetMap; } public void clearSqlResultSet() { sqlResultSetMap.clear(); } class SqlParser { public String fixSql(String sql) { if ( sql != null) return sql.replace("\n", " "); return sql; } public String mergeSql(String sql) { String newSql = ParameterizedOutputVisitorUtils.parameterize(sql, DbType.mysql); return fixSql( newSql ); } } } ================================================ FILE: src/main/java/io/mycat/statistic/stat/TableStat.java ================================================ package io.mycat.statistic.stat; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import io.mycat.server.parser.ServerParse; /** * SQL统计中,统计出来每个表的读,写的TPS,分辨出当前最热的表， * 并且根据是否有关联JOIN来区分为几个不同的“区域”,是一个重要功能,意味着,某些表可以转移到其他的数据库里,做智能优化。 * * 首先是每个表的读写TPS 2个指标,有时段。然后是 哪那些表有JOIN查询 ，来区分 独立的区域 * * @author zhuam * */ public class TableStat implements Comparable { //1、读写 //2、主表 //3、关联表 次数 //4、读写 TPS public String table; private final AtomicLong rCount = new AtomicLong(0); private final AtomicLong wCount = new AtomicLong(0); // 关联表 private final ConcurrentHashMap relaTableMap = new ConcurrentHashMap(); /** * 最后执行时间 */ private long lastExecuteTime; public TableStat(String table) { super(); this.table = table; } public void reset() { this.rCount.set(0); this.wCount.set(0); this.relaTableMap.clear(); this.lastExecuteTime = 0; } public void update(int sqlType, String sql, long startTime, long endTime, List relaTables) { //记录 RW switch(sqlType) { case ServerParse.SELECT: this.rCount.incrementAndGet(); break; case ServerParse.UPDATE: case ServerParse.INSERT: case ServerParse.DELETE: case ServerParse.REPLACE: this.wCount.incrementAndGet(); break; } // 记录 关联表执行情况 for(String table: relaTables) { RelaTable relaTable = this.relaTableMap.get( table ); if ( relaTable == null ) { relaTable = new RelaTable(table, 1); } else { relaTable.incCount(); } this.relaTableMap.put(table, relaTable); } this.lastExecuteTime = endTime; } public String getTable() { return table; } public long getRCount() { return this.rCount.get(); } public long getWCount() { return this.wCount.get(); } public int getCount() { return (int)(getRCount()+getWCount()); } public List getRelaTables() { List tables = new ArrayList(); tables.addAll( this.relaTableMap.values() ); return tables; } public long getLastExecuteTime() { return lastExecuteTime; } @Override public int compareTo(TableStat o) { long para = o.getCount() - getCount(); long para2 = o.getLastExecuteTime() - getLastExecuteTime(); return para == 0? (para2 == 0? o.getTable().hashCode() - getTable().hashCode() :(int) para2) : (int)para ; } @Override public boolean equals(Object obj) { if(obj instanceof TableStat) { return this.compareTo((TableStat)obj) == 0; } else { return super.equals(obj); } } /** * 关联表 * @author Ben * */ public static class RelaTable { private String tableName; private int count; public RelaTable(String tableName, int count) { super(); this.tableName = tableName; this.count = count; } public String getTableName() { return this.tableName; } public int getCount() { return this.count; } public void incCount() { this.count++; } } } ================================================ FILE: src/main/java/io/mycat/statistic/stat/TableStatAnalyzer.java ================================================ package io.mycat.statistic.stat; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.druid.DbType; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.ast.statement.SQLDeleteStatement; import com.alibaba.druid.sql.ast.statement.SQLExprTableSource; import com.alibaba.druid.sql.ast.statement.SQLInsertStatement; import com.alibaba.druid.sql.ast.statement.SQLReplaceStatement; import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; import com.alibaba.druid.sql.ast.statement.SQLUpdateStatement; import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter; import com.alibaba.druid.sql.parser.SQLParserUtils; import com.alibaba.druid.sql.parser.SQLStatementParser; import com.alibaba.druid.sql.visitor.SQLASTVisitorAdapter; import io.mycat.server.parser.ServerParse; /** * 按SQL表名进行计算 * * @author zhuam * */ public class TableStatAnalyzer implements QueryResultListener { private static final Logger LOGGER = LoggerFactory.getLogger(TableStatAnalyzer.class); private Map tableStatMap = new ConcurrentHashMap<>(); private ReentrantLock lock = new ReentrantLock(); //解析SQL 提取表名 private SQLParser sqlParser = new SQLParser(); private final static TableStatAnalyzer instance = new TableStatAnalyzer(); private TableStatAnalyzer() {} public static TableStatAnalyzer getInstance() { return instance; } @Override public void onQueryResult(QueryResult queryResult) { int sqlType = queryResult.getSqlType(); String sql = queryResult.getSql(); switch(sqlType) { case ServerParse.SELECT: case ServerParse.UPDATE: case ServerParse.INSERT: case ServerParse.DELETE: case ServerParse.REPLACE: //关联表提取 String masterTable = null; List relaTables = new ArrayList(); List tables = sqlParser.parseTableNames(sql); for(int i = 0; i < tables.size(); i++) { String table = tables.get(i); if ( i == 0 ) { masterTable = table; } else { relaTables.add( table ); } } if ( masterTable != null ) { TableStat tableStat = getTableStat( masterTable ); tableStat.update(sqlType, sql, queryResult.getStartTime(), queryResult.getEndTime(), relaTables); } break; } } private TableStat getTableStat(String tableName) { TableStat userStat = tableStatMap.get(tableName); if (userStat == null) { if(lock.tryLock()){ try{ userStat = new TableStat(tableName); tableStatMap.put(tableName, userStat); } finally { lock.unlock(); } }else{ while(userStat == null){ userStat = tableStatMap.get(tableName); } } } return userStat; } public Map getTableStatMap() { Map map = new LinkedHashMap(tableStatMap.size()); map.putAll(tableStatMap); return map; } /** * 获取 table 访问排序统计 */ public List getTableStats(boolean isClear) { SortedSet tableStatSortedSet = new TreeSet<>(tableStatMap.values()); List list = new ArrayList<>(tableStatSortedSet); return list; } public void ClearTable() { tableStatMap.clear(); } /** * 解析 table name */ private static class SQLParser { private SQLStatement parseStmt(String sql) { SQLStatementParser statParser = SQLParserUtils.createSQLStatementParser(sql, "mysql"); SQLStatement stmt = statParser.parseStatement(); return stmt; } /** * 去掉库名、去掉`` * @param tableName * @return */ private String fixName(String tableName) { if ( tableName != null ) { tableName = tableName.replace("`", ""); int dotIdx = tableName.indexOf("."); if ( dotIdx > 0 ) { tableName = tableName.substring(1 + dotIdx).trim(); } } return tableName; } /** * 解析 SQL table name */ public List parseTableNames(String sql) { final List tables = new ArrayList(); try{ SQLStatement stmt = parseStmt(sql); if (stmt instanceof SQLReplaceStatement) { String table = ((SQLReplaceStatement) stmt).getTableName().getSimpleName(); tables.add( fixName( table ) ); } else if (stmt instanceof SQLInsertStatement ) { String table = ((SQLInsertStatement)stmt).getTableName().getSimpleName(); tables.add( fixName( table ) ); } else if (stmt instanceof SQLUpdateStatement ) { String table = ((SQLUpdateStatement)stmt).getTableName().getSimpleName(); tables.add( fixName( table ) ); } else if (stmt instanceof SQLDeleteStatement ) { // Ken.li 2020/04/02 //String table = ((SQLDeleteStatement)stmt).getTableName().getSimpleName(); //tables.add( fixName( table ) ); ((SQLDeleteStatement)stmt).getTableSource().accept(new SQLASTVisitorAdapter() { public boolean visit(SQLExprTableSource x){ tables.add( fixName( x.toString() ) ); return super.visit(x); } }); if (((SQLDeleteStatement)stmt).getFrom() != null) { ((SQLDeleteStatement)stmt).getFrom().accept(new SQLASTVisitorAdapter() { public boolean visit(SQLExprTableSource x){ if (tables.contains(x.getAlias())) { tables.remove(x.getAlias()); tables.add( fixName( x.toString() ) ); } return super.visit(x); } }); } } else if (stmt instanceof SQLSelectStatement ) { //TODO: modify by owenludong DbType dbType = ((SQLSelectStatement) stmt).getDbType(); if (DbType.mysql.equals(dbType)) { stmt.accept(new MySqlASTVisitorAdapter() { public boolean visit(SQLExprTableSource x){ tables.add( fixName( x.toString() ) ); return super.visit(x); } }); } else { stmt.accept(new SQLASTVisitorAdapter() { public boolean visit(SQLExprTableSource x){ tables.add( fixName( x.toString() ) ); return super.visit(x); } }); } } } catch (Exception e) { LOGGER.error("TableStatAnalyzer err:" + sql, e); } return tables; } } /* public static void main(String[] args) { List sqls = new ArrayList(); sqls.add( "SELECT id, name, age FROM v1select1 a LEFT OUTER JOIN v1select2 b ON a.id = b.id WHERE a.name = 12 "); sqls.add( "insert into v1user_insert(id, name) values(1,3)"); sqls.add( "delete from v1user_delete where id= 2"); sqls.add( "update v1user_update set id=2 where id=3"); sqls.add( "select ename,deptno,sal from v1user_subquery1 where deptno=(select deptno from v1user_subquery2 where loc='NEW YORK')"); sqls.add( "replace into v1user_insert(id, name) values(1,3)"); sqls.add( "select * from v1xx where id=3 group by zz"); sqls.add( "select * from v1yy where xx=3 limit 0,3"); sqls.add( "SELECT * FROM (SELECT * FROM posts ORDER BY dateline DESC) GROUP BY tid ORDER BY dateline DESC LIMIT 10"); for(String sql: sqls) { List tables = TableStatAnalyzer.getInstance().sqlParser.parseTableNames(sql); for(String t: tables) { System.out.println( t ); } } } */ } ================================================ FILE: src/main/java/io/mycat/statistic/stat/UserSqlHighStat.java ================================================ package io.mycat.statistic.stat; import java.util.ArrayList; import java.util.List; import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.druid.DbType; import com.alibaba.druid.sql.visitor.ParameterizedOutputVisitorUtils; public class UserSqlHighStat { private static final int CAPACITY_SIZE = 1024; private ConcurrentHashMap sqlFrequencyMap = new ConcurrentHashMap<>(); private SqlParser sqlParser = new SqlParser(); private static final Logger LOGGER = LoggerFactory.getLogger(UserSqlHighStat.class); public void addSql(String sql, long executeTime, long startTime, long endTime,String host) { String newSql = this.sqlParser.mergeSql(sql); if (newSql != null) { this.sqlFrequencyMap.compute(newSql, (s, sqlFrequency) -> { if (sqlFrequency == null) { sqlFrequency = new SqlFrequency(); sqlFrequency.setSql(s); } sqlFrequency.setLastTime(endTime); sqlFrequency.incCount(); sqlFrequency.setExecuteTime(executeTime); sqlFrequency.setHost(host); return sqlFrequency; }); } } /** * 获取 SQL 访问频率 */ public List getSqlFrequency(boolean isClear) { List list = new ArrayList<>(this.sqlFrequencyMap.values()); if (isClear) { clearSqlFrequency(); } return list; } private void clearSqlFrequency() { sqlFrequencyMap.clear(); } public void recycle() { if (sqlFrequencyMap.size() > CAPACITY_SIZE) { ConcurrentHashMap sqlFrequencyMap2 = new ConcurrentHashMap<>(); SortedSet sqlFrequencySortedSet = new TreeSet<>(this.sqlFrequencyMap.values()); List keyList = new ArrayList(sqlFrequencySortedSet); int i = 0; for (SqlFrequency key : keyList) { if (i == CAPACITY_SIZE) { break; } sqlFrequencyMap2.put(key.getSql(), key); i++; } sqlFrequencyMap = sqlFrequencyMap2; } } private static class SqlParser { public String fixSql(String sql) { if (sql != null) { return sql.replace("\n", " "); } return sql; } public String mergeSql(String sql) { try { String newSql = ParameterizedOutputVisitorUtils.parameterize(sql, DbType.mysql); return fixSql(newSql); } catch (Exception e) { LOGGER.warn("user sql high parse:{} error",sql, e); return null; } } } } ================================================ FILE: src/main/java/io/mycat/statistic/stat/UserSqlLargeStat.java ================================================ package io.mycat.statistic.stat; import java.util.ArrayList; import java.util.List; import java.util.SortedSet; import java.util.concurrent.ConcurrentSkipListSet; public class UserSqlLargeStat { private final int count; private SortedSet sqls; public UserSqlLargeStat(int count) { this.count = count; this.sqls = new ConcurrentSkipListSet<>(); } public List getSqls() { List list = new ArrayList<>(sqls); return list; } public void add(String sql, long sqlRows, long executeTime, long startTime, long endTime,String host) { SqlLarge sqlLarge = new SqlLarge(sql, sqlRows, executeTime, startTime, endTime,host); this.add( sqlLarge ); } public void add(SqlLarge sql) { sqls.add(sql); } public void reset() { this.clear(); } public void clear() { sqls.clear(); } public void recycle() { if(sqls.size() > count){ SortedSet sqls2 = new ConcurrentSkipListSet<>(); List keyList = new ArrayList(sqls); int i = 0; for(SqlLarge key : keyList){ if(i == count) { break; } sqls2.add(key); i++; } sqls = sqls2; } } /** * 记录 SQL 及返回行数 */ public static class SqlLarge implements Comparable { private String sql; private long sqlRows; private long executeTime; private long startTime; private long endTime; private String host; public SqlLarge(String sql, long sqlRows, long executeTime, long startTime, long endTime,String host) { super(); this.sql = sql; this.sqlRows = sqlRows; this.executeTime = executeTime; this.startTime = startTime; this.endTime = endTime; this.host = host; } public String getSql() { return sql; } public long getSqlRows() { return sqlRows; } public long getStartTime() { return startTime; } public long getExecuteTime() { return executeTime; } public long getEndTime() { return endTime; } @Override public int compareTo(SqlLarge o) { long para = o.sqlRows - sqlRows; return para == 0 ? (o.sql.hashCode() - sql.hashCode()) :(int) (para); } @Override public boolean equals(Object arg0) { return super.equals(arg0); } @Override public int hashCode() { return super.hashCode(); } public String getHost() { return host; } } } ================================================ FILE: src/main/java/io/mycat/statistic/stat/UserSqlLastStat.java ================================================ package io.mycat.statistic.stat; import java.util.*; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.locks.ReentrantLock; /** * 最后执行的 Sql * * @author zhuam * */ public class UserSqlLastStat { private static final int MAX_RECORDS = 1024; private SortedSet sqls; public UserSqlLastStat(int count) { this.sqls = new ConcurrentSkipListSet<>(); } public List getSqls() { List keyList = new ArrayList(sqls); return keyList; } public void add(String sql, long executeTime, long startTime, long endTime,String host) { SqlLast sqlLast = new SqlLast(sql, executeTime, startTime, endTime,host); sqls.add(sqlLast); } public void reset() { this.clear(); } public void clear() { sqls.clear(); } public void recycle(){ if(sqls.size() > MAX_RECORDS){ SortedSet sqls2 = new ConcurrentSkipListSet<>(); List keyList = new ArrayList(sqls); int i = 0; for(SqlLast key : keyList){ if(i == MAX_RECORDS) { break; } sqls2.add(key); i++; } sqls = sqls2; } } /** * 记录SQL */ public static class SqlLast implements Comparable{ private String sql; private long executeTime; private long startTime; private long endTime; private String host; public SqlLast(String sql, long executeTime, long startTime, long endTime,String host) { super(); this.sql = sql; this.executeTime = executeTime; this.startTime = startTime; this.endTime = endTime; this.host = host; } public String getSql() { return sql; } public long getStartTime() { return startTime; } public long getExecuteTime() { return executeTime; } public long getEndTime() { return endTime; } @Override public int compareTo(SqlLast o) { long st1 = o == null ? 0 : o.getStartTime(); return (int) ( st1 - this.getStartTime()); } @Override public boolean equals(Object obj) { if(obj instanceof SqlLast) { return this.compareTo((SqlLast)obj) == 0; } else { return super.equals(obj); } } public String getHost() { return host; } } } ================================================ FILE: src/main/java/io/mycat/statistic/stat/UserSqlRWStat.java ================================================ package io.mycat.statistic.stat; import java.util.TimeZone; import java.util.concurrent.atomic.AtomicLong; import io.mycat.server.parser.ServerParse; /** * SQL R/W 执行状态 * 因为这里的所有元素都近似为非必须原子更新的，即： * 例如：rCount和netInBytes没有必要非得保持同步更新，在同一个事务内 * 只要最后更新了即可，所以将其中的元素拆成一个一个原子类，没必要保证精确的保持原样不加任何锁 * * @author zhuam * */ public class UserSqlRWStat { /** * R/W 次数 */ private AtomicLong rCount = new AtomicLong(0L); private AtomicLong wCount = new AtomicLong(0L); /** * 每秒QPS */ private int qps = 0; /** * Net In/Out 字节数 */ private AtomicLong netInBytes = new AtomicLong(0L); private AtomicLong netOutBytes = new AtomicLong(0L); /** * 最大的并发 */ private int concurrentMax = 1; /** * 执行耗时 * * 10毫秒内、 10 - 200毫秒内、 1秒内、 超过 1秒 */ private final Histogram timeHistogram = new Histogram( new long[] { 10, 200, 1000, 2000 } ); /** * 执行所在时段 * * 22-06 夜间、 06-13 上午、 13-18下午、 18-22 晚间 */ private final Histogram executeHistogram = new Histogram(new long[] { 6, 13, 18, 22 }); /** * 最后执行时间 * 不用很精确，所以不加任何锁 */ private long lastExecuteTime; private int time_zone_offset = 0; private int one_hour = 3600 * 1000; public UserSqlRWStat() { this.time_zone_offset = TimeZone.getDefault().getRawOffset(); } public void reset() { this.rCount = new AtomicLong(0L); this.wCount = new AtomicLong(0L); this.concurrentMax = 1; this.lastExecuteTime = 0; this.netInBytes = new AtomicLong(0L); this.netOutBytes = new AtomicLong(0L); this.timeHistogram.reset(); this.executeHistogram.reset(); } public void add(int sqlType, String sql, long executeTime, long netInBytes, long netOutBytes, long startTime, long endTime) { switch(sqlType) { case ServerParse.SELECT: case ServerParse.SHOW: this.rCount.incrementAndGet(); break; case ServerParse.UPDATE: case ServerParse.INSERT: case ServerParse.DELETE: case ServerParse.REPLACE: this.wCount.incrementAndGet(); break; } //SQL执行所在的耗时区间 if ( executeTime <= 10 ) { this.timeHistogram.record(10); } else if ( executeTime > 10 && executeTime <= 200 ) { this.timeHistogram.record(200); } else if ( executeTime > 200 && executeTime <= 1000 ) { this.timeHistogram.record(1000); } else if ( executeTime > 1000) { this.timeHistogram.record(2000); } //SQL执行所在的时间区间 long hour0 = endTime / ( 24L * (long)one_hour ) * ( 24L * (long)one_hour )- (long)time_zone_offset; long hour06 = hour0 + 6L * (long)one_hour - 1L; long hour13 = hour0 + 13L * (long)one_hour - 1L; long hour18 = hour0 + 18L * (long)one_hour - 1L; long hour22 = hour0 + 22L * (long)one_hour - 1L; if ( endTime <= hour06 || endTime > hour22 ) { this.executeHistogram.record(6); } else if ( endTime > hour06 && endTime <= hour13 ) { this.executeHistogram.record(13); } else if ( endTime > hour13 && endTime <= hour18 ) { this.executeHistogram.record(18); } else if ( endTime > hour18 && endTime <= hour22 ) { this.executeHistogram.record(22); } this.lastExecuteTime = endTime; this.netInBytes.addAndGet(netInBytes); this.netOutBytes.addAndGet(netOutBytes); } public long getLastExecuteTime() { return lastExecuteTime; } public long getNetInBytes() { return netInBytes.get(); } public long getNetOutBytes() { return netOutBytes.get(); } public int getConcurrentMax() { return concurrentMax; } public void setConcurrentMax(int concurrentMax) { this.concurrentMax = concurrentMax; } public int getQps() { return qps; } public void setQps(int qps) { this.qps = qps; } public long getRCount() { return this.rCount.get(); } public long getWCount() { return this.wCount.get(); } public Histogram getTimeHistogram() { return this.timeHistogram; } public Histogram getExecuteHistogram() { return this.executeHistogram; } } ================================================ FILE: src/main/java/io/mycat/statistic/stat/UserStat.java ================================================ package io.mycat.statistic.stat; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantReadWriteLock; import io.mycat.MycatServer; import io.mycat.server.parser.ServerParse; import io.mycat.statistic.SQLRecord; import io.mycat.statistic.SQLRecorder; /** * 用户状态 * * @author Ben */ public class UserStat { private long SQL_SLOW_TIME = 100; private String user; /** * 最大的并发 */ private final AtomicInteger runningCount = new AtomicInteger(); private final AtomicInteger concurrentMax = new AtomicInteger(); /** * SQL 大集合插入、返回记录 */ private UserSqlLargeStat sqlLargeStat = null; /** * SQL 执行记录 */ private UserSqlLastStat sqlLastStat = null; /** * CURD 执行分布 */ private UserSqlRWStat sqlRwStat = null; /** * 用户高频SQL分析 */ private UserSqlHighStat sqlHighStat = null; /** * 慢查询记录器 TOP 10 */ private SQLRecorder sqlRecorder; /** * 大结果集记录 */ private SqlResultSizeRecorder sqlResultSizeRecorder = null; /** * 读写锁 */ // private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); public UserStat(String user) { super(); int size = MycatServer.getInstance().getConfig().getSystem().getSqlRecordCount(); this.user = user; this.sqlRwStat = new UserSqlRWStat(); this.sqlLastStat = new UserSqlLastStat(50); this.sqlLargeStat = new UserSqlLargeStat(10); this.sqlHighStat = new UserSqlHighStat(); this.sqlRecorder = new SQLRecorder( size ); this.sqlResultSizeRecorder = new SqlResultSizeRecorder(); } public String getUser() { return user; } public SQLRecorder getSqlRecorder() { return sqlRecorder; } public UserSqlRWStat getRWStat() { return sqlRwStat; } public UserSqlLastStat getSqlLastStat() { return this.sqlLastStat; } public UserSqlLargeStat getSqlLargeRowStat() { return this.sqlLargeStat; } public UserSqlHighStat getSqlHigh(){ return this.sqlHighStat; } public SqlResultSizeRecorder getSqlResultSizeRecorder() { return this.sqlResultSizeRecorder; } public void setSlowTime(long time) { this.SQL_SLOW_TIME = time; this.sqlRecorder.clear(); } public void clearSql() { this.sqlLastStat.reset(); } public void clearSqlslow() { this.sqlRecorder.clear(); } public void clearRwStat() { this.sqlRwStat.reset(); } public void reset() { this.sqlRecorder.clear(); this.sqlResultSizeRecorder.clearSqlResultSet(); this.sqlRwStat.reset(); this.sqlLastStat.reset(); this.runningCount.set(0); this.concurrentMax.set(0); } /** * 更新状态 * * @param sqlType * @param sql * @param startTime */ public void update(String schema, int sqlType, String sql, long sqlRows, long netInBytes, long netOutBytes, long startTime, long endTime ,int rseultSetSize,String host) { //before 计算最大并发数 //----------------------------------------------------- int invoking = runningCount.incrementAndGet(); for (;;) { int max = concurrentMax.get(); if (invoking > max) { if (concurrentMax.compareAndSet(max, invoking)) { break; } } else { break; } } //----------------------------------------------------- // this.lock.writeLock().lock(); // try { //慢查询记录 long executeTime = endTime - startTime; if ( executeTime >= SQL_SLOW_TIME ){ SQLRecord record = new SQLRecord(); record.executeTime = executeTime; record.statement = sql; record.startTime = startTime; record.host = host; record.schema = schema; this.sqlRecorder.add(record); } //执行状态记录 this.sqlRwStat.setConcurrentMax( concurrentMax.get() ); this.sqlRwStat.add(sqlType, sql, executeTime, netInBytes, netOutBytes, startTime, endTime); //记录最新执行的SQL this.sqlLastStat.add(sql, executeTime, startTime, endTime ,host); //记录高频SQL this.sqlHighStat.addSql(sql, executeTime, startTime, endTime,host); //记录SQL Select 返回超过 10000 行的 大结果集 if ( sqlType == ServerParse.SELECT && sqlRows > 10000 ) { this.sqlLargeStat.add(sql, sqlRows, executeTime, startTime, endTime,host); } //记录超过阈值的大结果集sql if(rseultSetSize>=MycatServer.getInstance().getConfig().getSystem().getMaxResultSet()){ this.sqlResultSizeRecorder.addSql(sql, rseultSetSize); } // } finally { // this.lock.writeLock().unlock(); // } //after //----------------------------------------------------- runningCount.decrementAndGet(); } } ================================================ FILE: src/main/java/io/mycat/statistic/stat/UserStatAnalyzer.java ================================================ package io.mycat.statistic.stat; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import io.mycat.server.parser.ServerParse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * 按访问用户 计算SQL的运行状态 * * @author Ben * */ public class UserStatAnalyzer implements QueryResultListener { private Cache userStatMap = CacheBuilder.newBuilder().maximumSize(8192).build(); private final static UserStatAnalyzer instance = new UserStatAnalyzer(); private static final Logger LOGGER = LoggerFactory.getLogger(UserStatAnalyzer.class); private UserStatAnalyzer() { } public static UserStatAnalyzer getInstance() { return instance; } @Override public void onQueryResult(QueryResult query) { switch( query.getSqlType() ) { case ServerParse.SELECT: case ServerParse.UPDATE: case ServerParse.INSERT: case ServerParse.DELETE: case ServerParse.REPLACE: String host = query.getHost(); if (host==null){ host = ""; } String schema = query.getSchema(); String user = query.getUser(); int sqlType = query.getSqlType(); String sql = query.getSql(); long sqlRows = query.getSqlRows(); long netInBytes = query.getNetInBytes(); long netOutBytes = query.getNetOutBytes(); long startTime = query.getStartTime(); long endTime = query.getEndTime(); int resultSetSize=query.getResultSize(); try { UserStat userStat = userStatMap.get(user, new Callable() { @Override public UserStat call() throws Exception { return new UserStat(user); } }); userStat.update(schema,sqlType, sql, sqlRows, netInBytes, netOutBytes, startTime, endTime, resultSetSize, host); }catch (ExecutionException e){ LOGGER.error("new UserStat occurs error",e); } break; } } public Map getUserStatMap() { return userStatMap.asMap(); } } ================================================ FILE: src/main/java/io/mycat/util/ByteBufferUtil.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.util; /* * BE ADVISED: New imports added here might introduce new dependencies for * the clientutil jar. If in doubt, run the `ant test-clientutil-jar' target * afterward, and ensure the tests still pass. */ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.io.InputStream; import java.net.InetAddress; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Arrays; /** * Utility methods to make ByteBuffers less painful * The following should illustrate the different ways byte buffers can be used * * public void testArrayOffet() * { * * byte[] b = "test_slice_array".getBytes(); * ByteBuffer bb = ByteBuffer.allocate(1024); * * assert bb.position() == 0; * assert bb.limit() == 1024; * assert bb.capacity() == 1024; * * bb.put(b); * * assert bb.position() == b.length; * assert bb.remaining() == bb.limit() - bb.position(); * * ByteBuffer bb2 = bb.slice(); * * assert bb2.position() == 0; * * //slice should begin at other buffers current position * assert bb2.arrayOffset() == bb.position(); * * //to match the position in the underlying array one needs to * //track arrayOffset * assert bb2.limit()+bb2.arrayOffset() == bb.limit(); * * * assert bb2.remaining() == bb.remaining(); * * } * * } * */ public class ByteBufferUtil { public static final ByteBuffer EMPTY_BYTE_BUFFER = ByteBuffer.wrap(new byte[0]); public static int compareUnsigned(ByteBuffer o1, ByteBuffer o2) { return FastByteOperations.compareUnsigned(o1, o2); } public static int compare(byte[] o1, ByteBuffer o2) { return FastByteOperations.compareUnsigned(o1, 0, o1.length, o2); } public static int compare(ByteBuffer o1, byte[] o2) { return FastByteOperations.compareUnsigned(o1, o2, 0, o2.length); } /** * Decode a String representation. * This method assumes that the encoding charset is UTF_8. * * @param buffer a byte buffer holding the string representation * @return the decoded string */ public static String string(ByteBuffer buffer) throws CharacterCodingException { return string(buffer, StandardCharsets.UTF_8); } /** * Decode a String representation. * This method assumes that the encoding charset is UTF_8. * * @param buffer a byte buffer holding the string representation * @param position the starting position in {@code buffer} to start decoding from * @param length the number of bytes from {@code buffer} to use * @return the decoded string */ public static String string(ByteBuffer buffer, int position, int length) throws CharacterCodingException { return string(buffer, position, length, StandardCharsets.UTF_8); } /** * Decode a String representation. * * @param buffer a byte buffer holding the string representation * @param position the starting position in {@code buffer} to start decoding from * @param length the number of bytes from {@code buffer} to use * @param charset the String encoding charset * @return the decoded string */ public static String string(ByteBuffer buffer, int position, int length, Charset charset) throws CharacterCodingException { ByteBuffer copy = buffer.duplicate(); copy.position(position); copy.limit(copy.position() + length); return string(copy, charset); } /** * Decode a String representation. * * @param buffer a byte buffer holding the string representation * @param charset the String encoding charset * @return the decoded string */ public static String string(ByteBuffer buffer, Charset charset) throws CharacterCodingException { return charset.newDecoder().decode(buffer.duplicate()).toString(); } /** * You should almost never use this. Instead, use the write* methods to avoid copies. */ public static byte[] getArray(ByteBuffer buffer) { int length = buffer.remaining(); if (buffer.hasArray()) { int boff = buffer.arrayOffset() + buffer.position(); return Arrays.copyOfRange(buffer.array(), boff, boff + length); } // else, DirectByteBuffer.get() is the fastest route byte[] bytes = new byte[length]; buffer.duplicate().get(bytes); return bytes; } /** * ByteBuffer adaptation of org.apache.commons.lang3.ArrayUtils.lastIndexOf method * * @param buffer the array to traverse for looking for the object, may be null * @param valueToFind the value to find * @param startIndex the start index (i.e. BB position) to travers backwards from * @return the last index (i.e. BB position) of the value within the array * [between buffer.position() and buffer.limit()]; -1 if not found. */ public static int lastIndexOf(ByteBuffer buffer, byte valueToFind, int startIndex) { assert buffer != null; if (startIndex < buffer.position()) { return -1; } else if (startIndex >= buffer.limit()) { startIndex = buffer.limit() - 1; } for (int i = startIndex; i >= buffer.position(); i--) { if (valueToFind == buffer.get(i)) { return i; } } return -1; } /** * Encode a String in a ByteBuffer using UTF_8. * * @param s the string to encode * @return the encoded string */ public static ByteBuffer bytes(String s) { return ByteBuffer.wrap(s.getBytes(StandardCharsets.UTF_8)); } /** * Encode a String in a ByteBuffer using the provided charset. * * @param s the string to encode * @param charset the String encoding charset to use * @return the encoded string */ public static ByteBuffer bytes(String s, Charset charset) { return ByteBuffer.wrap(s.getBytes(charset)); } /** * @return a new copy of the data in @param buffer * USUALLY YOU SHOULD USE ByteBuffer.duplicate() INSTEAD, which creates a new Buffer * (so you can mutate its position without affecting the original) without copying the underlying array. */ public static ByteBuffer clone(ByteBuffer buffer) { assert buffer != null; if (buffer.remaining() == 0) { return EMPTY_BYTE_BUFFER; } ByteBuffer clone = ByteBuffer.allocate(buffer.remaining()); if (buffer.hasArray()) { System.arraycopy(buffer.array(), buffer.arrayOffset() + buffer.position(), clone.array(), 0, buffer.remaining()); } else { clone.put(buffer.duplicate()); clone.flip(); } return clone; } public static void arrayCopy(ByteBuffer src, int srcPos, byte[] dst, int dstPos, int length) { FastByteOperations.copy(src, srcPos, dst, dstPos, length); } /** * Transfer bytes from one ByteBuffer to another. * This function acts as System.arrayCopy() but for ByteBuffers. * * @param src the source ByteBuffer * @param srcPos starting position in the source ByteBuffer * @param dst the destination ByteBuffer * @param dstPos starting position in the destination ByteBuffer * @param length the number of bytes to copy */ public static void arrayCopy(ByteBuffer src, int srcPos, ByteBuffer dst, int dstPos, int length) { FastByteOperations.copy(src, srcPos, dst, dstPos, length); } public static void writeWithLength(byte[] bytes, DataOutput out) throws IOException { out.writeInt(bytes.length); out.write(bytes); } /* @return An unsigned short in an integer. */ public static int readShortLength(DataInput in) throws IOException { return in.readUnsignedShort(); } public static byte[] readBytes(DataInput in, int length) throws IOException { assert length > 0 : "length is not > 0: " + length; byte[] bytes = new byte[length]; in.readFully(bytes); return bytes; } /** * Convert a byte buffer to an integer. * Does not change the byte buffer position. * * @param bytes byte buffer to convert to integer * @return int representation of the byte buffer */ public static int toInt(ByteBuffer bytes) { return bytes.getInt(bytes.position()); } public static long toLong(ByteBuffer bytes) { return bytes.getLong(bytes.position()); } public static float toFloat(ByteBuffer bytes) { return bytes.getFloat(bytes.position()); } public static double toDouble(ByteBuffer bytes) { return bytes.getDouble(bytes.position()); } public static ByteBuffer bytes(int i) { return ByteBuffer.allocate(4).putInt(0, i); } public static ByteBuffer bytes(long n) { return ByteBuffer.allocate(8).putLong(0, n); } public static ByteBuffer bytes(float f) { return ByteBuffer.allocate(4).putFloat(0, f); } public static ByteBuffer bytes(double d) { return ByteBuffer.allocate(8).putDouble(0, d); } public static InputStream inputStream(ByteBuffer bytes) { final ByteBuffer copy = bytes.duplicate(); return new InputStream() { public int read() { if (!copy.hasRemaining()) { return -1; } return copy.get() & 0xFF; } @Override public int read(byte[] bytes, int off, int len) { if (!copy.hasRemaining()) { return -1; } len = Math.min(len, copy.remaining()); copy.get(bytes, off, len); return len; } @Override public int available() { return copy.remaining(); } }; } /** * Compare two ByteBuffer at specified offsets for length. * Compares the non equal bytes as unsigned. * @param bytes1 First byte buffer to compare. * @param offset1 Position to start the comparison at in the first array. * @param bytes2 Second byte buffer to compare. * @param offset2 Position to start the comparison at in the second array. * @param length How many bytes to compare? * @return -1 if byte1 is less than byte2, 1 if byte2 is less than byte1 or 0 if equal. */ public static int compareSubArrays(ByteBuffer bytes1, int offset1, ByteBuffer bytes2, int offset2, int length) { if (bytes1 == null) { return bytes2 == null ? 0 : -1; } if (bytes2 == null) { return 1; } assert bytes1.limit() >= offset1 + length : "The first byte array isn't long enough for the specified offset and length."; assert bytes2.limit() >= offset2 + length : "The second byte array isn't long enough for the specified offset and length."; for (int i = 0; i < length; i++) { byte byte1 = bytes1.get(offset1 + i); byte byte2 = bytes2.get(offset2 + i); // if (byte1 == byte2) // continue; // compare non-equal bytes as unsigned if( byte1 != byte2 ) { return (byte1 & 0xFF) < (byte2 & 0xFF) ? -1 : 1; } } return 0; } public static ByteBuffer bytes(InetAddress address) { return ByteBuffer.wrap(address.getAddress()); } // Returns whether {@code prefix} is a prefix of {@code value}. public static boolean isPrefix(ByteBuffer prefix, ByteBuffer value) { if (prefix.remaining() > value.remaining()) { return false; } int diff = value.remaining() - prefix.remaining(); return prefix.equals(value.duplicate().limit(value.remaining() - diff)); } /** trims size of bytebuffer to exactly number of bytes in it, to do not hold too much memory */ public static ByteBuffer minimalBufferFor(ByteBuffer buf) { return buf.capacity() > buf.remaining() || !buf.hasArray() ? ByteBuffer.wrap(getArray(buf)) : buf; } // Doesn't change bb position public static int getShortLength(ByteBuffer bb, int position) { int length = (bb.get(position) & 0xFF) << 8; return length | (bb.get(position + 1) & 0xFF); } // changes bb position public static int readShortLength(ByteBuffer bb) { int length = (bb.get() & 0xFF) << 8; return length | (bb.get() & 0xFF); } // changes bb position public static void writeShortLength(ByteBuffer bb, int length) { bb.put((byte) ((length >> 8) & 0xFF)); bb.put((byte) (length & 0xFF)); } // changes bb position public static ByteBuffer readBytes(ByteBuffer bb, int length) { ByteBuffer copy = bb.duplicate(); copy.limit(copy.position() + length); bb.position(bb.position() + length); return copy; } // changes bb position public static ByteBuffer readBytesWithShortLength(ByteBuffer bb) { int length = readShortLength(bb); return readBytes(bb, length); } } ================================================ FILE: src/main/java/io/mycat/util/ByteUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import java.nio.charset.Charset; import java.util.Date; public class ByteUtil { /** * compare to number or dicamal ascii byte array, for number :123456 ,store * to array [1,2,3,4,5,6] * * @param b1 * @param b2 * @return -1 means b1 < b2, or 0 means b1=b2 else return 1 */ public static int compareNumberByte(byte[] b1, byte[] b2) { if(b1 == null || b1.length == 0) { return -1; } else if(b2 == null || b2.length == 0) { return 1; } boolean isNegetive = b1[0] == 45 || b2[0] == 45; if (isNegetive == false && b1.length != b2.length) { return b1.length - b2.length; } int len = b1.length > b2.length ? b2.length : b1.length; int result = 0; int index = -1; for (int i = 0; i < len; i++) { int b1val = b1[i]; int b2val = b2[i]; if (b1val > b2val) { result = 1; index = i; break; } else if (b1val < b2val) { index = i; result = -1; break; } } if (index == 0) { // first byte compare return result; } else { if( b1.length != b2.length ) { int lenDelta = b1.length - b2.length; return isNegetive ? 0 - lenDelta : lenDelta; } else { return isNegetive ? 0 - result : result; } } } public static byte[] compareNumberArray2(byte[] b1, byte[] b2, int order) { if (b1.length <= 0 && b2.length > 0) { return b2; } if (b1.length > 0 && b2.length <= 0) { return b1; } int len = b1.length > b2.length ? b1.length : b2.length; for (int i = 0; i < len; i++) { if (b1[i] != b2[i]) { if (order == 1) { return ((b1[i] & 0xff) - (b2[i] & 0xff)) > 0 ? b1 : b2; } else { return ((b1[i] & 0xff) - (b2[i] & 0xff)) > 0 ? b2 : b1; } } } return b1; } public static byte[] getBytes(short data) { byte[] bytes = new byte[2]; bytes[0] = (byte) (data & 0xff); bytes[1] = (byte) ((data & 0xff00) >> 8); return bytes; } public static byte[] getBytes(char data) { byte[] bytes = new byte[2]; bytes[0] = (byte) (data); bytes[1] = (byte) (data >> 8); return bytes; } public static byte[] getBytes(int data) { byte[] bytes = new byte[4]; bytes[0] = (byte) (data & 0xff); bytes[1] = (byte) ((data & 0xff00) >> 8); bytes[2] = (byte) ((data & 0xff0000) >> 16); bytes[3] = (byte) ((data & 0xff000000) >> 24); return bytes; } public static byte[] getBytes(long data) { byte[] bytes = new byte[8]; bytes[0] = (byte) (data & 0xff); bytes[1] = (byte) ((data >> 8) & 0xff); bytes[2] = (byte) ((data >> 16) & 0xff); bytes[3] = (byte) ((data >> 24) & 0xff); bytes[4] = (byte) ((data >> 32) & 0xff); bytes[5] = (byte) ((data >> 40) & 0xff); bytes[6] = (byte) ((data >> 48) & 0xff); bytes[7] = (byte) ((data >> 56) & 0xff); return bytes; } public static byte[] getBytes(float data) { int intBits = Float.floatToIntBits(data); return getBytes(intBits); } public static byte[] getBytes(double data) { long intBits = Double.doubleToLongBits(data); return getBytes(intBits); } public static byte[] getBytes(String data, String charsetName) { Charset charset = Charset.forName(charsetName); return data.getBytes(charset); } public static byte[] getBytes(String data) { return getBytes(data, "GBK"); } public static short getShort(byte[] bytes) { return Short.parseShort(new String(bytes)); // return (short) ((0xff & bytes[0]) | (0xff00 & (bytes[1] << 8))); } public static char getChar(byte[] bytes) { return (char) ((0xff & bytes[0]) | (0xff00 & (bytes[1] << 8))); } public static int getInt(byte[] bytes) { return Integer.parseInt(new String(bytes)); // return (0xff & bytes[0]) | (0xff00 & (bytes[1] << 8)) | (0xff0000 & // (bytes[2] << 16)) | (0xff000000 & (bytes[3] << 24)); } public static long getLong(byte[] bytes) { return Long.parseLong(new String(bytes)); // return(0xffL & (long)bytes[0]) | (0xff00L & ((long)bytes[1] << 8)) | // (0xff0000L & ((long)bytes[2] << 16)) | (0xff000000L & ((long)bytes[3] // << 24)) // | (0xff00000000L & ((long)bytes[4] << 32)) | (0xff0000000000L & // ((long)bytes[5] << 40)) | (0xff000000000000L & ((long)bytes[6] << // 48)) | (0xff00000000000000L & ((long)bytes[7] << 56)); } public static double getDouble(byte[] bytes) { return Double.parseDouble(new String(bytes)); } public static float getFloat(byte[] bytes) { return Float.parseFloat(new String(bytes)); } public static String getString(byte[] bytes, String charsetName) { return new String(bytes, Charset.forName(charsetName)); } public static String getString(byte[] bytes) { return getString(bytes, "UTF-8"); } public static String getDate(byte[] bytes) { return new String(bytes); } public static String getTime(byte[] bytes) { return new String(bytes); } public static String getTimestmap(byte[] bytes) { return new String(bytes); } public static byte[] getBytes(Date date, boolean isTime) { if(isTime) { return getBytesFromTime(date); } else { return getBytesFromDate(date); } } /** * short从ASCII类型的值转换成数学二进制值。java里面不存在unsigned short，因此转换mysql unsiged值时需要用到java int. * @param bytes mysql里面的unsigned short字段，采用ASCII类型表达的值 * @return 数学二进制值 */ public static byte[] convertUnsignedShort2Binary(byte[] bytes) { int value = Integer.parseInt(new String(bytes)); byte[] binaryBytes = new byte[2]; binaryBytes[0] = (byte) (value & 0xff); binaryBytes[1] = (byte) ((value & 0xff00) >> 8); return binaryBytes; } private static byte[] getBytesFromTime(Date date) { int day = 0; int hour = DateUtil.getHour(date); int minute = DateUtil.getMinute(date); int second = DateUtil.getSecond(date); int microSecond = DateUtil.getMicroSecond(date); byte[] bytes = null; byte[] tmp = null; if(day == 0 && hour == 0 && minute == 0 && second == 0 && microSecond == 0) { bytes = new byte[1]; bytes[0] = (byte) 0; } else if(microSecond == 0) { bytes = new byte[1 + 8]; bytes[0] = (byte) 8; bytes[1] = (byte) 0; // is_negative (1) -- (1 if minus, 0 for plus) tmp = getBytes(day); bytes[2] = tmp[0]; bytes[3] = tmp[1]; bytes[4] = tmp[2]; bytes[5] = tmp[3]; bytes[6] = (byte) hour; bytes[7] = (byte) minute; bytes[8] = (byte) second; } else { bytes = new byte[1 + 12]; bytes[0] = (byte) 12; bytes[1] = (byte) 0; // is_negative (1) -- (1 if minus, 0 for plus) tmp = getBytes(day); bytes[2] = tmp[0]; bytes[3] = tmp[1]; bytes[4] = tmp[2]; bytes[5] = tmp[3]; bytes[6] = (byte) hour; bytes[7] = (byte) minute; bytes[8] = (byte) second; tmp = getBytes(microSecond); bytes[9] = tmp[0]; bytes[10] = tmp[1]; bytes[11] = tmp[2]; bytes[12] = tmp[3]; } return bytes; } private static byte[] getBytesFromDate(Date date) { int year = DateUtil.getYear(date); int month = DateUtil.getMonth(date); int day = DateUtil.getDay(date); int hour = DateUtil.getHour(date); int minute = DateUtil.getMinute(date); int second = DateUtil.getSecond(date); int microSecond = DateUtil.getMicroSecond(date); byte[] bytes = null; byte[] tmp = null; if(year == 0 && month == 0 && day == 0 && hour == 0 && minute == 0 && second == 0 && microSecond == 0) { bytes = new byte[1]; bytes[0] = (byte) 0; } else if(hour == 0 && minute == 0 && second == 0 && microSecond == 0) { bytes = new byte[1 + 4]; bytes[0] = (byte) 4; tmp = getBytes((short) year); bytes[1] = tmp[0]; bytes[2] = tmp[1]; bytes[3] = (byte) month; bytes[4] = (byte) day; } else if(microSecond == 0) { bytes = new byte[1 + 7]; bytes[0] = (byte) 7; tmp = getBytes((short) year); bytes[1] = tmp[0]; bytes[2] = tmp[1]; bytes[3] = (byte) month; bytes[4] = (byte) day; bytes[5] = (byte) hour; bytes[6] = (byte) minute; bytes[7] = (byte) second; } else { bytes = new byte[1 + 11]; bytes[0] = (byte) 11; tmp = getBytes((short) year); bytes[1] = tmp[0]; bytes[2] = tmp[1]; bytes[3] = (byte) month; bytes[4] = (byte) day; bytes[5] = (byte) hour; bytes[6] = (byte) minute; bytes[7] = (byte) second; tmp = getBytes(microSecond); bytes[8] = tmp[0]; bytes[9] = tmp[1]; bytes[10] = tmp[2]; bytes[11] = tmp[3]; } return bytes; } // 支持 byte dump //--------------------------------------------------------------------- public static String dump(byte[] data, int offset, int length) { StringBuilder sb = new StringBuilder(); sb.append(" byte dump log "); sb.append(System.lineSeparator()); sb.append(" offset ").append( offset ); sb.append(" length ").append( length ); sb.append(System.lineSeparator()); int lines = (length - 1) / 16 + 1; for (int i = 0, pos = 0; i < lines; i++, pos += 16) { sb.append(String.format("0x%04X ", i * 16)); for (int j = 0, pos1 = pos; j < 16; j++, pos1++) { sb.append(pos1 < length ? String.format("%02X ", data[offset + pos1]) : " "); } sb.append(" "); for (int j = 0, pos1 = pos; j < 16; j++, pos1++) { sb.append(pos1 < length ? print(data[offset + pos1]) : '.'); } sb.append(System.lineSeparator()); } sb.append(length).append(" bytes").append(System.lineSeparator()); return sb.toString(); } public static char print(byte b) { return (b < 32 || b > 127) ? '.' : (char) b; } /* * 返回小数点的位置 * @return 找不到 ,其他都是小数点的位置 * */ public static int getDot(byte[] array) { for(int s = 0; s < array.length; s++){ if(array[s] == 46) { return s; } } return array.length; } /* * @return 返回是否是科学计数法 * */ public static boolean hasE(byte[] array) { for(int s = 0; s < array.length; s++){ if(array[s] == 'E' || array[s] == 'e') { return true; } } return false; } /* * @比較 對於b1取0 到b1End進行比較 * 對於b2取0 到b2End進行比較 * */ public static int compareNumberByte(byte[] b1, int b1End, byte[] b2, int b2End) { if(b1 == null || b1.length == 0) { return -1; } else if(b2 == null || b2.length == 0) { return 1; } boolean isNegetive = b1[0] == 45 || b2[0] == 45; // 45 表示负数符号 //正数 长度不等 直接返回 长度 //都是正数长度不等,直接返回长度的比较 if (isNegetive == false && b1End != b2End) { return b1End - b2End; } //取短的一个 int len = b1End > b2End ? b2End : b1End; int result = 0; int index = -1; for (int i = 0; i < len; i++) { int b1val = b1[i]; int b2val = b2[i]; if (b1val > b2val) { result = 1; index = i; break; } else if (b1val < b2val) { index = i; result = -1; break; } } if (index == 0) { //正负数直接符号比较 // first byte compare ,数值与符号的比较 一正 一负数 return result; } else { if( b1End != b2End ) { //都是正数 长度不等, 都是负数 长度不等 int lenDelta = b1End - b2End; return isNegetive ? 0 - lenDelta : lenDelta; } else { //长度相等 符号相同 //位数相同 直接取比较结果的 正数就是结果,否则就是比较结果取反 return isNegetive ? 0 - result : result; } } } /* * double類型的b1 b2進行比較 * 首先:判斷是否是科學計數法 是直接創建Double對象進行比較 * 否則利用byte進行比較 * 先判斷整數位 再判斷小數位 * */ public static int compareDouble(byte[] b1, byte[] b2) { if(b1 == null || b1.length == 0) { return -1; } else if(b2 == null || b2.length == 0) { return 1; } boolean isHasE = hasE(b1) || hasE(b2); if(isHasE){ return Double.valueOf(new String(b1)).compareTo( Double.valueOf(new String(b2))) ; } int d1 = getDot(b1); int d2 = getDot(b2); //判斷整數位 int result = compareNumberByte(b1, d1, b2, d2); //符号相等 if(result == 0){ //判斷小數位 boolean isNegetive = b1[0] == 45 || b2[0] == 45; // 45 表示负数符号 int xsLen1 = b1.length - d1; int xsLen2 = b2.length - d2; int len = xsLen1 > xsLen2 ? xsLen2 : xsLen1; //小数位数中的 小数 int temp = 0; for(int i = 0; i < len ; i++) { temp = b1[i + d1] - b2 [i+ d2]; if(temp != 0){ break; } } if(temp == 0){ //0.12 0.123 或者 -0.12 -0.123 int lenDelta = xsLen1 - xsLen2; result = isNegetive? 0 - lenDelta : lenDelta; } else{ //0.12 0.113 或者 -0.12 -0.113 result = isNegetive? 0 - temp : temp; } } return result; } } ================================================ FILE: src/main/java/io/mycat/util/CircularArrayList.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import java.util.AbstractList; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.RandomAccess; public class CircularArrayList extends AbstractList implements RandomAccess { private final int n; // buffer length private final List buf; // a List implementing RandomAccess private int head = 0; private int tail = 0; public CircularArrayList(int capacity) { n = capacity + 1; buf = new ArrayList(Collections.nCopies(n, (E) null)); } public int capacity() { return n - 1; } private int wrapIndex(int i) { int m = i % n; if (m < 0) { // java modulus can be negative m += n; } return m; } // This method is O(n) but will never be called if the // CircularArrayList is used in its typical/intended role. private void shiftBlock(int startIndex, int endIndex) { assert (endIndex > startIndex); for (int i = endIndex - 1; i >= startIndex; i--) { set(i + 1, get(i)); } } @Override public int size() { return tail - head + (tail < head ? n : 0); } @Override public E get(int i) { if (i < 0 || i >= size()) { throw new IndexOutOfBoundsException(); } return buf.get(wrapIndex(head + i)); } @Override public E set(int i, E e) { if (i < 0 || i >= size()) { throw new IndexOutOfBoundsException(); } return buf.set(wrapIndex(head + i), e); } @Override public void add(int i, E e) { int s = size(); if (s == n - 1) { throw new IllegalStateException("Cannot add element." + " CircularArrayList is filled to capacity."); } if (i < 0 || i > s) { throw new IndexOutOfBoundsException(); } tail = wrapIndex(tail + 1); if (i < s) { shiftBlock(i, s); } set(i, e); } @Override public E remove(int i) { int s = size(); if (i < 0 || i >= s) { throw new IndexOutOfBoundsException(); } E e = get(i); if (i > 0) { shiftBlock(0, i); } head = wrapIndex(head + 1); return e; } } ================================================ FILE: src/main/java/io/mycat/util/CollectionUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.Map; import java.util.Set; /** * @author mycat */ public class CollectionUtil { /** * @param orig * if null, return intersect */ public static Set intersectSet(Set orig, Set intersect) { if (orig == null) { return intersect; } if (intersect == null || orig.isEmpty()) { return Collections.emptySet(); } Set set = new HashSet(orig.size()); for (Object p : orig) { if (intersect.contains(p)) { set.add(p); } } return set; } public static boolean isEmpty(Collection collection){ return collection==null || collection.isEmpty(); } public static boolean isEmpty(Map map){ return map==null || map.isEmpty(); } } ================================================ FILE: src/main/java/io/mycat/util/CompareUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class CompareUtil { private static final Logger LOGGER = LoggerFactory.getLogger(CompareUtil.class); public static int compareInt(int l,int r){ if(l > r){ return 1; }else if(l < r){ return -1; }else{ return 0; } } public static int compareDouble(double l,double r){ if(l > r){ return 1; }else if(l < r){ return -1; }else{ return 0; } } public static int compareFloat(float l,float r){ if(l > r){ return 1; }else if(l < r){ return -1; }else{ return 0; } } public static int compareLong(long l,long r){ // System.out.println(l + " " + r); if(l > r){ return 1; }else if(l < r){ return -1; }else{ return 0; } } public static int compareString(String l,String r){ // return compareStringForChinese(l,r); if(l == null) { return -1; } else if(r == null) { return 1; } return l.compareTo(r); } public static int compareChar(char l,char r){ if(l > r){ return 1; }else if(l < r){ return -1; }else{ return 0; } } public static int compareUtilDate(Object left,Object right){ java.util.Date l = (java.util.Date)left; java.util.Date r = (java.util.Date)right; return l.compareTo(r); } public static int compareSqlDate(Object left,Object right){ java.sql.Date l = (java.sql.Date)left; java.sql.Date r = (java.sql.Date)right; return l.compareTo(r); } private static int compareStringForChinese(String s1, String s2) { String m_s1 = null, m_s2 = null; try { //先将两字符串编码成GBK m_s1 = new String(s1.getBytes("GB2312"), "GBK"); m_s2 = new String(s2.getBytes("GB2312"), "GBK"); } catch (Exception ex) { LOGGER.error("compareStringForChineseError", ex); return s1.compareTo(s2); } int res = chineseCompareTo(m_s1, m_s2); // System.out.println("比较：" + s1 + " | " + s2 + "==== Result: " + res); return res; } //获取一个汉字/字母的Char值 private static int getCharCode(String s){ if (s==null || s.length()==0) { return -1;//保护代码 } byte [] b = s.getBytes(); int value = 0; //保证取第一个字符（汉字或者英文） for (int i = 0; i < b.length && i <= 2; i ++){ value = value*100+b[i]; } if(value < 0){ value += 100000; } return value; } //比较两个字符串 private static int chineseCompareTo(String s1, String s2){ int len1 = s1.length(); int len2 = s2.length(); int n = Math.min(len1, len2); for (int i = 0; i < n; i ++){ int s1_code = getCharCode(s1.charAt(i) + ""); int s2_code = getCharCode(s2.charAt(i) + ""); if (s1_code != s2_code){ return s1_code - s2_code; } } return len1 - len2; } } ================================================ FILE: src/main/java/io/mycat/util/CompressUtil.java ================================================ package io.mycat.util; import com.google.common.collect.Lists; import io.mycat.backend.mysql.BufferUtil; import io.mycat.backend.mysql.MySQLMessage; import io.mycat.net.AbstractConnection; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.zip.Deflater; import java.util.zip.Inflater; /** * 压缩数据包协议 * * http://dev.mysql.com/doc/internals/en/compressed-packet-header.html * * (包头) * 3 Bytes 压缩长度 * 1 Bytes 压缩序列号 * 3 Bytes 压缩前的长度 * * (包体) * n Bytes 压缩内容 或 未压缩内容 * * | -------------------------------------------------------------------------------------- | * | comp-length | seq-id | uncomp-len | Compressed Payload | * | ------------------------------------------------ ------------------------------------- | * | 22 00 00 | 00 | 32 00 00 | compress("\x2e\x00\x00\x00\x03select ...") | * | -------------------------------------------------------------------------------------- | * * Q:为什么消息体是 压缩内容 或者未压缩内容? * A:这是因为mysql内部有一个约定，如果查询语句payload小于50字节时， 对内容不压缩而保持原貌的方式，而mysql此举是为了减少CPU性能开销 * */ public class CompressUtil { public static final int MINI_LENGTH_TO_COMPRESS = 50; public static final int NO_COMPRESS_PACKET_LENGTH = MINI_LENGTH_TO_COMPRESS + 4; /** * 压缩数据包 * @param input * @param con * @param compressUnfinishedDataQueue * @return */ public static ByteBuffer compressMysqlPacket(ByteBuffer input, AbstractConnection con, ConcurrentLinkedQueue compressUnfinishedDataQueue) { byte[] byteArrayFromBuffer = getByteArrayFromBuffer(input); con.recycle(input); byteArrayFromBuffer = mergeBytes(byteArrayFromBuffer, compressUnfinishedDataQueue); return compressMysqlPacket(byteArrayFromBuffer, con, compressUnfinishedDataQueue); } /** * 压缩数据包 * @param data * @param con * @param compressUnfinishedDataQueue * @return */ private static ByteBuffer compressMysqlPacket(byte[] data, AbstractConnection con, ConcurrentLinkedQueue compressUnfinishedDataQueue) { ByteBuffer byteBuf = con.allocate(); byteBuf = con.checkWriteBuffer(byteBuf, data.length, false); //TODO: 数据量大的时候, 此处是是性能的堵点 MySQLMessage msg = new MySQLMessage(data); while ( msg.hasRemaining() ) { //包体的长度 int packetLength = 0; //可读的长度 int readLength = msg.length() - msg.position(); if ( readLength > 3 ) { packetLength = msg.readUB3(); msg.move(-3); } //校验数据包完整性 if ( readLength < packetLength + 4 ) { byte[] packet = msg.readBytes(readLength); if (packet.length != 0) { compressUnfinishedDataQueue.add(packet); //不完整的包 } } else { byte[] packet = msg.readBytes(packetLength + 4); if ( packet.length != 0 ) { if ( packet.length <= NO_COMPRESS_PACKET_LENGTH ) { BufferUtil.writeUB3(byteBuf, packet.length); //压缩长度 byteBuf.put(packet[3]); //压缩序号 BufferUtil.writeUB3(byteBuf, 0); //压缩前的长度设置为0 byteBuf.put(packet); //包体 } else { byte[] compress = compress(packet); //压缩 BufferUtil.writeUB3(byteBuf, compress.length); byteBuf.put(packet[3]); BufferUtil.writeUB3(byteBuf, packet.length); byteBuf.put(compress); } } } } return byteBuf; } /** * 解压数据包,同时做分包处理 * * @param data * @param decompressUnfinishedDataQueue * @return */ public static List decompressMysqlPacket(byte[] data, ConcurrentLinkedQueue decompressUnfinishedDataQueue) { MySQLMessage msg = new MySQLMessage(data); //包头 //----------------------------------------- int packetLength = msg.readUB3(); //压缩的包长 byte packetId = msg.read(); //压缩的包号 int oldLen = msg.readUB3(); //压缩前的长度 //未压缩, 直接返回 if ( packetLength == data.length - 4 ) { return Lists.newArrayList(data); //压缩不成功的, 直接返回 } else if (oldLen == 0) { byte[] readBytes = msg.readBytes(); return splitPack(readBytes, decompressUnfinishedDataQueue); //解压 } else { byte[] de = decompress(data, 7, data.length - 7); return splitPack(de, decompressUnfinishedDataQueue); } } /** * 分包处理 * * @param in * @param decompressUnfinishedDataQueue * @return */ private static List splitPack(byte[] in, ConcurrentLinkedQueue decompressUnfinishedDataQueue) { //合并 in = mergeBytes(in, decompressUnfinishedDataQueue); List smallPackList = new ArrayList<>(); MySQLMessage msg = new MySQLMessage(in); while ( msg.hasRemaining() ) { int readLength = msg.length() - msg.position(); int packetLength = 0; if (readLength > 3) { packetLength = msg.readUB3(); msg.move(-3); } if (readLength < packetLength + 4) { byte[] packet = msg.readBytes(readLength); if ( packet.length != 0 ) { decompressUnfinishedDataQueue.add(packet); } } else { byte[] packet = msg.readBytes(packetLength + 4); if ( packet.length != 0 ) { smallPackList.add(packet); } } } return smallPackList; } /** * 合并 解压未完成的字节 */ private static byte[] mergeBytes(byte[] in, ConcurrentLinkedQueue decompressUnfinishedDataQueue) { if ( !decompressUnfinishedDataQueue.isEmpty() ) { ByteArrayOutputStream out = new ByteArrayOutputStream(); try { while ( !decompressUnfinishedDataQueue.isEmpty() ) { out.write(decompressUnfinishedDataQueue.poll()); } out.write(in); in = out.toByteArray(); } catch (IOException e) { throw new RuntimeException(e); } finally { try { out.close(); } catch (IOException e) { } } } return in; } private static byte[] getByteArrayFromBuffer(ByteBuffer byteBuf) { byteBuf.flip(); byte[] row = new byte[byteBuf.limit()]; byteBuf.get(row); byteBuf.clear(); return row; } public static byte[] compress(ByteBuffer byteBuf) { return compress(getByteArrayFromBuffer(byteBuf)); } /** * 适用于mysql与客户端交互时zlib 压缩 * * @param data * @return */ public static byte[] compress(byte[] data) { byte[] output = null; Deflater compresser = new Deflater(); compresser.setInput(data); compresser.finish(); ByteArrayOutputStream out = new ByteArrayOutputStream(data.length); byte[] result = new byte[1024]; try { while (!compresser.finished()) { int length = compresser.deflate(result); out.write(result, 0, length); } output = out.toByteArray(); } finally { try { out.close(); } catch (Exception e) { } compresser.end(); } return output; } /** * 适用于mysql与客户端交互时zlib解压 * * @param data 数据 * @param off 偏移量 * @param len 长度 * @return */ public static byte[] decompress(byte[] data, int off, int len) { byte[] output = null; Inflater decompresser = new Inflater(); decompresser.reset(); decompresser.setInput(data, off, len); ByteArrayOutputStream out = new ByteArrayOutputStream(data.length); try { byte[] result = new byte[1024]; while (!decompresser.finished()) { int i = decompresser.inflate(result); out.write(result, 0, i); } output = out.toByteArray(); } catch (Exception e) { throw new RuntimeException(e); } finally { try { out.close(); } catch (Exception e) { } decompresser.end(); } return output; } } ================================================ FILE: src/main/java/io/mycat/util/DateUtil.java ================================================ package io.mycat.util; import java.text.ParseException; import java.util.Date; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; /** * 使用joda解析date,可以得到date的year,month,day等字段值 * @author CrazyPig * */ public class DateUtil { public static final String DEFAULT_DATE_PATTERN = "YYYY-MM-dd HH:mm:ss"; public static final String DATE_PATTERN_FULL = "YYYY-MM-dd HH:mm:ss.SSSSSS"; public static final String DATE_PATTERN_ONLY_DATE = "YYYY-MM-dd"; public static final String DEFAULT_TIME_PATTERN = "HHH:mm:ss"; public static final String TIME_PATTERN_FULL = "HHH:mm:ss.SSSSSS"; /** * 根据日期字符串解析得到date类型日期 * @param dateStr * @return * @throws ParseException */ public static Date parseDate(String dateStr) throws ParseException { return parseDate(dateStr, DEFAULT_DATE_PATTERN); } /** * 根据日期字符串和日期格式解析得到date类型日期 * @param dateStr * @param datePattern * @return * @throws ParseException */ public static Date parseDate(String dateStr, String datePattern) throws ParseException { DateTime dt = DateTimeFormat.forPattern(datePattern).parseDateTime(dateStr); return dt.toDate(); } /** * 获取date对象年份 * @param date * @return */ public static int getYear(Date date) { DateTime dt = new DateTime(date); return dt.getYear(); } /** * 获取date对象月份 * @param date * @return */ public static int getMonth(Date date) { DateTime dt = new DateTime(date); return dt.getMonthOfYear(); } /** * 获取date对象天数 * @param date * @return */ public static int getDay(Date date) { DateTime dt = new DateTime(date); return dt.getDayOfMonth(); } /** * 获取date对象小时数 * @param date * @return */ public static int getHour(Date date) { DateTime dt = new DateTime(date); return dt.getHourOfDay(); } /** * 获取date对象分钟数 * @param date * @return */ public static int getMinute(Date date) { DateTime dt = new DateTime(date); return dt.getMinuteOfHour(); } /** * 获取date对象秒数 * @param date * @return */ public static int getSecond(Date date) { DateTime dt = new DateTime(date); return dt.getSecondOfMinute(); } /** * 获取date对象毫秒数 * @param date * @return */ public static int getMicroSecond(Date date) { DateTime dt = new DateTime(date); return dt.getMillisOfSecond(); } } ================================================ FILE: src/main/java/io/mycat/util/DecryptUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import io.mycat.config.util.ConfigException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.crypto.Cipher; import java.security.*; import java.security.interfaces.RSAPrivateKey; import java.security.interfaces.RSAPublicKey; import java.security.spec.PKCS8EncodedKeySpec; import java.security.spec.RSAPrivateKeySpec; import java.security.spec.RSAPublicKeySpec; import java.security.spec.X509EncodedKeySpec; import java.util.Arrays; /** * @author songwie * */ public class DecryptUtil { private static final String DEFAULT_PRIVATE_KEY_STRING = "MIIBVAIBADANBgkqhkiG9w0BAQEFAASCAT4wggE6AgEAAkEAocbCrurZGbC5GArEHKlAfDSZi7gFBnd4yxOt0rwTqKBFzGyhtQLu5PRKjEiOXVa95aeIIBJ6OhC2f8FjqFUpawIDAQABAkAPejKaBYHrwUqUEEOe8lpnB6lBAsQIUFnQI/vXU4MV+MhIzW0BLVZCiarIQqUXeOhThVWXKFt8GxCykrrUsQ6BAiEA4vMVxEHBovz1di3aozzFvSMdsjTcYRRo82hS5Ru2/OECIQC2fAPoXixVTVY7bNMeuxCP4954ZkXp7fEPDINCjcQDywIgcc8XLkkPcs3Jxk7uYofaXaPbg39wuJpEmzPIxi3k0OECIGubmdpOnin3HuCP/bbjbJLNNoUdGiEmFL5hDI4UdwAdAiEAtcAwbm08bKN7pwwvyqaCBC//VnEWaq39DCzxr+Z2EIk="; public static final String DEFAULT_PUBLIC_KEY_STRING = "MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKHGwq7q2RmwuRgKxBypQHw0mYu4BQZ3eMsTrdK8E6igRcxsobUC7uT0SoxIjl1WveWniCASejoQtn/BY6hVKWsCAwEAAQ=="; public static void main(String[] args) throws Exception { System.out.println("其中 0:user:password是加密字符串,有两种格式\n" + "\n" + "dataHost加密格式\n" + "1:hostM1:root:123456\n" + "1代表是dataHost加密\n" + "hostM1是\n" + "\t\t\t\n" + "\t\t\n" + "\n" + "mycat用户登录密码加密格式\n" + "0:root:123456\n" + "0代表mycat用户登录密码加密\n" + "\t\n" + "\t\t1\n" + "\t\td6D+pOmkuUoY09p4/aivwMsScLa7zfjIwAxvkEhr3v7en06mEXoX9DTTjQNug5CfvGf7Wy9oLcthYI3yLMSjIg==\n" + "\t\tTESTDB"); String password = args[0]; System.out.println(encrypt(password)); } public static String mycatDecrypt(String usingDecrypt,String user ,String passwrod){ if("1".equals(usingDecrypt)||"true".equalsIgnoreCase(usingDecrypt)){ //type:user:password //0:test:test boolean flag = false; try { String passwrods[] = DecryptUtil.decrypt(passwrod).split(":"); if("0".equals(passwrods[0]) && user.equals(passwrods[1])){ flag = true; return passwrods[2]; } if(flag==false){ throw new ConfigException("user " + user + " passwrod need to decrype ,but decrype password is wrong !"); } } catch (Exception e2) { throw new ConfigException("user " + user + " passwrod need to decrype ,but decrype password is wrong !",e2); } } return passwrod; } public static String DBHostDecrypt(String usingDecrypt,String host,String user ,String passwrod){ if("1".equals(usingDecrypt)||"true".equalsIgnoreCase(usingDecrypt)){ //type:host:user:password //1:myhost1:test:test boolean flag = false; try { String passwrods[] = DecryptUtil.decrypt(passwrod).split(":"); if("1".equals(passwrods[0]) && host.equals(passwrods[1]) && user.equals(passwrods[2])){ return passwrods[3]; } if(flag==false){ throw new ConfigException("user " + user + " passwrod need to decrype ,but decrype password is wrong !"); } } catch (Exception e2) { throw new ConfigException("host " + host + ",user " + user + " passwrod need to decrype ,but decrype password is wrong !",e2); } } return passwrod; } public static String decrypt(String cipherText) throws Exception { return decrypt((String) null, cipherText); } public static String decrypt(String publicKeyText, String cipherText) throws Exception { PublicKey publicKey = getPublicKey(publicKeyText); return decrypt(publicKey, cipherText); } public static PublicKey getPublicKey(String publicKeyText) { if (publicKeyText == null || publicKeyText.length() == 0) { publicKeyText = DecryptUtil.DEFAULT_PUBLIC_KEY_STRING; } try { byte[] publicKeyBytes = Base64.base64ToByteArray(publicKeyText); X509EncodedKeySpec x509KeySpec = new X509EncodedKeySpec( publicKeyBytes); KeyFactory keyFactory = KeyFactory.getInstance("RSA"); return keyFactory.generatePublic(x509KeySpec); } catch (Exception e) { throw new IllegalArgumentException("Failed to get public key", e); } } public static String decrypt(PublicKey publicKey, String cipherText) throws Exception { Cipher cipher = Cipher.getInstance("RSA"); try { cipher.init(Cipher.DECRYPT_MODE, publicKey); } catch (InvalidKeyException e) { // 因为 IBM JDK 不支持私钥加密, 公钥解密, 所以要反转公私钥 // 也就是说对于解密, 可以通过公钥的参数伪造一个私钥对象欺骗 IBM JDK RSAPublicKey rsaPublicKey = (RSAPublicKey) publicKey; RSAPrivateKeySpec spec = new RSAPrivateKeySpec(rsaPublicKey.getModulus(), rsaPublicKey.getPublicExponent()); Key fakePrivateKey = KeyFactory.getInstance("RSA").generatePrivate(spec); cipher = Cipher.getInstance("RSA"); //It is a stateful object. so we need to get new one. cipher.init(Cipher.DECRYPT_MODE, fakePrivateKey); } if (cipherText == null || cipherText.length() == 0) { return cipherText; } byte[] cipherBytes = Base64.base64ToByteArray(cipherText); byte[] plainBytes = cipher.doFinal(cipherBytes); return new String(plainBytes); } public static String encrypt(String plainText) throws Exception { return encrypt((String) null, plainText); } public static String encrypt(String key, String plainText) throws Exception { if (key == null) { key = DEFAULT_PRIVATE_KEY_STRING; } byte[] keyBytes = Base64.base64ToByteArray(key); return encrypt(keyBytes, plainText); } public static String encrypt(byte[] keyBytes, String plainText) throws Exception { PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(keyBytes); KeyFactory factory = KeyFactory.getInstance("RSA"); PrivateKey privateKey = factory.generatePrivate(spec); Cipher cipher = Cipher.getInstance("RSA"); try { cipher.init(Cipher.ENCRYPT_MODE, privateKey); } catch (InvalidKeyException e) { //For IBM JDK, 原因请看解密方法中的说明 RSAPrivateKey rsaPrivateKey = (RSAPrivateKey) privateKey; RSAPublicKeySpec publicKeySpec = new RSAPublicKeySpec(rsaPrivateKey.getModulus(), rsaPrivateKey.getPrivateExponent()); Key fakePublicKey = KeyFactory.getInstance("RSA").generatePublic(publicKeySpec); cipher = Cipher.getInstance("RSA"); cipher.init(Cipher.ENCRYPT_MODE, fakePublicKey); } byte[] encryptedBytes = cipher.doFinal(plainText.getBytes("UTF-8")); String encryptedString = Base64.byteArrayToBase64(encryptedBytes); return encryptedString; } public static byte[][] genKeyPairBytes(int keySize) throws NoSuchAlgorithmException { byte[][] keyPairBytes = new byte[2][]; KeyPairGenerator gen = KeyPairGenerator.getInstance("RSA"); gen.initialize(keySize, new SecureRandom()); KeyPair pair = gen.generateKeyPair(); keyPairBytes[0] = pair.getPrivate().getEncoded(); keyPairBytes[1] = pair.getPublic().getEncoded(); return keyPairBytes; } public static String[] genKeyPair(int keySize) throws NoSuchAlgorithmException { byte[][] keyPairBytes = genKeyPairBytes(keySize); String[] keyPairs = new String[2]; keyPairs[0] = Base64.byteArrayToBase64(keyPairBytes[0]); keyPairs[1] = Base64.byteArrayToBase64(keyPairBytes[1]); return keyPairs; } static class Base64 { /** * Translates the specified byte array into a Base64 string as per Preferences.put(byte[]). */ public static String byteArrayToBase64(byte[] a) { return byteArrayToBase64(a, false); } /** * Translates the specified byte array into an "alternate representation" Base64 string. This non-standard variant * uses an alphabet that does not contain the uppercase alphabetic characters, which makes it suitable for use in * situations where case-folding occurs. */ public static String byteArrayToAltBase64(byte[] a) { return byteArrayToBase64(a, true); } private static String byteArrayToBase64(byte[] a, boolean alternate) { int aLen = a.length; int numFullGroups = aLen / 3; int numBytesInPartialGroup = aLen - 3 * numFullGroups; int resultLen = 4 * ((aLen + 2) / 3); StringBuilder result = new StringBuilder(resultLen); char[] intToAlpha = (alternate ? intToAltBase64 : intToBase64); // Translate all full groups from byte array elements to Base64 int inCursor = 0; for (int i = 0; i < numFullGroups; i++) { int byte0 = a[inCursor++] & 0xff; int byte1 = a[inCursor++] & 0xff; int byte2 = a[inCursor++] & 0xff; result.append(intToAlpha[byte0 >> 2]); result.append(intToAlpha[(byte0 << 4) & 0x3f | (byte1 >> 4)]); result.append(intToAlpha[(byte1 << 2) & 0x3f | (byte2 >> 6)]); result.append(intToAlpha[byte2 & 0x3f]); } // Translate partial group if present if (numBytesInPartialGroup != 0) { int byte0 = a[inCursor++] & 0xff; result.append(intToAlpha[byte0 >> 2]); if (numBytesInPartialGroup == 1) { result.append(intToAlpha[(byte0 << 4) & 0x3f]); result.append("=="); } else { // assert numBytesInPartialGroup == 2; int byte1 = a[inCursor++] & 0xff; result.append(intToAlpha[(byte0 << 4) & 0x3f | (byte1 >> 4)]); result.append(intToAlpha[(byte1 << 2) & 0x3f]); result.append('='); } } // assert inCursor == a.length; // assert result.length() == resultLen; return result.toString(); } /** * This array is a lookup table that translates 6-bit positive integer index values into their "Base64 Alphabet" * equivalents as specified in Table 1 of RFC 2045. */ private static final char intToBase64[] = { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/' }; /** * This array is a lookup table that translates 6-bit positive integer index values into their * "Alternate Base64 Alphabet" equivalents. This is NOT the real Base64 Alphabet as per in Table 1 of RFC 2045. This * alternate alphabet does not use the capital letters. It is designed for use in environments where "case folding" * occurs. */ private static final char intToAltBase64[] = { '!', '"', '#', '$', '%', '&', '\'', '(', ')', ',', '-', '.', ':', ';', '<', '>', '@', '[', ']', '^', '`', '_', '{', '|', '}', '~', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '?' }; /** * Translates the specified Base64 string (as per Preferences.get(byte[])) into a byte array. * * @throw IllegalArgumentException if s is not a valid Base64 string. */ public static byte[] base64ToByteArray(String s) { return base64ToByteArray(s, false); } /** * Translates the specified "alternate representation" Base64 string into a byte array. * * @throw IllegalArgumentException or ArrayOutOfBoundsException if s is not a valid alternate * representation Base64 string. */ public static byte[] altBase64ToByteArray(String s) { return base64ToByteArray(s, true); } private static byte[] base64ToByteArray(String s, boolean alternate) { byte[] alphaToInt = (alternate ? altBase64ToInt : base64ToInt); int sLen = s.length(); int numGroups = sLen / 4; if (4 * numGroups != sLen) { throw new IllegalArgumentException("String length must be a multiple of four."); } int missingBytesInLastGroup = 0; int numFullGroups = numGroups; if (sLen != 0) { if (s.charAt(sLen - 1) == '=') { missingBytesInLastGroup++; numFullGroups--; } if (s.charAt(sLen - 2) == '=') { missingBytesInLastGroup++; } } byte[] result = new byte[3 * numGroups - missingBytesInLastGroup]; // Translate all full groups from base64 to byte array elements int inCursor = 0, outCursor = 0; for (int i = 0; i < numFullGroups; i++) { int ch0 = base64toInt(s.charAt(inCursor++), alphaToInt); int ch1 = base64toInt(s.charAt(inCursor++), alphaToInt); int ch2 = base64toInt(s.charAt(inCursor++), alphaToInt); int ch3 = base64toInt(s.charAt(inCursor++), alphaToInt); result[outCursor++] = (byte) ((ch0 << 2) | (ch1 >> 4)); result[outCursor++] = (byte) ((ch1 << 4) | (ch2 >> 2)); result[outCursor++] = (byte) ((ch2 << 6) | ch3); } // Translate partial group, if present if (missingBytesInLastGroup != 0) { int ch0 = base64toInt(s.charAt(inCursor++), alphaToInt); int ch1 = base64toInt(s.charAt(inCursor++), alphaToInt); result[outCursor++] = (byte) ((ch0 << 2) | (ch1 >> 4)); if (missingBytesInLastGroup == 1) { int ch2 = base64toInt(s.charAt(inCursor++), alphaToInt); result[outCursor++] = (byte) ((ch1 << 4) | (ch2 >> 2)); } } // assert inCursor == s.length()-missingBytesInLastGroup; // assert outCursor == result.length; return result; } /** * Translates the specified character, which is assumed to be in the "Base 64 Alphabet" into its equivalent 6-bit * positive integer. * * @throw IllegalArgumentException or ArrayOutOfBoundsException if c is not in the Base64 Alphabet. */ private static int base64toInt(char c, byte[] alphaToInt) { int result = alphaToInt[c]; if (result < 0) { throw new IllegalArgumentException("Illegal character " + c); } return result; } /** * This array is a lookup table that translates unicode characters drawn from the "Base64 Alphabet" (as specified in * Table 1 of RFC 2045) into their 6-bit positive integer equivalents. Characters that are not in the Base64 * alphabet but fall within the bounds of the array are translated to -1. */ private static final byte base64ToInt[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 }; /** * This array is the analogue of base64ToInt, but for the nonstandard variant that avoids the use of uppercase * alphabetic characters. */ private static final byte altBase64ToInt[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, -1, 62, 9, 10, 11, -1, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 12, 13, 14, -1, 15, 63, 16, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 17, -1, 18, 19, 21, 20, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 22, 23, 24, 25 }; } } ================================================ FILE: src/main/java/io/mycat/util/ExecutorUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import java.util.concurrent.LinkedTransferQueue; /** * @author mycat */ public class ExecutorUtil { public static final NameableExecutor create(String name, int size) { return create(name, size, true); } private static final NameableExecutor create(String name, int size, boolean isDaemon) { NameableThreadFactory factory = new NameableThreadFactory(name, isDaemon); return new NameableExecutor(name, size, new LinkedTransferQueue(), factory); } } ================================================ FILE: src/main/java/io/mycat/util/FastByteOperations.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.util; import java.lang.reflect.Field; import java.nio.Buffer; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.security.AccessController; import java.security.PrivilegedAction; import sun.misc.Unsafe; import com.google.common.primitives.Longs; import com.google.common.primitives.UnsignedBytes; import com.google.common.primitives.UnsignedLongs; /** * Utility code to do optimized byte-array comparison. * This is borrowed and slightly modified from Guava's {@link UnsignedBytes} * class to be able to compare arrays that start at non-zero offsets. */ public class FastByteOperations { /** * Lexicographically compare two byte arrays. */ public static int compareUnsigned(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { return BestHolder.BEST.compare(b1, s1, l1, b2, s2, l2); } public static int compareUnsigned(ByteBuffer b1, byte[] b2, int s2, int l2) { return BestHolder.BEST.compare(b1, b2, s2, l2); } public static int compareUnsigned(byte[] b1, int s1, int l1, ByteBuffer b2) { return -BestHolder.BEST.compare(b2, b1, s1, l1); } public static int compareUnsigned(ByteBuffer b1, ByteBuffer b2) { return BestHolder.BEST.compare(b1, b2); } public static void copy(ByteBuffer src, int srcPosition, byte[] trg, int trgPosition, int length) { BestHolder.BEST.copy(src, srcPosition, trg, trgPosition, length); } public static void copy(ByteBuffer src, int srcPosition, ByteBuffer trg, int trgPosition, int length) { BestHolder.BEST.copy(src, srcPosition, trg, trgPosition, length); } public interface ByteOperations { abstract public int compare(byte[] buffer1, int offset1, int length1, byte[] buffer2, int offset2, int length2); abstract public int compare(ByteBuffer buffer1, byte[] buffer2, int offset2, int length2); abstract public int compare(ByteBuffer buffer1, ByteBuffer buffer2); abstract public void copy(ByteBuffer src, int srcPosition, byte[] trg, int trgPosition, int length); abstract public void copy(ByteBuffer src, int srcPosition, ByteBuffer trg, int trgPosition, int length); } /** * Provides a lexicographical comparer implementation; either a Java * implementation or a faster implementation based on {@link Unsafe}. *

*

Uses reflection to gracefully fall back to the Java implementation if * {@code Unsafe} isn't available. */ private static class BestHolder { static final String UNSAFE_COMPARER_NAME = FastByteOperations.class.getName() + "$UnsafeOperations"; static final ByteOperations BEST = getBest(); /** * Returns the Unsafe-using Comparer, or falls back to the pure-Java * implementation if unable to do so. */ static ByteOperations getBest() { String arch = System.getProperty("os.arch"); boolean unaligned = arch.equals("i386") || arch.equals("x86") || arch.equals("amd64") || arch.equals("x86_64"); if (!unaligned) { return new PureJavaOperations(); } try { Class theClass = Class.forName(UNSAFE_COMPARER_NAME); // yes, UnsafeComparer does implement Comparer @SuppressWarnings("unchecked") ByteOperations comparer = (ByteOperations) theClass.getConstructor().newInstance(); return comparer; } catch (Throwable t) { //JVMStabilityInspector.inspectThrowable(t); // ensure we really catch *everything* return new PureJavaOperations(); } } } @SuppressWarnings("unused") // used via reflection public static final class UnsafeOperations implements ByteOperations { static final Unsafe theUnsafe; /** * The offset to the first element in a byte array. */ static final long BYTE_ARRAY_BASE_OFFSET; static final long DIRECT_BUFFER_ADDRESS_OFFSET; static { theUnsafe = (Unsafe) AccessController.doPrivileged( new PrivilegedAction() { @Override public Object run() { try { Field f = Unsafe.class.getDeclaredField("theUnsafe"); f.setAccessible(true); return f.get(null); } catch (NoSuchFieldException e) { // It doesn't matter what we throw; // it's swallowed in getBest(). throw new Error(); } catch (IllegalAccessException e) { throw new Error(); } } }); try { BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class); DIRECT_BUFFER_ADDRESS_OFFSET = theUnsafe.objectFieldOffset(Buffer.class.getDeclaredField("address")); } catch (Exception e) { throw new AssertionError(e); } // sanity check - this should never fail if (theUnsafe.arrayIndexScale(byte[].class) != 1) { throw new AssertionError(); } } static final boolean BIG_ENDIAN = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); public int compare(byte[] buffer1, int offset1, int length1, byte[] buffer2, int offset2, int length2) { return compareTo(buffer1, BYTE_ARRAY_BASE_OFFSET + offset1, length1, buffer2, BYTE_ARRAY_BASE_OFFSET + offset2, length2); } public int compare(ByteBuffer buffer1, byte[] buffer2, int offset2, int length2) { Object obj1; long offset1; if (buffer1.hasArray()) { obj1 = buffer1.array(); offset1 = BYTE_ARRAY_BASE_OFFSET + buffer1.arrayOffset(); } else { obj1 = null; offset1 = theUnsafe.getLong(buffer1, DIRECT_BUFFER_ADDRESS_OFFSET); } int length1; { int position = buffer1.position(); int limit = buffer1.limit(); length1 = limit - position; offset1 += position; } return compareTo(obj1, offset1, length1, buffer2, BYTE_ARRAY_BASE_OFFSET + offset2, length2); } public int compare(ByteBuffer buffer1, ByteBuffer buffer2) { return compareTo(buffer1, buffer2); } public void copy(ByteBuffer src, int srcPosition, byte[] trg, int trgPosition, int length) { if (src.hasArray()) { System.arraycopy(src.array(), src.arrayOffset() + srcPosition, trg, trgPosition, length); } else { copy(null, srcPosition + theUnsafe.getLong(src, DIRECT_BUFFER_ADDRESS_OFFSET), trg, trgPosition, length); } } public void copy(ByteBuffer srcBuf, int srcPosition, ByteBuffer trgBuf, int trgPosition, int length) { Object src; long srcOffset; if (srcBuf.hasArray()) { src = srcBuf.array(); srcOffset = BYTE_ARRAY_BASE_OFFSET + srcBuf.arrayOffset(); } else { src = null; srcOffset = theUnsafe.getLong(srcBuf, DIRECT_BUFFER_ADDRESS_OFFSET); } copy(src, srcOffset + srcPosition, trgBuf, trgPosition, length); } public static void copy(Object src, long srcOffset, ByteBuffer trgBuf, int trgPosition, int length) { if (trgBuf.hasArray()) { copy(src, srcOffset, trgBuf.array(), trgBuf.arrayOffset() + trgPosition, length); } else { copy(src, srcOffset, null, trgPosition + theUnsafe.getLong(trgBuf, DIRECT_BUFFER_ADDRESS_OFFSET), length); } } public static void copy(Object src, long srcOffset, byte[] trg, int trgPosition, int length) { if (length <= MIN_COPY_THRESHOLD) { for (int i = 0 ; i < length ; i++) { trg[trgPosition + i] = theUnsafe.getByte(src, srcOffset + i); } } else { copy(src, srcOffset, trg, BYTE_ARRAY_BASE_OFFSET + trgPosition, length); } } // 1M, copied from java.nio.Bits (unfortunately a package-private class) private static final long UNSAFE_COPY_THRESHOLD = 1 << 20; private static final long MIN_COPY_THRESHOLD = 6; public static void copy(Object src, long srcOffset, Object dst, long dstOffset, long length) { while (length > 0) { long size = (length > UNSAFE_COPY_THRESHOLD) ? UNSAFE_COPY_THRESHOLD : length; // if src or dst are null, the offsets are absolute base addresses: theUnsafe.copyMemory(src, srcOffset, dst, dstOffset, size); length -= size; srcOffset += size; dstOffset += size; } } public static int compareTo(ByteBuffer buffer1, ByteBuffer buffer2) { Object obj1; long offset1; int length1; if (buffer1.hasArray()) { obj1 = buffer1.array(); offset1 = BYTE_ARRAY_BASE_OFFSET + buffer1.arrayOffset(); } else { obj1 = null; offset1 = theUnsafe.getLong(buffer1, DIRECT_BUFFER_ADDRESS_OFFSET); } offset1 += buffer1.position(); length1 = buffer1.remaining(); return compareTo(obj1, offset1, length1, buffer2); } public static int compareTo(Object buffer1, long offset1, int length1, ByteBuffer buffer) { Object obj2; long offset2; int position = buffer.position(); int limit = buffer.limit(); if (buffer.hasArray()) { obj2 = buffer.array(); offset2 = BYTE_ARRAY_BASE_OFFSET + buffer.arrayOffset(); } else { obj2 = null; offset2 = theUnsafe.getLong(buffer, DIRECT_BUFFER_ADDRESS_OFFSET); } int length2 = limit - position; offset2 += position; return compareTo(buffer1, offset1, length1, obj2, offset2, length2); } /** * Lexicographically compare two arrays. * * @param buffer1 left operand: a byte[] or null * @param buffer2 right operand: a byte[] or null * @param memoryOffset1 Where to start comparing in the left buffer (pure memory address if buffer1 is null, or relative otherwise) * @param memoryOffset2 Where to start comparing in the right buffer (pure memory address if buffer1 is null, or relative otherwise) * @param length1 How much to compare from the left buffer * @param length2 How much to compare from the right buffer * @return 0 if equal, < 0 if left is less than right, etc. */ public static int compareTo(Object buffer1, long memoryOffset1, int length1, Object buffer2, long memoryOffset2, int length2) { int minLength = Math.min(length1, length2); /* * Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a * time is no slower than comparing 4 bytes at a time even on 32-bit. * On the other hand, it is substantially faster on 64-bit. */ int wordComparisons = minLength & ~7; for (int i = 0; i < wordComparisons ; i += Longs.BYTES) { long lw = theUnsafe.getLong(buffer1, memoryOffset1 + (long) i); long rw = theUnsafe.getLong(buffer2, memoryOffset2 + (long) i); if (lw != rw) { if (BIG_ENDIAN) { return UnsignedLongs.compare(lw, rw); } return UnsignedLongs.compare(Long.reverseBytes(lw), Long.reverseBytes(rw)); } } for (int i = wordComparisons ; i < minLength ; i++) { int b1 = theUnsafe.getByte(buffer1, memoryOffset1 + i) & 0xFF; int b2 = theUnsafe.getByte(buffer2, memoryOffset2 + i) & 0xFF; if (b1 != b2) { return b1 - b2; } } return length1 - length2; } } @SuppressWarnings("unused") public static final class PureJavaOperations implements ByteOperations { @Override public int compare(byte[] buffer1, int offset1, int length1, byte[] buffer2, int offset2, int length2) { // Short circuit equal case if (buffer1 == buffer2 && offset1 == offset2 && length1 == length2) { return 0; } int end1 = offset1 + length1; int end2 = offset2 + length2; for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) { int a = (buffer1[i] & 0xff); int b = (buffer2[j] & 0xff); if (a != b) { return a - b; } } return length1 - length2; } public int compare(ByteBuffer buffer1, byte[] buffer2, int offset2, int length2) { if (buffer1.hasArray()) { return compare(buffer1.array(), buffer1.arrayOffset() + buffer1.position(), buffer1.remaining(), buffer2, offset2, length2); } return compare(buffer1, ByteBuffer.wrap(buffer2, offset2, length2)); } public int compare(ByteBuffer buffer1, ByteBuffer buffer2) { int end1 = buffer1.limit(); int end2 = buffer2.limit(); for (int i = buffer1.position(), j = buffer2.position(); i < end1 && j < end2; i++, j++) { int a = (buffer1.get(i) & 0xff); int b = (buffer2.get(j) & 0xff); if (a != b) { return a - b; } } return buffer1.remaining() - buffer2.remaining(); } public void copy(ByteBuffer src, int srcPosition, byte[] trg, int trgPosition, int length) { if (src.hasArray()) { System.arraycopy(src.array(), src.arrayOffset() + srcPosition, trg, trgPosition, length); return; } src = src.duplicate(); src.position(srcPosition); src.get(trg, trgPosition, length); } public void copy(ByteBuffer src, int srcPosition, ByteBuffer trg, int trgPosition, int length) { if (src.hasArray() && trg.hasArray()) { System.arraycopy(src.array(), src.arrayOffset() + srcPosition, trg.array(), trg.arrayOffset() + trgPosition, length); return; } src = src.duplicate(); src.position(srcPosition).limit(srcPosition + length); trg = trg.duplicate(); trg.position(trgPosition); trg.put(src); } } } ================================================ FILE: src/main/java/io/mycat/util/FormatUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; /** * 格式化工具 * * @author mycat * @version 2008-11-24 下午12:58:17 */ public final class FormatUtil { // 右对齐格式化字符串 public static final int ALIGN_RIGHT = 0; // 左对齐格式化字符串 public static final int ALIGN_LEFT = 1; private static final char defaultSplitChar = ' '; private static final String[] timeFormat = new String[] { "d ", "h ", "m ", "s ", "ms" }; /** * 格式化后返回的字符串 * * @param s * 需要格式化的原始字符串，默认按左对齐。 * @param fillLength * 填充长度 * @return String */ public static String format(String s, int fillLength) { return format(s, fillLength, defaultSplitChar, ALIGN_LEFT); } /** * 格式化后返回的字符串 * * @param i * 需要格式化的数字类型，默认按右对齐。 * @param fillLength * 填充长度 * @return String */ public static String format(int i, int fillLength) { return format(Integer.toString(i), fillLength, defaultSplitChar, ALIGN_RIGHT); } /** * 格式化后返回的字符串 * * @param l * 需要格式化的数字类型，默认按右对齐。 * @param fillLength * 填充长度 * @return String */ public static String format(long l, int fillLength) { return format(Long.toString(l), fillLength, defaultSplitChar, ALIGN_RIGHT); } /** * @param s * 需要格式化的原始字符串 * @param fillLength * 填充长度 * @param fillChar * 填充的字符 * @param align * 填充方式（左边填充还是右边填充） * @return String */ public static String format(String s, int fillLength, char fillChar, int align) { if (s == null) { s = ""; } else { s = s.trim(); } int charLen = fillLength - s.length(); if (charLen > 0) { char[] fills = new char[charLen]; for (int i = 0; i < charLen; i++) { fills[i] = fillChar; } StringBuilder str = new StringBuilder(s); switch (align) { case ALIGN_RIGHT: str.insert(0, fills); break; case ALIGN_LEFT: str.append(fills); break; default: str.append(fills); } return str.toString(); } else { return s; } } /** * 格式化时间输出 *

* 1d 15h 4m 15s 987ms *

*/ public static String formatTime(long millis, int precision) { long[] la = new long[5]; la[0] = (millis / 86400000);// days la[1] = (millis / 3600000) % 24;// hours la[2] = (millis / 60000) % 60;// minutes la[3] = (millis / 1000) % 60;// seconds la[4] = (millis % 1000);// ms int index = 0; for (int i = 0; i < la.length; i++) { if (la[i] != 0) { index = i; break; } } StringBuilder buf = new StringBuilder(); int validLength = la.length - index; for (int i = 0; (i < validLength && i < precision); i++) { buf.append(la[index]).append(timeFormat[index]); index++; } return buf.toString(); } } ================================================ FILE: src/main/java/io/mycat/util/HexFormatUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; /** * @author mycat */ public final class HexFormatUtil { private final static char[] hexArray = "0123456789ABCDEF".toCharArray(); public static String bytesToHexString(byte[] bytes) { char[] hexChars = new char[bytes.length * 2]; for ( int j = 0; j < bytes.length; j++ ) { int v = bytes[j] & 0xFF; hexChars[j * 2] = hexArray[v >>> 4]; hexChars[j * 2 + 1] = hexArray[v & 0x0F]; } return new String(hexChars); } public static byte[] fromHex(String src) { String[] hex = src.split(" "); byte[] b = new byte[hex.length]; for (int i = 0; i < hex.length; i++) { b[i] = (byte) (Integer.parseInt(hex[i], 16) & 0xff); } return b; } public static String fromHex(String hexString, String charset) { try { byte[] b = fromHex(hexString); if (charset == null) { return new String(b); } return new String(b, charset); } catch (Exception e) { return null; } } public static int fromHex2B(String src) { byte[] b = fromHex(src); int position = 0; int i = (b[position++] & 0xff); i |= (b[position++] & 0xff) << 8; return i; } public static int fromHex4B(String src) { byte[] b = fromHex(src); int position = 0; int i = (b[position++] & 0xff); i |= (b[position++] & 0xff) << 8; i |= (b[position++] & 0xff) << 16; i |= (b[position++] & 0xff) << 24; return i; } public static long fromHex8B(String src) { byte[] b = fromHex(src); int position = 0; long l = (b[position++] & 0xff); l |= (long) (b[position++] & 0xff) << 8; l |= (long) (b[position++] & 0xff) << 16; l |= (long) (b[position++] & 0xff) << 24; l |= (long) (b[position++] & 0xff) << 32; l |= (long) (b[position++] & 0xff) << 40; l |= (long) (b[position++] & 0xff) << 48; l |= (long) (b[position++] & 0xff) << 56; return l; } } ================================================ FILE: src/main/java/io/mycat/util/IntegerUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; /** * @author mycat */ public final class IntegerUtil { static final byte[] minValue = "-2147483648".getBytes(); static final int[] sizeTable = { 9, 99, 999, 9999, 99999, 999999, 9999999, 99999999, 999999999, Integer.MAX_VALUE }; static final byte[] digitTens = { '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '2', '2', '2', '2', '2', '2', '2', '2', '2', '2', '3', '3', '3', '3', '3', '3', '3', '3', '3', '3', '4', '4', '4', '4', '4', '4', '4', '4', '4', '4', '5', '5', '5', '5', '5', '5', '5', '5', '5', '5', '6', '6', '6', '6', '6', '6', '6', '6', '6', '6', '7', '7', '7', '7', '7', '7', '7', '7', '7', '7', '8', '8', '8', '8', '8', '8', '8', '8', '8', '8', '9', '9', '9', '9', '9', '9', '9', '9', '9', '9', }; static final byte[] digitOnes = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', }; static final byte[] digits = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z' }; public static byte[] toBytes(int i) { if (i == Integer.MIN_VALUE) { return minValue; } int size = (i < 0) ? stringSize(-i) + 1 : stringSize(i); byte[] buf = new byte[size]; getBytes(i, size, buf); return buf; } static int stringSize(int x) { for (int i = 0;; i++) { if (x <= sizeTable[i]) { return i + 1; } } } static void getBytes(int i, int index, byte[] buf) { int q, r; int charPos = index; byte sign = 0; if (i < 0) { sign = '-'; i = -i; } // Generate two digits per iteration while (i >= 65536) { q = i / 100; // really: r = i - (q * 100); r = i - ((q << 6) + (q << 5) + (q << 2)); i = q; buf[--charPos] = digitOnes[r]; buf[--charPos] = digitTens[r]; } // Fall thru to fast mode for smaller numbers // assert(i <= 65536, i); for (;;) { q = (i * 52429) >>> (16 + 3); r = i - ((q << 3) + (q << 1)); // r = i-(q*10) ... buf[--charPos] = digits[r]; i = q; if (i == 0) { break; } } if (sign != 0) { buf[--charPos] = sign; } } } ================================================ FILE: src/main/java/io/mycat/util/LogUtil.java ================================================ package io.mycat.util; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.util.Locale; import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.config.model.SystemConfig; public class LogUtil { public static final Logger LOGGER = LoggerFactory.getLogger(LogUtil.class); /** * 將消息寫入到logs\switch.log中 * * @param * @param msg */ public synchronized static void writeDataSourceLog(String msg) { File file = new File(SystemConfig.getHomePath(), "logs" + File.separator + "switch.log"); FileOutputStream fileOut = null; try { File parent = file.getParentFile(); if (parent != null && !parent.exists()) { parent.mkdirs(); } fileOut = new FileOutputStream(file, true); long time = TimeUtil.currentTimeMillis(); DateTime dt2 = new DateTime(time); String m = String.format("%s: %s\r\n",dt2.toString(DateUtil.DEFAULT_DATE_PATTERN,Locale.CHINESE) , msg); fileOut.write(m.getBytes());; } catch (Exception e) { LOGGER.warn("write dataHost log err:", e); } finally { if (fileOut != null) { try { fileOut.close(); } catch (IOException e) { } } } } } ================================================ FILE: src/main/java/io/mycat/util/LongUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; /** * @author mycat */ public final class LongUtil { private static final byte[] minValue = "-9223372036854775808".getBytes(); public static byte[] toBytes(long i) { if (i == Long.MIN_VALUE) { return minValue; } int size = (i < 0) ? stringSize(-i) + 1 : stringSize(i); byte[] buf = new byte[size]; getBytes(i, size, buf); return buf; } static int stringSize(long x) { long p = 10; for (int i = 1; i < 19; i++) { if (x < p) { return i; } p = 10 * p; } return 19; } static void getBytes(long i, int index, byte[] buf) { long q; int r; int charPos = index; byte sign = 0; if (i < 0) { sign = '-'; i = -i; } // Get 2 digits/iteration using longs until quotient fits into an int while (i > Integer.MAX_VALUE) { q = i / 100; // really: r = i - (q * 100); r = (int) (i - ((q << 6) + (q << 5) + (q << 2))); i = q; buf[--charPos] = IntegerUtil.digitOnes[r]; buf[--charPos] = IntegerUtil.digitTens[r]; } // Get 2 digits/iteration using ints int q2; int i2 = (int) i; while (i2 >= 65536) { q2 = i2 / 100; // really: r = i2 - (q * 100); r = i2 - ((q2 << 6) + (q2 << 5) + (q2 << 2)); i2 = q2; buf[--charPos] = IntegerUtil.digitOnes[r]; buf[--charPos] = IntegerUtil.digitTens[r]; } // Fall thru to fast mode for smaller numbers // assert(i2 <= 65536, i2); for (;;) { q2 = (i2 * 52429) >>> (16 + 3); r = i2 - ((q2 << 3) + (q2 << 1)); // r = i2-(q2*10) ... buf[--charPos] = IntegerUtil.digits[r]; i2 = q2; if (i2 == 0) { break; } } if (sign != 0) { buf[--charPos] = sign; } } } ================================================ FILE: src/main/java/io/mycat/util/MysqlDefs.java ================================================ /* Copyright (C) 2002-2004 MySQL AB This program is free software; you can redistribute it and/or modify it under the terms of version 2 of the GNU AFFERO GENERAL PUBLIC LICENSE as published by the Free Software Foundation. There are special exceptions to the terms and conditions of the GPL as it is applied to this software. View the full text of the exception in file EXCEPTIONS-CONNECTOR-J in the directory of this software distribution. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU AFFERO GENERAL PUBLIC LICENSE for more details. You should have received a copy of the GNU AFFERO GENERAL PUBLIC LICENSE along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ /* * This program is free software; you can redistribute it and/or modify it under the terms of * the GNU AFFERO GENERAL PUBLIC LICENSE as published by the Free Software Foundation; either version 3 of the License, * or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. * See the GNU AFFERO GENERAL PUBLIC LICENSE for more details. * You should have received a copy of the GNU AFFERO GENERAL PUBLIC LICENSE along with this program; * if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ package io.mycat.util; import java.sql.Types; import java.util.HashMap; import java.util.Iterator; import java.util.Map; /** * copy from mysql-connector-j MysqlDefs contains many values that are needed * for communication with the MySQL server. * * @author Mark Matthews * @version $Id: MysqlDefs.java 4724 2005-12-20 23:27:01Z mmatthews $ */ public final class MysqlDefs { // ~ Static fields/initializers // --------------------------------------------- public static final int COM_BINLOG_DUMP = 18; public static final int COM_CHANGE_USER = 17; public static final int COM_CLOSE_STATEMENT = 25; public static final int COM_CONNECT_OUT = 20; public static final int COM_END = 29; public static final int COM_EXECUTE = 23; public static final int COM_FETCH = 28; public static final int COM_LONG_DATA = 24; public static final int COM_PREPARE = 22; public static final int COM_REGISTER_SLAVE = 21; public static final int COM_RESET_STMT = 26; public static final int COM_SET_OPTION = 27; public static final int COM_TABLE_DUMP = 19; public static final int CONNECT = 11; public static final int CREATE_DB = 5; public static final int DEBUG = 13; public static final int DELAYED_INSERT = 16; public static final int DROP_DB = 6; public static final int FIELD_LIST = 4; public static final int FIELD_TYPE_BIT = 16; public static final int FIELD_TYPE_BLOB = 252; public static final int FIELD_TYPE_DATE = 10; public static final int FIELD_TYPE_DATETIME = 12; // Data Types public static final int FIELD_TYPE_DECIMAL = 0; public static final int FIELD_TYPE_DOUBLE = 5; public static final int FIELD_TYPE_ENUM = 247; public static final int FIELD_TYPE_FLOAT = 4; public static final int FIELD_TYPE_GEOMETRY = 255; public static final int FIELD_TYPE_INT24 = 9; public static final int FIELD_TYPE_LONG = 3; public static final int FIELD_TYPE_LONG_BLOB = 251; public static final int FIELD_TYPE_LONGLONG = 8; public static final int FIELD_TYPE_MEDIUM_BLOB = 250; public static final int FIELD_TYPE_NEW_DECIMAL = 246; public static final int FIELD_TYPE_NEWDATE = 14; public static final int FIELD_TYPE_NULL = 6; public static final int FIELD_TYPE_SET = 248; public static final int FIELD_TYPE_SHORT = 2; public static final int FIELD_TYPE_STRING = 254; public static final int FIELD_TYPE_TIME = 11; public static final int FIELD_TYPE_TIMESTAMP = 7; public static final int FIELD_TYPE_TINY = 1; // Older data types public static final int FIELD_TYPE_TINY_BLOB = 249; public static final int FIELD_TYPE_VAR_STRING = 253; public static final int FIELD_TYPE_VARCHAR = 15; // Newer data types public static final int FIELD_TYPE_YEAR = 13; public static final int INIT_DB = 2; public static final long LENGTH_BLOB = 65535; public static final long LENGTH_LONGBLOB = 4294967295L; public static final long LENGTH_MEDIUMBLOB = 16777215; public static final long LENGTH_TINYBLOB = 255; // Limitations public static final int MAX_ROWS = 50000000; // From the MySQL FAQ /** * Used to indicate that the server sent no field-level character set * information, so the driver should use the connection-level character * encoding instead. */ public static final int NO_CHARSET_INFO = -1; public static final byte OPEN_CURSOR_FLAG = 1; public static final int PING = 14; public static final int PROCESS_INFO = 10; public static final int PROCESS_KILL = 12; public static final int QUERY = 3; public static final int QUIT = 1; // ~ Methods // ---------------------------------------------------------------- public static final int RELOAD = 7; public static final int SHUTDOWN = 8; // // Constants defined from mysql // // DB Operations public static final int SLEEP = 0; public static final int STATISTICS = 9; public static final int TIME = 15; /** * Maps the given MySQL type to the correct JDBC type. */ public static int mysqlToJavaType(int mysqlType) { int jdbcType; switch (mysqlType) { case MysqlDefs.FIELD_TYPE_NEW_DECIMAL: case MysqlDefs.FIELD_TYPE_DECIMAL: jdbcType = Types.DECIMAL; break; case MysqlDefs.FIELD_TYPE_TINY: jdbcType = Types.TINYINT; break; case MysqlDefs.FIELD_TYPE_SHORT: jdbcType = Types.SMALLINT; break; case MysqlDefs.FIELD_TYPE_LONG: jdbcType = Types.INTEGER; break; case MysqlDefs.FIELD_TYPE_FLOAT: jdbcType = Types.REAL; break; case MysqlDefs.FIELD_TYPE_DOUBLE: jdbcType = Types.DOUBLE; break; case MysqlDefs.FIELD_TYPE_NULL: jdbcType = Types.NULL; break; case MysqlDefs.FIELD_TYPE_TIMESTAMP: jdbcType = Types.TIMESTAMP; break; case MysqlDefs.FIELD_TYPE_LONGLONG: jdbcType = Types.BIGINT; break; case MysqlDefs.FIELD_TYPE_INT24: jdbcType = Types.INTEGER; break; case MysqlDefs.FIELD_TYPE_DATE: jdbcType = Types.DATE; break; case MysqlDefs.FIELD_TYPE_TIME: jdbcType = Types.TIME; break; case MysqlDefs.FIELD_TYPE_DATETIME: jdbcType = Types.TIMESTAMP; break; case MysqlDefs.FIELD_TYPE_YEAR: jdbcType = Types.DATE; break; case MysqlDefs.FIELD_TYPE_NEWDATE: jdbcType = Types.DATE; break; case MysqlDefs.FIELD_TYPE_ENUM: jdbcType = Types.CHAR; break; case MysqlDefs.FIELD_TYPE_SET: jdbcType = Types.CHAR; break; case MysqlDefs.FIELD_TYPE_TINY_BLOB: jdbcType = Types.VARBINARY; break; case MysqlDefs.FIELD_TYPE_MEDIUM_BLOB: jdbcType = Types.LONGVARBINARY; break; case MysqlDefs.FIELD_TYPE_LONG_BLOB: jdbcType = Types.LONGVARBINARY; break; case MysqlDefs.FIELD_TYPE_BLOB: jdbcType = Types.LONGVARBINARY; break; case MysqlDefs.FIELD_TYPE_VAR_STRING: case MysqlDefs.FIELD_TYPE_VARCHAR: jdbcType = Types.VARCHAR; break; case MysqlDefs.FIELD_TYPE_STRING: jdbcType = Types.CHAR; break; case MysqlDefs.FIELD_TYPE_GEOMETRY: jdbcType = Types.BINARY; break; case MysqlDefs.FIELD_TYPE_BIT: jdbcType = Types.BIT; break; default: jdbcType = Types.VARCHAR; } return jdbcType; } public static int javaTypeDetect(int javaType, int scale) { switch (javaType) { case Types.NUMERIC: { if (scale > 0) { return Types.DECIMAL; }else{ return javaType; } } default: { return javaType; } } } public static int javaTypeMysql(int javaType) { switch (javaType) { case Types.NUMERIC: return MysqlDefs.FIELD_TYPE_DECIMAL; case Types.DECIMAL: return MysqlDefs.FIELD_TYPE_NEW_DECIMAL; case Types.TINYINT: return MysqlDefs.FIELD_TYPE_TINY; case Types.SMALLINT: return MysqlDefs.FIELD_TYPE_SHORT; case Types.INTEGER: return MysqlDefs.FIELD_TYPE_LONG; case Types.REAL: return MysqlDefs.FIELD_TYPE_FLOAT; case Types.DOUBLE: return MysqlDefs.FIELD_TYPE_DOUBLE; case Types.NULL: return MysqlDefs.FIELD_TYPE_NULL; case Types.TIMESTAMP: return MysqlDefs.FIELD_TYPE_TIMESTAMP; case Types.BIGINT: return MysqlDefs.FIELD_TYPE_LONGLONG; case Types.DATE: return MysqlDefs.FIELD_TYPE_DATE; case Types.TIME: return MysqlDefs.FIELD_TYPE_TIME; case Types.VARBINARY: return MysqlDefs.FIELD_TYPE_TINY_BLOB; case Types.LONGVARBINARY: return MysqlDefs.FIELD_TYPE_BLOB; //对应sqlserver的image类型 case 27: return MysqlDefs.FIELD_TYPE_BLOB; case Types.VARCHAR: return MysqlDefs.FIELD_TYPE_VAR_STRING; case Types.CHAR: return MysqlDefs.FIELD_TYPE_STRING; case Types.BINARY: return MysqlDefs.FIELD_TYPE_GEOMETRY; case Types.BIT: return MysqlDefs.FIELD_TYPE_BIT; case Types.CLOB: return MysqlDefs.FIELD_TYPE_VAR_STRING; case Types.BLOB: return MysqlDefs.FIELD_TYPE_BLOB; //修改by magicdoom@gmail.com // 当jdbc连接非mysql的数据库时，需要把对应类型映射为mysql的类型，否则应用端会出错 case Types.NVARCHAR: return MysqlDefs.FIELD_TYPE_VAR_STRING; case Types.NCHAR: return MysqlDefs.FIELD_TYPE_STRING; case Types.NCLOB: return MysqlDefs.FIELD_TYPE_VAR_STRING; case Types.LONGNVARCHAR: return MysqlDefs.FIELD_TYPE_VAR_STRING; default: return MysqlDefs.FIELD_TYPE_VAR_STRING; //其他未知类型返回字符类型 // return Types.VARCHAR; } } /** * Maps the given MySQL type to the correct JDBC type. */ static int mysqlToJavaType(String mysqlType) { if (mysqlType.equalsIgnoreCase("BIT")) { return mysqlToJavaType(FIELD_TYPE_BIT); } else if (mysqlType.equalsIgnoreCase("TINYINT")) { //$NON-NLS-1$ return mysqlToJavaType(FIELD_TYPE_TINY); } else if (mysqlType.equalsIgnoreCase("SMALLINT")) { //$NON-NLS-1$ return mysqlToJavaType(FIELD_TYPE_SHORT); } else if (mysqlType.equalsIgnoreCase("MEDIUMINT")) { //$NON-NLS-1$ return mysqlToJavaType(FIELD_TYPE_INT24); } else if (mysqlType.equalsIgnoreCase("INT") || mysqlType.equalsIgnoreCase("INTEGER")) { //$NON-NLS-1$ //$NON-NLS-2$ return mysqlToJavaType(FIELD_TYPE_LONG); } else if (mysqlType.equalsIgnoreCase("BIGINT")) { //$NON-NLS-1$ return mysqlToJavaType(FIELD_TYPE_LONGLONG); } else if (mysqlType.equalsIgnoreCase("INT24")) { //$NON-NLS-1$ return mysqlToJavaType(FIELD_TYPE_INT24); } else if (mysqlType.equalsIgnoreCase("REAL")) { //$NON-NLS-1$ return mysqlToJavaType(FIELD_TYPE_DOUBLE); } else if (mysqlType.equalsIgnoreCase("FLOAT")) { //$NON-NLS-1$ return mysqlToJavaType(FIELD_TYPE_FLOAT); } else if (mysqlType.equalsIgnoreCase("DECIMAL")) { //$NON-NLS-1$ return mysqlToJavaType(FIELD_TYPE_DECIMAL); } else if (mysqlType.equalsIgnoreCase("NUMERIC")) { //$NON-NLS-1$ return mysqlToJavaType(FIELD_TYPE_DECIMAL); } else if (mysqlType.equalsIgnoreCase("DOUBLE")) { //$NON-NLS-1$ return mysqlToJavaType(FIELD_TYPE_DOUBLE); } else if (mysqlType.equalsIgnoreCase("CHAR")) { //$NON-NLS-1$ return mysqlToJavaType(FIELD_TYPE_STRING); } else if (mysqlType.equalsIgnoreCase("VARCHAR")) { //$NON-NLS-1$ return mysqlToJavaType(FIELD_TYPE_VAR_STRING); } else if (mysqlType.equalsIgnoreCase("DATE")) { //$NON-NLS-1$ return mysqlToJavaType(FIELD_TYPE_DATE); } else if (mysqlType.equalsIgnoreCase("TIME")) { //$NON-NLS-1$ return mysqlToJavaType(FIELD_TYPE_TIME); } else if (mysqlType.equalsIgnoreCase("YEAR")) { //$NON-NLS-1$ return mysqlToJavaType(FIELD_TYPE_YEAR); } else if (mysqlType.equalsIgnoreCase("TIMESTAMP")) { //$NON-NLS-1$ return mysqlToJavaType(FIELD_TYPE_TIMESTAMP); } else if (mysqlType.equalsIgnoreCase("DATETIME")) { //$NON-NLS-1$ return mysqlToJavaType(FIELD_TYPE_DATETIME); } else if (mysqlType.equalsIgnoreCase("TINYBLOB")) { //$NON-NLS-1$ return java.sql.Types.BINARY; } else if (mysqlType.equalsIgnoreCase("BLOB")) { //$NON-NLS-1$ return java.sql.Types.LONGVARBINARY; } else if (mysqlType.equalsIgnoreCase("MEDIUMBLOB")) { //$NON-NLS-1$ return java.sql.Types.LONGVARBINARY; } else if (mysqlType.equalsIgnoreCase("LONGBLOB")) { //$NON-NLS-1$ return java.sql.Types.LONGVARBINARY; } else if (mysqlType.equalsIgnoreCase("TINYTEXT")) { //$NON-NLS-1$ return java.sql.Types.VARCHAR; } else if (mysqlType.equalsIgnoreCase("TEXT")) { //$NON-NLS-1$ return java.sql.Types.LONGVARCHAR; } else if (mysqlType.equalsIgnoreCase("MEDIUMTEXT")) { //$NON-NLS-1$ return java.sql.Types.LONGVARCHAR; } else if (mysqlType.equalsIgnoreCase("LONGTEXT")) { //$NON-NLS-1$ return java.sql.Types.LONGVARCHAR; } else if (mysqlType.equalsIgnoreCase("ENUM")) { //$NON-NLS-1$ return mysqlToJavaType(FIELD_TYPE_ENUM); } else if (mysqlType.equalsIgnoreCase("SET")) { //$NON-NLS-1$ return mysqlToJavaType(FIELD_TYPE_SET); } else if (mysqlType.equalsIgnoreCase("GEOMETRY")) { return mysqlToJavaType(FIELD_TYPE_GEOMETRY); } else if (mysqlType.equalsIgnoreCase("BINARY")) { return Types.BINARY; // no concrete type on the wire } else if (mysqlType.equalsIgnoreCase("VARBINARY")) { return Types.VARBINARY; // no concrete type on the wire } // Punt return java.sql.Types.OTHER; } /** * @param mysqlType * @return */ public static String typeToName(int mysqlType) { switch (mysqlType) { case MysqlDefs.FIELD_TYPE_DECIMAL: return "FIELD_TYPE_DECIMAL"; case MysqlDefs.FIELD_TYPE_TINY: return "FIELD_TYPE_TINY"; case MysqlDefs.FIELD_TYPE_SHORT: return "FIELD_TYPE_SHORT"; case MysqlDefs.FIELD_TYPE_LONG: return "FIELD_TYPE_LONG"; case MysqlDefs.FIELD_TYPE_FLOAT: return "FIELD_TYPE_FLOAT"; case MysqlDefs.FIELD_TYPE_DOUBLE: return "FIELD_TYPE_DOUBLE"; case MysqlDefs.FIELD_TYPE_NULL: return "FIELD_TYPE_NULL"; case MysqlDefs.FIELD_TYPE_TIMESTAMP: return "FIELD_TYPE_TIMESTAMP"; case MysqlDefs.FIELD_TYPE_LONGLONG: return "FIELD_TYPE_LONGLONG"; case MysqlDefs.FIELD_TYPE_INT24: return "FIELD_TYPE_INT24"; case MysqlDefs.FIELD_TYPE_DATE: return "FIELD_TYPE_DATE"; case MysqlDefs.FIELD_TYPE_TIME: return "FIELD_TYPE_TIME"; case MysqlDefs.FIELD_TYPE_DATETIME: return "FIELD_TYPE_DATETIME"; case MysqlDefs.FIELD_TYPE_YEAR: return "FIELD_TYPE_YEAR"; case MysqlDefs.FIELD_TYPE_NEWDATE: return "FIELD_TYPE_NEWDATE"; case MysqlDefs.FIELD_TYPE_ENUM: return "FIELD_TYPE_ENUM"; case MysqlDefs.FIELD_TYPE_SET: return "FIELD_TYPE_SET"; case MysqlDefs.FIELD_TYPE_TINY_BLOB: return "FIELD_TYPE_TINY_BLOB"; case MysqlDefs.FIELD_TYPE_MEDIUM_BLOB: return "FIELD_TYPE_MEDIUM_BLOB"; case MysqlDefs.FIELD_TYPE_LONG_BLOB: return "FIELD_TYPE_LONG_BLOB"; case MysqlDefs.FIELD_TYPE_BLOB: return "FIELD_TYPE_BLOB"; case MysqlDefs.FIELD_TYPE_VAR_STRING: return "FIELD_TYPE_VAR_STRING"; case MysqlDefs.FIELD_TYPE_STRING: return "FIELD_TYPE_STRING"; case MysqlDefs.FIELD_TYPE_VARCHAR: return "FIELD_TYPE_VARCHAR"; case MysqlDefs.FIELD_TYPE_GEOMETRY: return "FIELD_TYPE_GEOMETRY"; default: return " Unknown MySQL Type # " + mysqlType; } } public static boolean isBianry(byte mysqlType) { int type = mysqlType; if(type < 0) { type += 256; } if(type == MysqlDefs.FIELD_TYPE_BLOB || type == MysqlDefs.FIELD_TYPE_TINY_BLOB || type == MysqlDefs.FIELD_TYPE_MEDIUM_BLOB || type == MysqlDefs.FIELD_TYPE_LONG_BLOB) { return true; } return false; } private static Map mysqlToJdbcTypesMap = new HashMap(); static { mysqlToJdbcTypesMap.put("BIT", new Integer( mysqlToJavaType(FIELD_TYPE_BIT))); mysqlToJdbcTypesMap.put("TINYINT", new Integer( mysqlToJavaType(FIELD_TYPE_TINY))); mysqlToJdbcTypesMap.put("SMALLINT", new Integer( mysqlToJavaType(FIELD_TYPE_SHORT))); mysqlToJdbcTypesMap.put("MEDIUMINT", new Integer( mysqlToJavaType(FIELD_TYPE_INT24))); mysqlToJdbcTypesMap.put("INT", new Integer( mysqlToJavaType(FIELD_TYPE_LONG))); mysqlToJdbcTypesMap.put("INTEGER", new Integer( mysqlToJavaType(FIELD_TYPE_LONG))); mysqlToJdbcTypesMap.put("BIGINT", new Integer( mysqlToJavaType(FIELD_TYPE_LONGLONG))); mysqlToJdbcTypesMap.put("INT24", new Integer( mysqlToJavaType(FIELD_TYPE_INT24))); mysqlToJdbcTypesMap.put("REAL", new Integer( mysqlToJavaType(FIELD_TYPE_DOUBLE))); mysqlToJdbcTypesMap.put("FLOAT", new Integer( mysqlToJavaType(FIELD_TYPE_FLOAT))); mysqlToJdbcTypesMap.put("DECIMAL", new Integer( mysqlToJavaType(FIELD_TYPE_DECIMAL))); mysqlToJdbcTypesMap.put("NUMERIC", new Integer( mysqlToJavaType(FIELD_TYPE_DECIMAL))); mysqlToJdbcTypesMap.put("DOUBLE", new Integer( mysqlToJavaType(FIELD_TYPE_DOUBLE))); mysqlToJdbcTypesMap.put("CHAR", new Integer( mysqlToJavaType(FIELD_TYPE_STRING))); mysqlToJdbcTypesMap.put("VARCHAR", new Integer( mysqlToJavaType(FIELD_TYPE_VAR_STRING))); mysqlToJdbcTypesMap.put("DATE", new Integer( mysqlToJavaType(FIELD_TYPE_DATE))); mysqlToJdbcTypesMap.put("TIME", new Integer( mysqlToJavaType(FIELD_TYPE_TIME))); mysqlToJdbcTypesMap.put("YEAR", new Integer( mysqlToJavaType(FIELD_TYPE_YEAR))); mysqlToJdbcTypesMap.put("TIMESTAMP", new Integer( mysqlToJavaType(FIELD_TYPE_TIMESTAMP))); mysqlToJdbcTypesMap.put("DATETIME", new Integer( mysqlToJavaType(FIELD_TYPE_DATETIME))); mysqlToJdbcTypesMap.put("TINYBLOB", new Integer(java.sql.Types.BINARY)); mysqlToJdbcTypesMap.put("BLOB", new Integer( java.sql.Types.LONGVARBINARY)); mysqlToJdbcTypesMap.put("MEDIUMBLOB", new Integer( java.sql.Types.LONGVARBINARY)); mysqlToJdbcTypesMap.put("LONGBLOB", new Integer( java.sql.Types.LONGVARBINARY)); mysqlToJdbcTypesMap .put("TINYTEXT", new Integer(java.sql.Types.VARCHAR)); mysqlToJdbcTypesMap .put("TEXT", new Integer(java.sql.Types.LONGVARCHAR)); mysqlToJdbcTypesMap.put("MEDIUMTEXT", new Integer( java.sql.Types.LONGVARCHAR)); mysqlToJdbcTypesMap.put("LONGTEXT", new Integer( java.sql.Types.LONGVARCHAR)); mysqlToJdbcTypesMap.put("ENUM", new Integer( mysqlToJavaType(FIELD_TYPE_ENUM))); mysqlToJdbcTypesMap.put("SET", new Integer( mysqlToJavaType(FIELD_TYPE_SET))); mysqlToJdbcTypesMap.put("GEOMETRY", new Integer( mysqlToJavaType(FIELD_TYPE_GEOMETRY))); } static final void appendJdbcTypeMappingQuery(StringBuffer buf, String mysqlTypeColumnName) { buf.append("CASE "); Map typesMap = new HashMap(); typesMap.putAll(mysqlToJdbcTypesMap); typesMap.put("BINARY", new Integer(Types.BINARY)); typesMap.put("VARBINARY", new Integer(Types.VARBINARY)); Iterator mysqlTypes = typesMap.keySet().iterator(); while (mysqlTypes.hasNext()) { String mysqlTypeName = (String) mysqlTypes.next(); buf.append(" WHEN "); buf.append(mysqlTypeColumnName); buf.append("='"); buf.append(mysqlTypeName); buf.append("' THEN "); buf.append(typesMap.get(mysqlTypeName)); if (mysqlTypeName.equalsIgnoreCase("DOUBLE") || mysqlTypeName.equalsIgnoreCase("FLOAT") || mysqlTypeName.equalsIgnoreCase("DECIMAL") || mysqlTypeName.equalsIgnoreCase("NUMERIC")) { buf.append(" WHEN "); buf.append(mysqlTypeColumnName); buf.append("='"); buf.append(mysqlTypeName); buf.append(" unsigned' THEN "); buf.append(typesMap.get(mysqlTypeName)); } } buf.append(" ELSE "); buf.append(Types.OTHER); buf.append(" END "); } public static final String SQL_STATE_BASE_TABLE_NOT_FOUND = "S0002"; //$NON-NLS-1$ public static final String SQL_STATE_BASE_TABLE_OR_VIEW_ALREADY_EXISTS = "S0001"; //$NON-NLS-1$ public static final String SQL_STATE_BASE_TABLE_OR_VIEW_NOT_FOUND = "42S02"; //$NON-NLS-1$ public static final String SQL_STATE_COLUMN_ALREADY_EXISTS = "S0021"; //$NON-NLS-1$ public static final String SQL_STATE_COLUMN_NOT_FOUND = "S0022"; //$NON-NLS-1$ public static final String SQL_STATE_COMMUNICATION_LINK_FAILURE = "08S01"; //$NON-NLS-1$ public static final String SQL_STATE_CONNECTION_FAIL_DURING_TX = "08007"; //$NON-NLS-1$ public static final String SQL_STATE_CONNECTION_IN_USE = "08002"; //$NON-NLS-1$ public static final String SQL_STATE_CONNECTION_NOT_OPEN = "08003"; //$NON-NLS-1$ public static final String SQL_STATE_CONNECTION_REJECTED = "08004"; //$NON-NLS-1$ public static final String SQL_STATE_DATE_TRUNCATED = "01004"; //$NON-NLS-1$ public static final String SQL_STATE_DATETIME_FIELD_OVERFLOW = "22008"; //$NON-NLS-1$ public static final String SQL_STATE_DEADLOCK = "41000"; //$NON-NLS-1$ public static final String SQL_STATE_DISCONNECT_ERROR = "01002"; //$NON-NLS-1$ public static final String SQL_STATE_DIVISION_BY_ZERO = "22012"; //$NON-NLS-1$ public static final String SQL_STATE_DRIVER_NOT_CAPABLE = "S1C00"; //$NON-NLS-1$ public static final String SQL_STATE_ERROR_IN_ROW = "01S01"; //$NON-NLS-1$ public static final String SQL_STATE_GENERAL_ERROR = "S1000"; //$NON-NLS-1$ public static final String SQL_STATE_ILLEGAL_ARGUMENT = "S1009"; //$NON-NLS-1$ public static final String SQL_STATE_INDEX_ALREADY_EXISTS = "S0011"; //$NON-NLS-1$ public static final String SQL_STATE_INDEX_NOT_FOUND = "S0012"; //$NON-NLS-1$ public static final String SQL_STATE_INSERT_VALUE_LIST_NO_MATCH_COL_LIST = "21S01"; //$NON-NLS-1$ public static final String SQL_STATE_INVALID_AUTH_SPEC = "28000"; //$NON-NLS-1$ public static final String SQL_STATE_INVALID_CHARACTER_VALUE_FOR_CAST = "22018"; // $NON_NLS // - // 1 // $ public static final String SQL_STATE_INVALID_COLUMN_NUMBER = "S1002"; //$NON-NLS-1$ public static final String SQL_STATE_INVALID_CONNECTION_ATTRIBUTE = "01S00"; //$NON-NLS-1$ public static final String SQL_STATE_MEMORY_ALLOCATION_FAILURE = "S1001"; //$NON-NLS-1$ public static final String SQL_STATE_MORE_THAN_ONE_ROW_UPDATED_OR_DELETED = "01S04"; //$NON-NLS-1$ public static final String SQL_STATE_NO_DEFAULT_FOR_COLUMN = "S0023"; //$NON-NLS-1$ public static final String SQL_STATE_NO_ROWS_UPDATED_OR_DELETED = "01S03"; //$NON-NLS-1$ public static final String SQL_STATE_NUMERIC_VALUE_OUT_OF_RANGE = "22003"; //$NON-NLS-1$ public static final String SQL_STATE_PRIVILEGE_NOT_REVOKED = "01006"; //$NON-NLS-1$ public static final String SQL_STATE_SYNTAX_ERROR = "42000"; //$NON-NLS-1$ public static final String SQL_STATE_TIMEOUT_EXPIRED = "S1T00"; //$NON-NLS-1$ public static final String SQL_STATE_TRANSACTION_RESOLUTION_UNKNOWN = "08007"; // $NON_NLS // - // 1 // $ public static final String SQL_STATE_UNABLE_TO_CONNECT_TO_DATASOURCE = "08001"; //$NON-NLS-1$ public static final String SQL_STATE_WRONG_NO_OF_PARAMETERS = "07001"; //$NON-NLS-1$ public static final String SQL_STATE_INVALID_TRANSACTION_TERMINATION = "2D000"; // $NON_NLS // - // 1 // $ } ================================================ FILE: src/main/java/io/mycat/util/NameableExecutor.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; /** * @author mycat */ public class NameableExecutor extends ThreadPoolExecutor { protected String name; public NameableExecutor(String name, int size, BlockingQueue queue, ThreadFactory factory) { super(size, size, Long.MAX_VALUE, TimeUnit.NANOSECONDS, queue, factory); this.name = name; } public String getName() { return name; } } ================================================ FILE: src/main/java/io/mycat/util/NameableThreadFactory.java ================================================ package io.mycat.util; import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicInteger; public class NameableThreadFactory implements ThreadFactory { private final ThreadGroup group; private final String namePrefix; private final AtomicInteger threadId; private final boolean isDaemon; public NameableThreadFactory(String name, boolean isDaemon) { SecurityManager s = System.getSecurityManager(); this.group = (s != null) ? s.getThreadGroup() : Thread.currentThread() .getThreadGroup(); this.namePrefix = name; this.threadId = new AtomicInteger(0); this.isDaemon = isDaemon; } public Thread newThread(Runnable r) { Thread t = new Thread(group, r, namePrefix + threadId.getAndIncrement()); t.setDaemon(isDaemon); return t; } } ================================================ FILE: src/main/java/io/mycat/util/ObjectUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import java.beans.BeanInfo; import java.beans.IntrospectionException; import java.beans.Introspector; import java.beans.PropertyDescriptor; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.util.Arrays; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author mycat */ public final class ObjectUtil { private static final Logger LOGGER = LoggerFactory.getLogger(ObjectUtil.class); public static Object getStaticFieldValue(String className,String fieldName) { Class clazz = null; try { clazz = Class.forName(className); Field field = clazz.getField(fieldName); if(field!=null) { return field.get(null); } } catch (ClassNotFoundException e) { //LOGGER.error("getStaticFieldValue", e); } catch (NoSuchFieldException e) { // LOGGER.error("getStaticFieldValue", e); } catch (IllegalAccessException e) { // LOGGER.error("getStaticFieldValue", e); } return null; } public static Object copyObject(Object object) { ByteArrayOutputStream b = new ByteArrayOutputStream(); ObjectOutputStream s = null; try { s = new ObjectOutputStream(b); s.writeObject(object); ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(b.toByteArray())); return ois.readObject(); } catch (IOException e) { throw new RuntimeException(e); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } } /** * 递归地比较两个数组是否相同，支持多维数组。 *

* 如果比较的对象不是数组，则此方法的结果同ObjectUtil.equals。 *

* * @param array1 * 数组1 * @param array2 * 数组2 * @return 如果相等, 则返回true */ public static boolean equals(Object array1, Object array2) { if (array1 == array2) { return true; } if ((array1 == null) || (array2 == null)) { return false; } Class clazz = array1.getClass(); if (!clazz.equals(array2.getClass())) { return false; } if (!clazz.isArray()) { return array1.equals(array2); } // array1和array2为同类型的数组 if (array1 instanceof long[]) { long[] longArray1 = (long[]) array1; long[] longArray2 = (long[]) array2; if (longArray1.length != longArray2.length) { return false; } for (int i = 0; i < longArray1.length; i++) { if (longArray1[i] != longArray2[i]) { return false; } } return true; } else if (array1 instanceof int[]) { int[] intArray1 = (int[]) array1; int[] intArray2 = (int[]) array2; if (intArray1.length != intArray2.length) { return false; } for (int i = 0; i < intArray1.length; i++) { if (intArray1[i] != intArray2[i]) { return false; } } return true; } else if (array1 instanceof short[]) { short[] shortArray1 = (short[]) array1; short[] shortArray2 = (short[]) array2; if (shortArray1.length != shortArray2.length) { return false; } for (int i = 0; i < shortArray1.length; i++) { if (shortArray1[i] != shortArray2[i]) { return false; } } return true; } else if (array1 instanceof byte[]) { byte[] byteArray1 = (byte[]) array1; byte[] byteArray2 = (byte[]) array2; if (byteArray1.length != byteArray2.length) { return false; } for (int i = 0; i < byteArray1.length; i++) { if (byteArray1[i] != byteArray2[i]) { return false; } } return true; } else if (array1 instanceof double[]) { double[] doubleArray1 = (double[]) array1; double[] doubleArray2 = (double[]) array2; if (doubleArray1.length != doubleArray2.length) { return false; } for (int i = 0; i < doubleArray1.length; i++) { if (Double.doubleToLongBits(doubleArray1[i]) != Double.doubleToLongBits(doubleArray2[i])) { return false; } } return true; } else if (array1 instanceof float[]) { float[] floatArray1 = (float[]) array1; float[] floatArray2 = (float[]) array2; if (floatArray1.length != floatArray2.length) { return false; } for (int i = 0; i < floatArray1.length; i++) { if (Float.floatToIntBits(floatArray1[i]) != Float.floatToIntBits(floatArray2[i])) { return false; } } return true; } else if (array1 instanceof boolean[]) { boolean[] booleanArray1 = (boolean[]) array1; boolean[] booleanArray2 = (boolean[]) array2; if (booleanArray1.length != booleanArray2.length) { return false; } for (int i = 0; i < booleanArray1.length; i++) { if (booleanArray1[i] != booleanArray2[i]) { return false; } } return true; } else if (array1 instanceof char[]) { char[] charArray1 = (char[]) array1; char[] charArray2 = (char[]) array2; if (charArray1.length != charArray2.length) { return false; } for (int i = 0; i < charArray1.length; i++) { if (charArray1[i] != charArray2[i]) { return false; } } return true; } else { Object[] objectArray1 = (Object[]) array1; Object[] objectArray2 = (Object[]) array2; if (objectArray1.length != objectArray2.length) { return false; } for (int i = 0; i < objectArray1.length; i++) { if (!equals(objectArray1[i], objectArray2[i])) { return false; } } return true; } } public static void copyProperties(Object fromObj, Object toObj) { Class fromClass = fromObj.getClass(); Class toClass = toObj.getClass(); try { BeanInfo fromBean = Introspector.getBeanInfo(fromClass); BeanInfo toBean = Introspector.getBeanInfo(toClass); PropertyDescriptor[] toPd = toBean.getPropertyDescriptors(); List fromPd = Arrays.asList(fromBean .getPropertyDescriptors()); for (PropertyDescriptor propertyDescriptor : toPd) { propertyDescriptor.getDisplayName(); PropertyDescriptor pd = fromPd.get(fromPd .indexOf(propertyDescriptor)); if (pd.getDisplayName().equals( propertyDescriptor.getDisplayName()) && !pd.getDisplayName().equals("class") && propertyDescriptor.getWriteMethod() != null) { propertyDescriptor.getWriteMethod().invoke(toObj, pd.getReadMethod().invoke(fromObj, null)); } } } catch (IntrospectionException e) { throw new RuntimeException(e); } catch (IllegalArgumentException e) { throw new RuntimeException(e); } catch (IllegalAccessException e) { throw new RuntimeException(e); } catch (InvocationTargetException e) { throw new RuntimeException(e); } } } ================================================ FILE: src/main/java/io/mycat/util/ProcessUtil.java ================================================ package io.mycat.util; import io.mycat.migrate.MigrateUtils; import io.mycat.util.dataMigrator.DataMigratorUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.util.Arrays; import java.util.List; public class ProcessUtil { private static Logger LOGGER = LoggerFactory.getLogger((ProcessUtil.class)); public static int exec(String cmd) { Process process = null; try { Runtime runtime = Runtime.getRuntime(); process = runtime.exec(cmd); new StreamGobble(process.getInputStream(), "INFO").start(); new StreamGobble(process.getErrorStream(), "ERROR").start(); return process.waitFor(); } catch (Throwable t) { LOGGER.error(t.getMessage()); } finally { if (process != null) process.destroy(); } return 0; } public static String execReturnString(List cmd) { Process process = null; try { // Runtime runtime = Runtime.getRuntime(); // process = runtime.exec(cmd); ProcessBuilder pb = new ProcessBuilder(cmd); pb.redirectErrorStream(true); process=pb.start(); StreamGobble inputGobble = new StreamGobble(process.getInputStream(), "INFO"); inputGobble.start(); new StreamGobble(process.getErrorStream(), "ERROR").start(); process.waitFor(); return inputGobble.getResult(); } catch (Throwable t) { LOGGER.error(t.getMessage()); } finally { if (process != null) process.destroy(); } return null; } public static String execReturnString(String cmd) { Process process = null; try { Runtime runtime = Runtime.getRuntime(); process = runtime.exec(cmd); StreamGobble inputGobble = new StreamGobble(process.getInputStream(), "INFO"); inputGobble.start(); new StreamGobble(process.getErrorStream(), "ERROR").start(); process.waitFor(); return inputGobble.getResult(); } catch (Throwable t) { LOGGER.error(t.getMessage()); } finally { if (process != null) process.destroy(); } return null; } public static int exec(String cmd,File dir) { Process process = null; try { Runtime runtime = Runtime.getRuntime(); process = runtime.exec(cmd,null,dir); new StreamGobble(process.getInputStream(), "INFO").start(); new StreamGobble(process.getErrorStream(), "ERROR").start(); return process.waitFor(); } catch (Throwable t) { LOGGER.error(t.getMessage()); } finally { if (process != null) process.destroy(); } return 0; } public static void main(String[] args) { // List argss= Arrays.asList("mysqldump", "-h127.0.0.1", "-P3301", "-uczn", // "-p123", "base1","test", "--single-transaction","-q","--default-character-set=utf8mb4","--hex-blob","--where=(_slot>=100 and _slot<=1000) or (_slot>=2000 and _slot <=100000)", "--master-data=1","-Tc:\\999" // ,"--fields-enclosed-by=\\\"","--fields-terminated-by=,", "--lines-terminated-by=\\n", "--fields-escaped-by=\\\\"); // String result= ProcessUtil.execReturnString(argss); // System.out.println(result); } } ================================================ FILE: src/main/java/io/mycat/util/RandomUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; /** * @author mycat */ public class RandomUtil { private static final byte[] bytes = { '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'q', 'w', 'e', 'r', 't', 'y', 'u', 'i', 'o', 'p', 'a', 's', 'd', 'f', 'g', 'h', 'j', 'k', 'l', 'z', 'x', 'c', 'v', 'b', 'n', 'm', 'Q', 'W', 'E', 'R', 'T', 'Y', 'U', 'I', 'O', 'P', 'A', 'S', 'D', 'F', 'G', 'H', 'J', 'K', 'L', 'Z', 'X', 'C', 'V', 'B', 'N', 'M' }; private static final long multiplier = 0x5DEECE66DL; private static final long addend = 0xBL; private static final long mask = (1L << 48) - 1; private static final long integerMask = (1L << 33) - 1; private static final long seedUniquifier = 8682522807148012L; private static long seed; static { long s = seedUniquifier + System.nanoTime(); s = (s ^ multiplier) & mask; seed = s; } public static final byte[] randomBytes(int size) { byte[] bb = bytes; byte[] ab = new byte[size]; for (int i = 0; i < size; i++) { ab[i] = randomByte(bb); } return ab; } private static byte randomByte(byte[] b) { int ran = (int) ((next() & integerMask) >>> 16); return b[ran % b.length]; } private static long next() { long oldSeed = seed; long nextSeed = 0L; do { nextSeed = (oldSeed * multiplier + addend) & mask; } while (oldSeed == nextSeed); seed = nextSeed; return nextSeed; } /** * 随机指定范围内N个不重复的数 * 最简单最基本的方法 * @param min 指定范围最小值（包含） * @param max 指定范围最大值(不包含) * @param n 随机数个数 */ public static int[] getNRandom(int min, int max, int n){ if (n > (max - min + 1) || max < min) { return null; } int[] result = new int[n]; for(int i = 0 ; i < n ; i++){ result[i] = -9999; } int count = 0; while(count < n) { int num = (int) ((Math.random() * (max - min)) + min); boolean flag = true; for (int j = 0; j < n; j++) { if(num == result[j]){ flag = false; break; } } if(flag){ result[count] = num; count++; } } return result; } } ================================================ FILE: src/main/java/io/mycat/util/ResultSetUtil.java ================================================ package io.mycat.util; import java.sql.Connection; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.util.List; import io.mycat.net.mysql.FieldPacket; import io.mycat.net.mysql.RowDataPacket; /** * * @author struct * */ public class ResultSetUtil { public static int toFlag(ResultSetMetaData metaData, int column) throws SQLException { int flags = 0; if (metaData.isNullable(column) == 1) { flags |= 1; } if (metaData.isSigned(column)) { flags |= 16; } if (metaData.isAutoIncrement(column)) { flags |= 128; } return flags; } public static void resultSetToFieldPacket(String charset, List fieldPks, ResultSet rs, boolean isSpark) throws SQLException { ResultSetMetaData metaData = rs.getMetaData(); int colunmCount = metaData.getColumnCount(); if (colunmCount > 0) { //String values=""; for (int i = 0; i < colunmCount; i++) { int j = i + 1; FieldPacket fieldPacket = new FieldPacket(); fieldPacket.orgName = StringUtil.encode(metaData.getColumnName(j),charset); fieldPacket.name = StringUtil.encode(metaData.getColumnLabel(j), charset); if (! isSpark){ fieldPacket.orgTable = StringUtil.encode(metaData.getTableName(j), charset); fieldPacket.table = StringUtil.encode(metaData.getTableName(j), charset); fieldPacket.db = StringUtil.encode(metaData.getSchemaName(j),charset); fieldPacket.flags = toFlag(metaData, j); } fieldPacket.length = metaData.getColumnDisplaySize(j); fieldPacket.decimals = (byte) metaData.getScale(j); int javaType = MysqlDefs.javaTypeDetect( metaData.getColumnType(j), fieldPacket.decimals); fieldPacket.type = (byte) (MysqlDefs.javaTypeMysql(javaType) & 0xff); if(MysqlDefs.isBianry((byte) fieldPacket.type)) { // 63 represent binary character set fieldPacket.charsetIndex = 63; } fieldPks.add(fieldPacket); //values+=metaData.getColumnLabel(j)+"|"+metaData.getColumnName(j)+" "; } // System.out.println(values); } } public static RowDataPacket parseRowData(byte[] row, List fieldValues) { RowDataPacket rowDataPkg = new RowDataPacket(fieldValues.size()); rowDataPkg.read(row); return rowDataPkg; } public static String getColumnValAsString(byte[] row, List fieldValues, int columnIndex) { RowDataPacket rowDataPkg = new RowDataPacket(fieldValues.size()); rowDataPkg.read(row); byte[] columnData = rowDataPkg.fieldValues.get(columnIndex); //columnData 为空时,直接返回null return columnData==null?null:new String(columnData); } public static byte[] getColumnVal(byte[] row, List fieldValues, int columnIndex) { RowDataPacket rowDataPkg = new RowDataPacket(fieldValues.size()); rowDataPkg.read(row); byte[] columnData = rowDataPkg.fieldValues.get(columnIndex); return columnData; } public static byte[] fromHex(String hexString) { String[] hex = hexString.split(" "); byte[] b = new byte[hex.length]; for (int i = 0; i < hex.length; i++) { b[i] = (byte) (Integer.parseInt(hex[i], 16) & 0xff); } return b; } public static void main(String[] args) throws Exception { // byte[] byt = // fromHex("20 00 00 02 03 64 65 66 00 00 00 0A 40 40 73 71 6C 5F 6D 6F 64 65 00 0C 21 00 BA 00 00 00 FD 01 00 1F 00 00"); // MysqlPacketBuffer buffer = new MysqlPacketBuffer(byt); // /* // * ResultSetHeaderPacket packet = new ResultSetHeaderPacket(); // * packet.init(buffer); // */ // FieldPacket[] fields = new FieldPacket[(int) 1]; // for (int i = 0; i < 1; i++) { // fields[i] = new FieldPacket(); // fields[i].init(buffer); // } // System.out.println(1 | 0200); } } ================================================ FILE: src/main/java/io/mycat/util/SelectorUtil.java ================================================ package io.mycat.util; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.channels.SelectableChannel; import java.nio.channels.SelectionKey; import java.nio.channels.Selector; import java.util.ConcurrentModificationException; /** * Selector工具类 * Created by Hash Zhang on 2017/7/24. */ public class SelectorUtil { private static final Logger logger = LoggerFactory.getLogger(SelectorUtil.class); public static final int REBUILD_COUNT_THRESHOLD = 512; public static final long MIN_SELECT_TIME_IN_NANO_SECONDS = 500000L; public static Selector rebuildSelector(final Selector oldSelector) throws IOException { final Selector newSelector; try { newSelector = Selector.open(); } catch (Exception e) { logger.warn("Failed to create a new Selector.", e); return null; } int nChannels = 0; for (;;) { try { for (SelectionKey key: oldSelector.keys()) { Object a = key.attachment(); try { if (!key.isValid() || key.channel().keyFor(newSelector) != null) { continue; } int interestOps = key.interestOps(); key.cancel(); key.channel().register(newSelector, interestOps, a); nChannels ++; } catch (Exception e) { logger.warn("Failed to re-register a Channel to the new Selector.", e); } } } catch (ConcurrentModificationException e) { // Probably due to concurrent modification of the key set. continue; } break; } oldSelector.close(); return newSelector; } } ================================================ FILE: src/main/java/io/mycat/util/SetIgnoreUtil.java ================================================ package io.mycat.util; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; /** * 忽略部分SET 指令 * * 实际使用中PHP用户经常会操作多个SET指令组成一个Stmt , 所以该指令检测功能独立出来 * * @author zhuam * */ public class SetIgnoreUtil { private static List ptrnIgnoreList = new ArrayList(); static { //TODO: 忽略部分 SET 指令, 避免WARN 不断的刷日志 String[] ignores = new String[] { "(?i)set (sql_mode)", "(?i)set (interactive_timeout|wait_timeout|net_read_timeout|net_write_timeout|lock_wait_timeout|slave_net_timeout)", "(?i)set (connect_timeout|delayed_insert_timeout|innodb_lock_wait_timeout|innodb_rollback_on_timeout)", "(?i)set (profiling|profiling_history_size)" }; for (int i = 0; i < ignores.length; ++i) { ptrnIgnoreList.add(Pattern.compile(ignores[i])); } } public static boolean isIgnoreStmt(String stmt) { boolean ignore = false; Matcher matcherIgnore; for (Pattern ptrnIgnore : ptrnIgnoreList) { matcherIgnore = ptrnIgnore.matcher( stmt ); if (matcherIgnore.find()) { ignore = true; break; } } return ignore; } } ================================================ FILE: src/main/java/io/mycat/util/SmallSet.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import java.io.Serializable; import java.util.AbstractSet; import java.util.ArrayList; import java.util.Iterator; import java.util.NoSuchElementException; import java.util.Set; /** * usually one element * * @author mycat */ public final class SmallSet extends AbstractSet implements Set, Cloneable, Serializable { private static final long serialVersionUID = 2037649294658559180L; private final int initSize; private ArrayList list; private E single; private int size; public SmallSet() { this(2); } public SmallSet(int initSize) { this.initSize = initSize; } @Override public boolean add(E e) { switch (size) { case 0: ++size; single = e; return true; case 1: if (isEquals(e, single)) { return false; } list = new ArrayList(initSize); list.add(single); list.add(e); ++size; return true; default: for (int i = 0; i < list.size(); ++i) { E e1 = list.get(i); if (isEquals(e1, e)) { return false; } } list.add(e); ++size; return true; } } private boolean isEquals(E e1, E e2) { if (e1 == null) { return e2 == null; } return e1.equals(e2); } @Override protected Object clone() throws CloneNotSupportedException { return super.clone(); } @Override public Iterator iterator() { return new Iterator() { private int i; private boolean next; @Override public boolean hasNext() { return i < size; } @Override public E next() { next = true; switch (size) { case 0: throw new NoSuchElementException(); case 1: switch (i) { case 0: ++i; return single; default: throw new NoSuchElementException(); } default: try { E e = list.get(i); ++i; return e; } catch (IndexOutOfBoundsException e) { throw new NoSuchElementException(e.getMessage()); } } } @Override public void remove() { if (!next) { throw new IllegalStateException(); } switch (size) { case 0: throw new IllegalStateException(); case 1: size = i = 0; single = null; if (list != null && !list.isEmpty()) { list.remove(0); } break; default: list.remove(--i); if (--size == 1) { single = list.get(0); } break; } next = false; } }; } @Override public int size() { return size; } } ================================================ FILE: src/main/java/io/mycat/util/SplitUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; /** * @author mycat */ public class SplitUtil { private static final String[] EMPTY_STRING_ARRAY = new String[0]; /** * 解析字符串
* 比如:c1='$',c2='-' 输入字符串：mysql_db$0-2
* 输出array:mysql_db[0],mysql_db[1],mysql_db[2] */ public static String[] split2(String src, char c1, char c2) { if (src == null) { return null; } int length = src.length(); if (length == 0) { return EMPTY_STRING_ARRAY; } List list = new LinkedList(); String[] p = split(src, c1, true); if (p.length > 1) { String[] scope = split(p[1], c2, true); int min = Integer.parseInt(scope[0]); int max = Integer.parseInt(scope[scope.length - 1]); for (int x = min; x <= max; x++) { list.add(new StringBuilder(p[0]).append('[').append(x).append(']').toString()); } } else { list.add(p[0]); } return list.toArray(new String[list.size()]); } public static String[] split(String src) { return split(src, null, -1); } public static String[] split(String src, char separatorChar) { if (src == null) { return null; } int length = src.length(); if (length == 0) { return EMPTY_STRING_ARRAY; } List list = new LinkedList(); int i = 0; int start = 0; boolean match = false; while (i < length) { if (src.charAt(i) == separatorChar) { if (match) { list.add(src.substring(start, i)); match = false; } start = ++i; continue; } match = true; i++; } if (match) { list.add(src.substring(start, i)); } return list.toArray(new String[list.size()]); } public static String[] split(String src, char separatorChar, boolean trim) { if (src == null) { return null; } int length = src.length(); if (length == 0) { return EMPTY_STRING_ARRAY; } List list = new LinkedList(); int i = 0; int start = 0; boolean match = false; while (i < length) { if (src.charAt(i) == separatorChar) { if (match) { if (trim) { list.add(src.substring(start, i).trim()); } else { list.add(src.substring(start, i)); } match = false; } start = ++i; continue; } match = true; i++; } if (match) { if (trim) { list.add(src.substring(start, i).trim()); } else { list.add(src.substring(start, i)); } } return list.toArray(new String[list.size()]); } public static String[] split(String str, String separatorChars) { return split(str, separatorChars, -1); } public static String[] split(String src, String separatorChars, int max) { if (src == null) { return null; } int length = src.length(); if (length == 0) { return EMPTY_STRING_ARRAY; } List list = new LinkedList(); int sizePlus1 = 1; int i = 0; int start = 0; boolean match = false; if (separatorChars == null) {// null表示使用空白作为分隔符 while (i < length) { if (Character.isWhitespace(src.charAt(i))) { if (match) { if (sizePlus1++ == max) { i = length; } list.add(src.substring(start, i)); match = false; } start = ++i; continue; } match = true; i++; } } else if (separatorChars.length() == 1) {// 优化分隔符长度为1的情形 char sep = separatorChars.charAt(0); while (i < length) { if (src.charAt(i) == sep) { if (match) { if (sizePlus1++ == max) { i = length; } list.add(src.substring(start, i)); match = false; } start = ++i; continue; } match = true; i++; } } else {// 一般情形 while (i < length) { if (separatorChars.indexOf(src.charAt(i)) >= 0) { if (match) { if (sizePlus1++ == max) { i = length; } list.add(src.substring(start, i)); match = false; } start = ++i; continue; } match = true; i++; } } if (match) { list.add(src.substring(start, i)); } return list.toArray(new String[list.size()]); } /** * 解析字符串，比如:
* 1. c1='$',c2='-',c3='[',c4=']' 输入字符串：mysql_db$0-2
* 输出mysql_db[0],mysql_db[1],mysql_db[2]
* 2. c1='$',c2='-',c3='#',c4='0' 输入字符串：mysql_db$0-2
* 输出mysql_db#0,mysql_db#1,mysql_db#2
* 3. c1='$',c2='-',c3='0',c4='0' 输入字符串：mysql_db$0-2
* 输出mysql_db0,mysql_db1,mysql_db2
*/ public static String[] split(String src, char c1, char c2, char c3, char c4) { if (src == null) { return null; } int length = src.length(); if (length == 0) { return EMPTY_STRING_ARRAY; } List list = new LinkedList(); if (src.indexOf(c1) == -1) { list.add(src.trim()); } else { String[] s = split(src, c1, true); String[] scope = split(s[1], c2, true); int min = Integer.parseInt(scope[0]); int max = Integer.parseInt(scope[scope.length - 1]); if (c3 == '0') { for (int x = min; x <= max; x++) { list.add(new StringBuilder(s[0]).append(x).toString()); } } else if (c4 == '0') { for (int x = min; x <= max; x++) { list.add(new StringBuilder(s[0]).append(c3).append(x).toString()); } } else { for (int x = min; x <= max; x++) { list.add(new StringBuilder(s[0]).append(c3).append(x).append(c4).toString()); } } } return list.toArray(new String[list.size()]); } public static String[] split(String src, char fi, char se, char th) { return split(src, fi, se, th, '0', '0'); } public static String[] split(String src, char fi, char se, char th, char left, char right) { List list = new LinkedList(); String[] pools = split(src, fi, true); for (int i = 0; i < pools.length; i++) { if (pools[i].indexOf(se) == -1) { list.add(pools[i]); continue; } String[] s = split(pools[i], se, th, left, right); for (int j = 0; j < s.length; j++) { list.add(s[j]); } } return list.toArray(new String[list.size()]); } public static String[] splitByByteSize(String string, int size) { if (size < 2) { return new String[]{string}; } byte[] bytes = string.getBytes(); if (bytes.length <= size) { return new String[]{string}; } // 分成的条数不确定(整除的情况下也许会多出一条),所以先用list再转化为array List list = new ArrayList(); int offset = 0;// 偏移量,也就是截取的字符串的首字节的位置 int length = 0;// 截取的字符串的长度,可能是size,可能是size-1 int position = 0;// 可能的截取点,根据具体情况判断是不是在此截取 while (position < bytes.length) { position = offset + size; if (position > bytes.length) { // 最后一条 String s = new String(bytes, offset, bytes.length - offset); list.add(s); break; } if (bytes[position - 1] > 0 || (bytes[position - 1] < 0 && bytes[position - 2] < 0)){ // 截断点是字母,或者是汉字 length = size; } else { // 截断点在汉字中间 length = size - 1; } String s = new String(bytes, offset, length); list.add(s); offset += length; } String[] array = new String[list.size()]; for (int i = 0; i < array.length; i++) { array[i] = (String) list.get(i); } return array; } } ================================================ FILE: src/main/java/io/mycat/util/StreamGobble.java ================================================ package io.mycat.util; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; public class StreamGobble extends Thread { InputStream is; String type; private StringBuffer result=new StringBuffer(); public String getResult() { return result.toString(); } private static Logger LOG = LoggerFactory.getLogger((StreamGobble.class)); StreamGobble(InputStream is, String type) { this.is = is; this.type = type; } public void run() { try { InputStreamReader isr = new InputStreamReader(is); BufferedReader br = new BufferedReader(isr); String line = null; while ((line = br.readLine()) != null) { result.append(line).append("\n"); LOG.info(line); } } catch (IOException ioe) { LOG.error(ioe.getMessage()); } } } ================================================ FILE: src/main/java/io/mycat/util/StringUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import io.mycat.MycatServer; import io.mycat.sqlengine.mpp.LoadData; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.ByteArrayOutputStream; import java.io.UnsupportedEncodingException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Random; /** * @author mycat */ public class StringUtil { public static final String TABLE_COLUMN_SEPARATOR = "."; private static final Logger LOGGER = LoggerFactory.getLogger(StringUtil.class); private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; private static final Random RANDOM = new Random(); private static final char[] CHARS = { '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'q', 'w', 'e', 'r', 't', 'y', 'u', 'i', 'o', 'p', 'a', 's', 'd', 'f', 'g', 'h', 'j', 'k', 'l', 'z', 'x', 'c', 'v', 'b', 'n', 'm', 'Q', 'W', 'E', 'R', 'T', 'Y', 'U', 'I', 'O', 'P', 'A', 'S', 'D', 'F', 'G', 'H', 'J', 'K', 'L', 'Z', 'X', 'C', 'V', 'B', 'N', 'M' }; /** * 字符串hash算法：s[0]*31^(n-1) + s[1]*31^(n-2) + ... + s[n-1]
* 其中s[]为字符串的字符数组，换算成程序的表达式为：
* h = 31*h + s.charAt(i); => h = (h << 5) - h + s.charAt(i);
* * @param start * hash for s.substring(start, end) * @param end * hash for s.substring(start, end) */ public static long hash(String s, int start, int end) { if (start < 0) { start = 0; } if (end > s.length()) { end = s.length(); } long h = 0; for (int i = start; i < end; ++i) { h = (h << 5) - h + s.charAt(i); } return h; } public static byte[] encode(String src, String charset) { if (src == null) { return null; } try { return src.getBytes(charset); } catch (UnsupportedEncodingException e) { return src.getBytes(); } } public static String decode(byte[] src, String charset) { return decode(src, 0, src.length, charset); } public static String decode(byte[] src, int offset, int length, String charset) { try { return new String(src, offset, length, charset); } catch (UnsupportedEncodingException e) { return new String(src, offset, length); } } public static String getRandomString(int size) { StringBuilder s = new StringBuilder(size); int len = CHARS.length; for (int i = 0; i < size; i++) { int x = RANDOM.nextInt(); s.append(CHARS[(x < 0 ? -x : x) % len]); } return s.toString(); } public static String safeToString(Object object) { try { return object.toString(); } catch (Exception t) { LOGGER.error("safeToStringError", t); return ""; } } public static boolean isEmpty(String str) { return ((str == null) || (str.length() == 0)); } public static byte[] hexString2Bytes(char[] hexString, int offset, int length) { if (hexString == null) { return null; } if (length == 0) { return EMPTY_BYTE_ARRAY; } boolean odd = length << 31 == Integer.MIN_VALUE; byte[] bs = new byte[odd ? (length + 1) >> 1 : length >> 1]; for (int i = offset, limit = offset + length; i < limit; ++i) { char high, low; if (i == offset && odd) { high = '0'; low = hexString[i]; } else { high = hexString[i]; low = hexString[++i]; } int b; switch (high) { case '0': b = 0; break; case '1': b = 0x10; break; case '2': b = 0x20; break; case '3': b = 0x30; break; case '4': b = 0x40; break; case '5': b = 0x50; break; case '6': b = 0x60; break; case '7': b = 0x70; break; case '8': b = 0x80; break; case '9': b = 0x90; break; case 'a': case 'A': b = 0xa0; break; case 'b': case 'B': b = 0xb0; break; case 'c': case 'C': b = 0xc0; break; case 'd': case 'D': b = 0xd0; break; case 'e': case 'E': b = 0xe0; break; case 'f': case 'F': b = 0xf0; break; default: throw new IllegalArgumentException("illegal hex-string: " + new String(hexString, offset, length)); } switch (low) { case '0': break; case '1': b += 1; break; case '2': b += 2; break; case '3': b += 3; break; case '4': b += 4; break; case '5': b += 5; break; case '6': b += 6; break; case '7': b += 7; break; case '8': b += 8; break; case '9': b += 9; break; case 'a': case 'A': b += 10; break; case 'b': case 'B': b += 11; break; case 'c': case 'C': b += 12; break; case 'd': case 'D': b += 13; break; case 'e': case 'E': b += 14; break; case 'f': case 'F': b += 15; break; default: throw new IllegalArgumentException("illegal hex-string: " + new String(hexString, offset, length)); } bs[(i - offset) >> 1] = (byte) b; } return bs; } public static String dumpAsHex(byte[] src, int length) { StringBuilder out = new StringBuilder(length * 4); int p = 0; int rows = length / 8; for (int i = 0; (i < rows) && (p < length); i++) { int ptemp = p; for (int j = 0; j < 8; j++) { String hexVal = Integer.toHexString(src[ptemp] & 0xff); if (hexVal.length() == 1) { out.append('0'); } out.append(hexVal).append(' '); ptemp++; } out.append(" "); for (int j = 0; j < 8; j++) { int b = 0xff & src[p]; if (b > 32 && b < 127) { out.append((char) b).append(' '); } else { out.append(". "); } p++; } out.append('\n'); } int n = 0; for (int i = p; i < length; i++) { String hexVal = Integer.toHexString(src[i] & 0xff); if (hexVal.length() == 1) { out.append('0'); } out.append(hexVal).append(' '); n++; } for (int i = n; i < 8; i++) { out.append(" "); } out.append(" "); for (int i = p; i < length; i++) { int b = 0xff & src[i]; if (b > 32 && b < 127) { out.append((char) b).append(' '); } else { out.append(". "); } } out.append('\n'); return out.toString(); } public static byte[] escapeEasternUnicodeByteStream(byte[] src, String srcString, int offset, int length) { if ((src == null) || (src.length == 0)) { return src; } int bytesLen = src.length; int bufIndex = 0; int strIndex = 0; ByteArrayOutputStream out = new ByteArrayOutputStream(bytesLen); while (true) { if (srcString.charAt(strIndex) == '\\') {// write it out as-is out.write(src[bufIndex++]); } else {// Grab the first byte int loByte = src[bufIndex]; if (loByte < 0) { loByte += 256; // adjust for signedness/wrap-around } out.write(loByte);// We always write the first byte if (loByte >= 0x80) { if (bufIndex < (bytesLen - 1)) { int hiByte = src[bufIndex + 1]; if (hiByte < 0) { hiByte += 256; // adjust for signedness/wrap-around } out.write(hiByte);// write the high byte here, and // increment the index for the high // byte bufIndex++; if (hiByte == 0x5C) { out.write(hiByte);// escape 0x5c if necessary } } } else if (loByte == 0x5c && bufIndex < (bytesLen - 1)) { int hiByte = src[bufIndex + 1]; if (hiByte < 0) { hiByte += 256; // adjust for signedness/wrap-around } if (hiByte == 0x62) {// we need to escape the 0x5c out.write(0x5c); out.write(0x62); bufIndex++; } } bufIndex++; } if (bufIndex >= bytesLen) { break;// we're done } strIndex++; } return out.toByteArray(); } public static String toString(byte[] bytes) { if (bytes == null || bytes.length == 0) { return ""; } StringBuffer buffer = new StringBuffer(); for (byte byt : bytes) { buffer.append((char) byt); } return buffer.toString(); } public static boolean equalsIgnoreCase(String str1, String str2) { if (str1 == null) { return str2 == null; } return str1.equalsIgnoreCase(str2); } public static int countChar(String str, char c) { if (str == null || str.isEmpty()) { return 0; } final int len = str.length(); int cnt = 0; for (int i = 0; i < len; ++i) { if (c == str.charAt(i)) { ++cnt; } } return cnt; } public static String replaceOnce(String text, String repl, String with) { return replace(text, repl, with, 1); } public static String replace(String text, String repl, String with) { return replace(text, repl, with, -1); } public static String replace(String text, String repl, String with, int max) { if ((text == null) || (repl == null) || (with == null) || (repl.length() == 0) || (max == 0)) { return text; } StringBuffer buf = new StringBuffer(text.length()); int start = 0; int end = 0; while ((end = text.indexOf(repl, start)) != -1) { buf.append(text.substring(start, end)).append(with); start = end + repl.length(); if (--max == 0) { break; } } buf.append(text.substring(start)); return buf.toString(); } public static String replaceChars(String str, String searchChars, String replaceChars) { return replaceChars(str,searchChars,replaceChars,false); } public static String replaceChars(String str, String searchChars, String replaceChars,boolean force) { if(!force){ return str; } if ((str == null) || (str.length() == 0) || (searchChars == null) || (searchChars.length() == 0)) { return str; } char[] chars = str.toCharArray(); int len = chars.length; boolean modified = false; for (int i = 0, isize = searchChars.length(); i < isize; i++) { char searchChar = searchChars.charAt(i); if ((replaceChars == null) || (i >= replaceChars.length())) {// 删除 int pos = 0; for (int j = 0; j < len; j++) { if (chars[j] != searchChar) { chars[pos++] = chars[j]; } else { modified = true; } } len = pos; } else {// 替换 for (int j = 0; j < len; j++) { if (chars[j] == searchChar) { chars[j] = replaceChars.charAt(i); modified = true; } } } } if (!modified) { return str; } return new String(chars, 0, len); } /** * insert into tablexxx * * @param oriSql * @return */ public static String getTableName(String oriSql) { //此处应该优化为去掉sql中的注释，或兼容注释 String sql=null; if(oriSql.startsWith(LoadData.loadDataHint)) { sql=oriSql.substring(LoadData.loadDataHint.length()) ; } else { sql=oriSql; } int pos = 0; boolean insertFound = false; boolean intoFound = false; int tableStartIndx = -1; int tableEndIndex = -1; while (pos < sql.length()) { char ch = sql.charAt(pos); // 忽略处理注释 /* */ BEN if(ch == '/' && pos+4 < sql.length() && sql.charAt(pos+1) == '*') { if(sql.substring(pos+2).indexOf("*/") != -1) { pos += sql.substring(pos+2).indexOf("*/")+4; continue; } else { // 不应该发生这类情况。 throw new IllegalArgumentException("sql 注释 语法错误"); } } else if (ch <= ' ' || ch == '(' || ch=='`') {// if (tableStartIndx > 0) { tableEndIndex = pos; break; } else { pos++; continue; } } else if (ch == 'i' || ch == 'I') { if (intoFound) { if (tableStartIndx == -1 && ch!='`') { tableStartIndx = pos; } pos++; } else if (insertFound) {// into start // 必须全部都为INTO才认为是into if(pos+5 < sql.length() && (sql.charAt(pos+1) == 'n' || sql.charAt(pos+1) == 'N') && (sql.charAt(pos+2) == 't' || sql.charAt(pos+2) == 'T') && (sql.charAt(pos+3) == 'o' || sql.charAt(pos+3) == 'O') && (sql.charAt(pos+4) <= ' ')) { pos = pos + 5; intoFound = true; } else { pos++; } } else { // 矫正必须全部都为 INSERT才认为是insert // insert start if(pos+7 < sql.length() && (sql.charAt(pos+1) == 'n' || sql.charAt(pos+1) == 'N') && (sql.charAt(pos+2) == 's' || sql.charAt(pos+2) == 'S') && (sql.charAt(pos+3) == 'e' || sql.charAt(pos+3) == 'E') && (sql.charAt(pos+4) == 'r' || sql.charAt(pos+4) == 'R') && (sql.charAt(pos+5) == 't' || sql.charAt(pos+5) == 'T') && (sql.charAt(pos+6) <= ' ')) { pos = pos + 7; insertFound = true; } else { pos++; } } } else { if (tableStartIndx == -1) { tableStartIndx = pos; } pos++; } } return sql.substring(tableStartIndx, tableEndIndex); } /** * 移除`符号 * @param str * @return */ public static String removeBackquote(String str) { if (str == null){ return str; } str = str.trim(); //删除名字中的`tablename`和'value' if (str.length() >= 2) { char firstChar = str.charAt(0); int lastIndex = str.length() - 1; char tailChar = str.charAt(lastIndex); if ((firstChar == '`' && tailChar == '`') || (firstChar == '\'' && tailChar == '\'')) { return str.substring(1, lastIndex); } } return str; } public static String makeString(Object... args) { StringBuilder stringBuilder = new StringBuilder(); for (Object arg : args) { stringBuilder.append(arg); } return stringBuilder.toString(); } public static boolean isNull(String src) { if (src == null || src.trim().equals("") || src.trim().equalsIgnoreCase("undefined")) { return true; } return false; } public static String sha1(String data) throws NoSuchAlgorithmException { MessageDigest md = MessageDigest.getInstance("SHA1"); md.update(data.getBytes()); StringBuffer buf = new StringBuffer(); byte[] bits = md.digest(); for (int i = 0; i < bits.length; i++) { int a = bits[i]; if (a < 0) a += 256; if (a < 16) buf.append("0"); buf.append(Integer.toHexString(a)); } return buf.toString(); } public static void main(String[] args) { String s; s = removeBackquote("`"); assert ("`".equals(s)); s = removeBackquote("```"); assert ("`".equals(s)); s = removeBackquote("'`'"); assert ("`".equals(s)); s = removeBackquote("``"); assert ("".equals(s)); s = removeBackquote("`qqqqq"); assert ("`qqqqq".equals(s)); s = removeBackquote("'qqqqq"); assert ("'qqqqq".equals(s)); s = removeBackquote("qqqqq`"); assert ("qqqqq`".equals(s)); s = removeBackquote("qqqqq\'"); assert ("qqqqq\'".equals(s)); s = removeBackquote("\'qqqqq\'"); assert ("qqqqq".equals(s)); s = removeBackquote("`qqqqq'"); assert ("`qqqqq'".equals(s)); s = removeBackquote("'qqqqq`"); assert ("'qqqqq`".equals(s)); System.out.println(getTableName("insert into ssd (id) values (s)")); System.out.println(getTableName("insert into ssd(id) values (s)")); System.out.println(getTableName(" insert into ssd(id) values (s)")); System.out.println(getTableName(" insert into isd(id) values (s)")); System.out.println(getTableName("INSERT INTO test_activity_input (id,vip_no")); System.out.println(getTableName("/* ApplicationName=DBeaver 3.3.1 - Main connection */ insert into employee(id,name,sharding_id) values(4,’myhome’,10011)")); System.out.println(countChar("insert into ssd (id) values (s) ,(s),(7);",'(')); } } ================================================ FILE: src/main/java/io/mycat/util/TimeUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; /** * 弱精度的计时器，考虑性能不使用同步策略。 * * @author mycat */ public class TimeUtil { private static volatile long CURRENT_TIME = System.currentTimeMillis(); public static final long currentTimeMillis() { return CURRENT_TIME; } public static final long currentTimeNanos() { return System.nanoTime(); } public static final void update() { CURRENT_TIME = System.currentTimeMillis(); } } ================================================ FILE: src/main/java/io/mycat/util/ZKUtils.java ================================================ package io.mycat.util; import io.mycat.MycatServer; import io.mycat.config.loader.zkprocess.comm.ZkConfig; import io.mycat.config.loader.zkprocess.comm.ZkParamCfg; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.recipes.cache.PathChildrenCache; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.apache.curator.framework.state.ConnectionState; import org.apache.curator.framework.state.ConnectionStateListener; import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.curator.retry.RetryForever; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.io.Files; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.*; public class ZKUtils { private static final Logger LOGGER = LoggerFactory.getLogger(ZKUtils.class); static CuratorFramework curatorFramework = null; static ConcurrentMap watchMap = new ConcurrentHashMap<>(); static { curatorFramework = createConnection(); Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { @Override public void run() { if (curatorFramework != null) curatorFramework.close(); watchMap.clear(); } })); } public static String getZKBasePath() { String clasterID = ZkConfig.getInstance().getValue(ZkParamCfg.ZK_CFG_CLUSTERID); return "/mycat/" + clasterID + "/"; } public static CuratorFramework getConnection() { return curatorFramework; } private static CuratorFramework createConnection() { String url = ZkConfig.getInstance().getZkURL(); CuratorFramework curatorFramework = CuratorFrameworkFactory.newClient(url, new ExponentialBackoffRetry(100, 6)); // start connection curatorFramework.start(); // wait 3 second to establish connect try { curatorFramework.blockUntilConnected(3, TimeUnit.SECONDS); if (curatorFramework.getZookeeperClient().isConnected()) { return curatorFramework; } } catch (InterruptedException ignored) { Thread.currentThread().interrupt(); } // fail situation curatorFramework.close(); throw new RuntimeException("failed to connect to zookeeper service : " + url); } public static void closeWatch(List watchs) { for (String watch : watchs) { closeWatch(watch); } } public static void closeWatch(String path) { PathChildrenCache childrenCache = watchMap.get(path); if (childrenCache != null) { try { childrenCache.close(); } catch (IOException e) { throw new RuntimeException(e); } } } public static void addChildPathCache(String path, PathChildrenCacheListener listener) { NameableExecutor businessExecutor = MycatServer.getInstance().getBusinessExecutor(); ExecutorService executor = businessExecutor == null ? Executors.newFixedThreadPool(5) : businessExecutor; try { /** * 监听子节点的变化情况 */ final PathChildrenCache childrenCache = new PathChildrenCache(getConnection(), path, true); childrenCache.start(PathChildrenCache.StartMode.POST_INITIALIZED_EVENT); childrenCache.getListenable().addListener(listener, executor); watchMap.put(path, childrenCache); } catch (Exception e) { throw new RuntimeException(e); } } //写数据到某个路径底下 public static boolean writeProperty( String path, Map propertyMap) throws Exception { // save to zk //try { CuratorFramework client = ZKUtils.getConnection(); //lock.acquire(30,TimeUnit.SECONDS) ; Properties properties=new Properties(); ByteArrayOutputStream out=new ByteArrayOutputStream(); if(client.checkExists().forPath(path)==null) { for(String key : propertyMap.keySet()){ properties.setProperty(key, propertyMap.get(key)); } properties.store(out, "add"); client.create().creatingParentsIfNeeded().forPath(path,out.toByteArray()); return true; } else{ byte[] data = client.getData().forPath(path); properties.load(new ByteArrayInputStream(data)); boolean isUpdate = false; for(String key : propertyMap.keySet()){ String value = propertyMap.get(key); if(!String.valueOf(value).equals(properties.getProperty(key))) { properties.setProperty(key, String.valueOf(value)); isUpdate = true; } } properties.store(out, "update"); //数据有进行更新 if(isUpdate){ client.setData().forPath(path, out.toByteArray()); return true; } return false; } //}finally { // lock.release(); //} } public static void createPath(String path, String data) { //这边应该将结果写入到 dnindex.properties CuratorFramework client = ZKUtils.getConnection(); try { if(client.checkExists().forPath(path) == null) { client.create().creatingParentsIfNeeded().forPath(path, data.getBytes()); } else { client.setData().forPath(path, data.getBytes()); } } catch (Exception e) { System.out.println("放置数据失败"); e.printStackTrace(); } } public static String getDnIndexPath(){ return ZKUtils.getZKBasePath() + "bindata/dnindex.properties"; } } ================================================ FILE: src/main/java/io/mycat/util/cmd/CmdArgs.java ================================================ package io.mycat.util.cmd; import java.util.HashMap; import java.util.Map; /** * -host=192.168.1.1:8080 * - * @author me * */ public class CmdArgs { private static final CmdArgs cmdArgs=new CmdArgs(); private Map args; private CmdArgs(){ args=new HashMap<>(); } public static CmdArgs getInstance(String[] args){ Map cmdArgs=CmdArgs.cmdArgs.args; for(int i=0,l=args.length;i oldDataHosts; private Map oldDataNodes; private Map oldSchemas; private Map newDataHosts; private Map newDataNodes; private Map newSchemas; //即使发生主备切换也使用主数据源 private boolean isAwaysUseMaster; private Properties dnIndexProps; //此类主要目的是通过加载新旧配置文件来获取表迁移信息，migratorTables就是最终要获取的迁移信息集合 private List migratorTables = new ArrayList(); public ConfigComparer(boolean isAwaysUseMaster) throws Exception{ this.isAwaysUseMaster = isAwaysUseMaster; loadOldConfig(); loadNewConfig(); loadTablesFile(); } public List getMigratorTables(){ return migratorTables; } private void loadOldConfig(){ try{ oldLoader = new XMLSchemaLoader(); oldDataHosts = oldLoader.getDataHosts(); oldDataNodes = oldLoader.getDataNodes(); oldSchemas = oldLoader.getSchemas(); }catch(Exception e){ throw new ConfigException(" old config for migrate read fail!please check schema.xml or rule.xml "+e); } } private void loadNewConfig(){ try{ newLoader = new XMLSchemaLoader(NEW_SCHEMA, NEW_RULE); newDataHosts = newLoader.getDataHosts(); newDataNodes = newLoader.getDataNodes(); newSchemas = newLoader.getSchemas(); }catch(Exception e){ throw new ConfigException(" new config for migrate read fail!please check newSchema.xml or newRule.xml "+e); } } private void loadTablesFile() throws Exception{ Properties pro = new Properties(); if(!isAwaysUseMaster){ dnIndexProps = loadDnIndexProps(); } try{ pro.load(ConfigComparer.class.getResourceAsStream(TABLES_FILE)); }catch(Exception e){ throw new ConfigException("tablesFile.properties read fail!"); } Iterator> it = pro.entrySet().iterator(); while(it.hasNext()){ Entry entry = it.next(); String schemaName = entry.getKey().toString(); String tables = entry.getValue().toString(); loadMigratorTables(schemaName,getTables(tables)); } } private String[] getTables(String tables){ if(tables.equalsIgnoreCase("all") || tables.isEmpty()){ return new String[]{}; }else{ return tables.split(","); } } /* * 加载迁移表信息，tables大小为0表示迁移schema下所有表 */ private void loadMigratorTables(String schemaName,String[] tables){ if(!DataMigratorUtil.isKeyExistIgnoreCase(oldSchemas, schemaName)){ throw new ConfigException("oldSchema:"+schemaName+" is not exists!"); } if(!DataMigratorUtil.isKeyExistIgnoreCase(newSchemas,schemaName)){ throw new ConfigException("newSchema:"+schemaName+" is not exists!"); } Map oldTables = DataMigratorUtil.getValueIgnoreCase(oldSchemas, schemaName).getTables(); Map newTables = DataMigratorUtil.getValueIgnoreCase(newSchemas, schemaName).getTables(); if(tables.length>0){ //指定schema下的表进行迁移 for(int i =0;i oldSet = oldTables.keySet(); Set newSet = newTables.keySet(); if(!oldSet.equals(newSet)){ throw new ConfigException("new & old table config is not equal!"); } for(String tableName:oldSet){ TableConfig oldTable = oldTables.get(tableName); TableConfig newTable = newTables.get(tableName); loadMigratorTable(oldTable, newTable,schemaName,tableName); } } } private void loadMigratorTable(TableConfig oldTable,TableConfig newTable,String schemaName,String tableName){ //禁止配置非拆分表 if(oldTable == null || newTable == null){ throw new ConfigException("please check tableFile.properties,make sure "+schemaName+":"+tableName+" is sharding table "); } //忽略全局表 if(oldTable.isGlobalTable()||newTable.isGlobalTable()){ String message = "global table: "+schemaName+":"+tableName+" is ignore!"; System.out.println("Warn: "+message); LOGGER.warn(message); }else{ List oldDN = getDataNodes(oldTable,oldDataNodes,oldDataHosts); List newDN = getDataNodes(newTable,newDataNodes,newDataHosts); //忽略数据节点分布没有发生变化的表 if(isNeedMigrate(oldDN,newDN)){ LOGGER.info("old table:{}",oldTable); LOGGER.info("new table:{}",newTable); if (oldTable.getRule()== null){ LOGGER.info("beacuse {} rule is null,skip it,check it is er child table",oldTable); return; } if (newTable.getRule()==null){ LOGGER.info("beacuse {} rule is null,skip it,check it is er child table.",newTable); return; } checkRuleConfig(oldTable.getRule(), newTable.getRule(),schemaName,tableName); RuleConfig newRC=newTable.getRule(); TableMigrateInfo tmi = new TableMigrateInfo(schemaName, tableName, oldDN, newDN, newRC.getRuleAlgorithm(), newRC.getColumn()); migratorTables.add(tmi); }else{ String message = schemaName+":"+tableName+" is ignore,no need to migrate!"; LOGGER.warn(message); System.out.println("Warn: "+message); } } } //对比前后表数据节点分布是否一致 private boolean isNeedMigrate(List oldDN,List newDN){ if(oldDN.size() != newDN.size()){ return true; } return false; } //获取拆分表对应节点列表,具体到实例地址、库 private List getDataNodes(TableConfig tableConfig,Map dnConfig,Map dhConfig){ List dataNodes = new ArrayList(); //TO-DO ArrayList dataNodeNames = tableConfig.getDataNodes(); int i = 0; for(String name:dataNodeNames){ DataNodeConfig config = dnConfig.get(name); String db = config.getDatabase(); String dataHost = config.getDataHost(); DataHostConfig dh = dhConfig.get(dataHost); String dbType = dh.getDbType(); DBHostConfig[] writeHosts = dh.getWriteHosts(); DBHostConfig currentWriteHost; if(isAwaysUseMaster){ currentWriteHost = writeHosts[0]; }else{ //迁移数据发生在当前切换后的数据源 currentWriteHost = writeHosts[Integer.valueOf(dnIndexProps.getProperty(dh.getName()))]; } DataNode dn = new DataNode(name,currentWriteHost.getIp(), currentWriteHost.getPort(), currentWriteHost.getUser(), currentWriteHost.getPassword(), db, dbType,i++); dataNodes.add(dn); } return dataNodes; } //校验前后路由规则是否一致 private void checkRuleConfig(RuleConfig oldRC,RuleConfig newRC,String schemaName,String tableName){ if(!oldRC.getColumn().equalsIgnoreCase(newRC.getColumn())){ throw new ConfigException(schemaName+":"+tableName+" old & new partition column is not same!"); } AbstractPartitionAlgorithm oldAlg = oldRC.getRuleAlgorithm(); AbstractPartitionAlgorithm newAlg = newRC.getRuleAlgorithm(); //判断路由算法前后是否一致 if(!oldAlg.getClass().isAssignableFrom(newAlg.getClass())){ throw new ConfigException(schemaName+":"+tableName+" old & new rule Algorithm is not same!"); } } private Properties loadDnIndexProps() { Properties prop = new Properties(); InputStream is = null; try { is = ConfigComparer.class.getResourceAsStream(DN_INDEX_FILE); prop.load(is); } catch (Exception e) { throw new ConfigException("please check file \"dnindex.properties\" "+e.getMessage()); } finally { try { if(is !=null){ is.close(); } } catch (IOException e) { throw new ConfigException(e.getMessage()); } } return prop; } } ================================================ FILE: src/main/java/io/mycat/util/dataMigrator/DataClearRunner.java ================================================ package io.mycat.util.dataMigrator; import java.io.File; import java.io.IOException; import java.sql.Connection; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.druid.util.JdbcUtils; /** * 清理数据扩容缩容后的冗余数据 * @author haonan108 * */ public class DataClearRunner implements Runnable{ private static final Logger LOGGER = LoggerFactory.getLogger(DataClearRunner.class); private DataNode srcDn; private File tempFile; private TableMigrateInfo tableInfo; public DataClearRunner(TableMigrateInfo tableInfo,DataNode srcDn,File tempFile){ this.tableInfo = tableInfo; this.srcDn = srcDn; this.tempFile = tempFile; } @Override public void run() { String data = ""; long offset = 0; Connection con = null; try { long start = System.currentTimeMillis(); con = DataMigratorUtil.getMysqlConnection(srcDn); if(tableInfo.isExpantion()){ deleteDataDependFile(data, offset, con); }else{ //缩容，移除的节点直接truncate删除数据，非移除的节点按照临时文件的中值进行删除操作 List list = tableInfo.getRemovedDataNodes(); boolean isRemovedDn = false; for(DataNode dn:list){ if(srcDn.equals(dn)){ isRemovedDn = true; } } if(isRemovedDn){ String sql = "truncate "+tableInfo.getTableName(); JdbcUtils.execute(con, sql, new ArrayList<>()); }else{ deleteDataDependFile(data, offset, con); } } long end = System.currentTimeMillis(); System.out.println(tableInfo.getSchemaAndTableName()+" clean dataNode "+srcDn.getName()+" completed in "+(end-start)+"ms"); } catch (Exception e) { String errMessage = srcDn.toString()+":"+"clean data error!"; LOGGER.error(errMessage, e); tableInfo.setError(true); tableInfo.getErrMessage().append(errMessage+"\n"); } finally{ JdbcUtils.close(con); } } private void deleteDataDependFile(String data,long offset,Connection con) throws IOException, SQLException{ while((data=DataMigratorUtil.readData(tempFile,offset,DataMigrator.margs.getQueryPageSize())).length()>0){ offset += data.getBytes().length; if(data.startsWith(",")){ data = data.substring(1, data.length()); } if(data.endsWith(",")){ data = data.substring(0,data.length()-1); } String sql = "delete from "+tableInfo.getTableName()+" where "+tableInfo.getColumn()+" in ("+data+")"; JdbcUtils.execute(con, sql, new ArrayList<>()); } } } ================================================ FILE: src/main/java/io/mycat/util/dataMigrator/DataIO.java ================================================ package io.mycat.util.dataMigrator; import java.io.File; import java.io.IOException; /** * 数据导入导出接口，mysql、oracle等数据库通过实现此接口提供具体的数据导入导出功能 * @author haonan108 * */ public interface DataIO { /** * 导入数据 * @param dn 导入到具体的数据库 * @param file 导入的文件 * @throws IOException * @throws InterruptedException */ void importData(TableMigrateInfo table,DataNode dn,String tableName,File file) throws IOException, InterruptedException; /** * 根据条件导出迁移数据 * @param dn 导出哪个具体的数据库 * @param tableName 导出的表名称 * @param export 文件导出到哪里 * @param condion 导出文件依赖的具体条件 * @return * @throws IOException * @throws InterruptedException */ File exportData(TableMigrateInfo table,DataNode dn,String tableName,File exportPath,File condion) throws IOException, InterruptedException; } ================================================ FILE: src/main/java/io/mycat/util/dataMigrator/DataIOFactory.java ================================================ package io.mycat.util.dataMigrator; import io.mycat.util.dataMigrator.dataIOImpl.MysqlDataIO; import io.mycat.util.exception.DataMigratorException; public class DataIOFactory { public static final String MYSQL = "mysql"; public static final String ORACLE = "oracle"; public static DataIO createDataIO(String dbType){ switch (dbType) { case MYSQL: return new MysqlDataIO(); default: throw new DataMigratorException("dbType:"+dbType+" is not support for the moment!"); } } } ================================================ FILE: src/main/java/io/mycat/util/dataMigrator/DataMigrateRunner.java ================================================ package io.mycat.util.dataMigrator; import java.io.File; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * 数据迁移过程类 * @author haonan108 * */ public class DataMigrateRunner implements Runnable{ private static final Logger LOGGER = LoggerFactory.getLogger(DataMigrateRunner.class); private DataNode src; private DataNode target; private String tableName; private DataIO dataIO; private File conditionFile; private TableMigrateInfo table; public DataMigrateRunner(TableMigrateInfo table, DataNode src,DataNode target,String tableName,File conditionFile){ this.tableName = tableName; this.conditionFile= conditionFile; this.src = src; this.target = target; this.table = table; dataIO = DataIOFactory.createDataIO(src.getDbType()); } @Override public void run() { if(table.isError()) { return; } try { long start = System.currentTimeMillis(); File loadFile = dataIO.exportData(table,src, tableName, conditionFile.getParentFile(), conditionFile); dataIO.importData(table,target,tableName, loadFile); long end = System.currentTimeMillis(); System.out.println(table.getSchemaAndTableName()+" "+src.getName()+"->"+target.getName()+" completed in "+(end-start)+"ms"); } catch (Exception e) { String errMessage = table.getSchemaAndTableName()+" "+src.getName()+"->"+target.getName()+" migrate err! "+e.getMessage(); LOGGER.error(errMessage, e); table.setError(true); table.getErrMessage().append(errMessage); } } } ================================================ FILE: src/main/java/io/mycat/util/dataMigrator/DataMigrator.java ================================================ package io.mycat.util.dataMigrator; import java.io.File; import java.sql.SQLException; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * 数据迁移统一调度类，支持扩容缩容 * 原理：读取需要迁移的数据节点表所有拆分字段数据，按照扩容或缩容后的配置对拆分字段重新计算路由节点， * 将需要迁移的数据导出，然后导入到扩容或缩容后对应的数据节点 * @author haonan108 * */ public class DataMigrator { private static final Logger LOGGER = LoggerFactory.getLogger(DataMigrator.class); public static DataMigratorArgs margs; private List migrateTables; private ExecutorService executor; private List clearGroup = new ArrayList<>(); public DataMigrator(String[] args){ margs = new DataMigratorArgs(args); executor = new ThreadPoolExecutor(margs.getThreadCount(), margs.getThreadCount(), 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue(),new ThreadPoolExecutor.CallerRunsPolicy()); try { createTempParentDir(margs.getTempFileDir()); ConfigComparer loader = new ConfigComparer(margs.isAwaysUseMaster()); migrateTables = loader.getMigratorTables(); //建表 for(TableMigrateInfo table:migrateTables){ table.setTableStructure(); table.createTableToNewDataNodes(); } } catch (Exception e) { LOGGER.error(e.getMessage(),e); System.out.println(e.getMessage()); //配置错误退出迁移程序 System.exit(-1); } } public static void main(String[] args) throws SQLException { long start = System.currentTimeMillis(); DateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SSS"); System.out.println("\n"+format.format(new Date())+" [1]-> creating migrator schedule and temp files for migrate..."); //初始化配置 DataMigrator migrator = new DataMigrator(args); //生成中间文件 migrator.createTempFiles(); migrator.changeSize(); migrator.printInfo(); //迁移数据 System.out.println("\n"+format.format(new Date())+" [2]-> start migrate data..."); migrator.migrateData(); //清除中间临时文件、清除被迁移掉的冗余数据 System.out.println("\n"+format.format(new Date())+" [3]-> cleaning redundant data..."); migrator.clear(); //校验数据是否迁移成功 System.out.println("\n"+format.format(new Date())+" [4]-> validating tables migrate result..."); migrator.validate(); migrator.clearTempFiles(); long end = System.currentTimeMillis(); System.out.println("\n"+format.format(new Date())+" migrate data complete in "+(end-start)+"ms"); } //打印各个表的迁移数据信息 private void printInfo() { for(TableMigrateInfo table:migrateTables){ table.printMigrateInfo(); table.printMigrateSchedule(); } } //删除临时文件 private void clearTempFiles() { File tempFileDir = new File(margs.getTempFileDir()); if(tempFileDir.exists() && margs.isDeleteTempDir()){ DataMigratorUtil.deleteDir(tempFileDir); } } //生成需要进行迁移的数据依赖的拆分字段值文件 private void createTempFiles(){ for(TableMigrateInfo table:migrateTables){ //创建具体拆分表中间临时文件 createTableTempFiles(table); } executor.shutdown(); while(true){ if(executor.isTerminated()){ break; } try { Thread.sleep(200); } catch (InterruptedException e) { LOGGER.error("error",e); } } } private void migrateData() throws SQLException{ executor = new ThreadPoolExecutor(margs.getThreadCount(), margs.getThreadCount(), 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue(),new ThreadPoolExecutor.CallerRunsPolicy()); for(TableMigrateInfo table:migrateTables){ if(!table.isError()){ //忽略已出错的拆分表 List detailList = table.getDataNodesDetail(); for(DataNodeMigrateInfo info:detailList){ LOGGER.info("{}",info); executor.execute(new DataMigrateRunner(table, info.getSrc(), info.getTarget(), table.getTableName(), info.getTempFile())); } } } executor.shutdown(); while(true){ if(executor.isTerminated()){ break; } try { Thread.sleep(200); } catch (InterruptedException e) { LOGGER.error("error",e); } } } //缩容需要重新计算表大小 private void changeSize() throws SQLException { for(TableMigrateInfo table:migrateTables){ if(!table.isExpantion()){ List oldDn = table.getOldDataNodes(); long size = 0L; for(DataNode dn:oldDn){ size+=DataMigratorUtil.querySize(dn, table.getTableName()); } table.setSize(size); } } } //校验迁移计划中数据迁移情况同数据实际落盘是否一致 private void validate() throws SQLException { for(TableMigrateInfo table:migrateTables){ if (table.isError()) { continue; } long size = table.getSize().get(); long factSize = 0L; for(DataNode dn:table.getNewDataNodes()){ factSize+=DataMigratorUtil.querySize(dn, table.getTableName()); } if(factSize != size){ String message = "migrate error!after migrate should be:"+size+" but fact is:"+factSize; table.setError(true); table.setErrMessage(message); } } //打印最终迁移结果信息 String title = "migrate result"; Map result = new HashMap(); for(TableMigrateInfo table:migrateTables){ String resultMessage = table.isError()?"fail! reason: "+table.getErrMessage():"success"; result.put(table.getSchemaAndTableName(), resultMessage); } String info = DataMigratorUtil.printMigrateInfo(title, result, "->"); System.out.println(info); } //清除中间临时文件、导出的迁移数据文件、已被迁移的原始节点冗余数据 private void clear(){ for(TableMigrateInfo table:migrateTables){ makeClearDataGroup(table); } for(DataNodeClearGroup group:clearGroup){ clearData(group.getTempFiles(), group.getTableInfo()); } } //同一主机上的mysql执行按where条件删除数据并发多了性能反而下降很快 //按照主机ip进行分组，每个主机ip分配一个线程池，线程池大小可配置，默认为当前主机环境cpu核数的一半 private void makeClearDataGroup(TableMigrateInfo table){ List list = table.getDataNodesDetail(); //将数据节点按主机ip分组，每组分配一个线程池 for(DataNodeMigrateInfo dnInfo:list){ DataNode src = dnInfo.getSrc(); String ip =src.getIp(); File f = dnInfo.getTempFile(); DataNodeClearGroup group = getDataNodeClearGroup(ip,table); if(group == null){ group = new DataNodeClearGroup(ip, table); clearGroup.add(group); } group.getTempFiles().put(f, src); } } private DataNodeClearGroup getDataNodeClearGroup(String ip, TableMigrateInfo table){ DataNodeClearGroup result = null; for(DataNodeClearGroup group:clearGroup){ if(group.getIp().equals(ip) && group.getTableInfo().equals(table)){ result = group; } } return result; } private void clearData(Map map,TableMigrateInfo table){ if(table.isError()) { return; } ExecutorService executor = new ThreadPoolExecutor(margs.getThreadCount(), margs.getThreadCount(), 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue(),new ThreadPoolExecutor.CallerRunsPolicy()); Iterator> it = map.entrySet().iterator(); while(it.hasNext()){ Entry et = it.next(); File f =et.getKey(); DataNode srcDn = et.getValue(); executor.execute(new DataClearRunner(table, srcDn, f)); } executor.shutdown(); while(true){ if(executor.isTerminated()){ break; } try { Thread.sleep(200); } catch (InterruptedException e) { LOGGER.error("error",e); } } } private void createTempParentDir(String dir){ File outputDir = new File(dir); if(outputDir.exists()){ DataMigratorUtil.deleteDir(outputDir); } outputDir.mkdirs(); outputDir.setWritable(true); } private void createTableTempFiles(TableMigrateInfo table) { List oldDn = table.getOldDataNodes(); //生成迁移中间文件，并生成迁移执行计划 for(DataNode dn:oldDn){ executor.execute(new MigratorConditonFilesMaker(table,dn,margs.getTempFileDir(),margs.getQueryPageSize())); } } } ================================================ FILE: src/main/java/io/mycat/util/dataMigrator/DataMigratorArgs.java ================================================ package io.mycat.util.dataMigrator; import java.io.File; import io.mycat.config.model.SystemConfig; import io.mycat.util.cmd.CmdArgs; /** * 数据迁移工具依赖参数 * @author haonan108 * */ public class DataMigratorArgs { /** 并行线程数*/ public static final String THREAD_COUNT = "threadCount"; /** mysqldump命令所在路径 */ public static final String MYSQL_BIN = "mysqlBin"; /** 数据迁移生成的中间文件指定存放目录*/ public static final String TEMP_FILE_DIR = "tempFileDir"; /** 使用主数据源还是当前数据源(如果发生主备切换存在数据源选择问题)*/ public static final String IS_AWAYS_USE_MASTER = "isAwaysUseMaster"; /**生成中间临时文件一次加载的数据量*/ public static final String QUERY_PAGE_SIZE = "queryPageSize"; public static final String DEL_THRAD_COUNT = "delThreadCount"; /** mysqldump导出中间文件命令操作系统限制长度 */ public static final String MYSQL_DUMP_CMD_LENGTH = "cmdLength"; public static final String CHARSET = "charset"; public static final String GTID_PURGED = "gtidPurged"; /**完成扩容缩容后清除临时文件 默认为true*/ public static final String DELETE_TEMP_FILE_DIR = "deleteTempFileDir"; private static final int DEFAULT_THREAD_COUNT = Runtime.getRuntime().availableProcessors()*2; private static final int DEFAULT_DEL_THRAD_COUNT = Runtime.getRuntime().availableProcessors()/2; private static final int DEFAULT_CMD_LENGTH = 110*1024;//操作系统命令行限制长度 110KB private static final int DEFAULT_PAGE_SIZE = 100000;//默认一次读取10w条数据 private static final String DEFAULT_CHARSET = "utf8"; private CmdArgs cmdArgs; public DataMigratorArgs(String[] args){ cmdArgs = CmdArgs.getInstance(args); } public String getString(String name){ return cmdArgs.getString(name); } public String getMysqlBin(){ String result = getString(MYSQL_BIN); if(result ==null) { return ""; } if(!result.isEmpty() &&!result.endsWith("/")){ result +="/"; } return result; } public String getTempFileDir(){ String path = getString(TEMP_FILE_DIR); if(null == path || path.trim().isEmpty()){ return SystemConfig.getHomePath()+File.separator+"temp"; } return path; } public int getThreadCount(){ String count =getString(THREAD_COUNT); if(null == count||count.isEmpty()|| count.equals("0") ){ return DEFAULT_THREAD_COUNT; } return Integer.valueOf(count); } public int getDelThreadCount(){ String count =getString(DEL_THRAD_COUNT); if(null == count||count.isEmpty()|| count.equals("0") ){ return DEFAULT_DEL_THRAD_COUNT; } return Integer.valueOf(count); } public boolean isAwaysUseMaster(){ String result = getString(IS_AWAYS_USE_MASTER); if(null == result||result.isEmpty()||result.equals("true")){ return true; } return false; } public int getCmdLength(){ String result = getString(MYSQL_DUMP_CMD_LENGTH); if(null == result||result.isEmpty()){ return DEFAULT_CMD_LENGTH; } if(result.contains("*")){ String[] arr = result.split("\\*"); int j = 1; for (int i = 0; i < arr.length; i++) { j *= Integer.valueOf(arr[i]); } return j; } return Integer.valueOf(result); } public int getQueryPageSize(){ String result = getString(QUERY_PAGE_SIZE); if(null == result||result.isEmpty()){ return DEFAULT_PAGE_SIZE; } return Integer.valueOf(result); } public String getCharSet(){ String result = getString(CHARSET); if(null == result||result.isEmpty()){ return DEFAULT_CHARSET; } return result; } public boolean isGtidPurged(){ String result = getString(GTID_PURGED); if(null == result||result.isEmpty()){ return true; } return Boolean.parseBoolean(result); } public boolean isDeleteTempDir(){ String result = getString(DELETE_TEMP_FILE_DIR); if(null == result||result.isEmpty()||result.equals("true")){ return true; } return false; } } ================================================ FILE: src/main/java/io/mycat/util/dataMigrator/DataMigratorUtil.java ================================================ package io.mycat.util.dataMigrator; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.util.*; import java.util.Map.Entry; import java.util.regex.Matcher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.druid.util.JdbcUtils; public class DataMigratorUtil { private static final Logger LOGGER = LoggerFactory.getLogger(DataMigratorUtil.class); static{ try { Class.forName("com.mysql.jdbc.Driver"); } catch (ClassNotFoundException e) { LOGGER.error("",e); } } /** * 添加数据到文件末尾 * @param file * @param content * @throws IOException */ public static void appendDataToFile(File file, String content) throws IOException { RandomAccessFile randomFile = null; try { // 打开一个随机访问文件流，按读写方式 randomFile = new RandomAccessFile(file, "rw"); // 文件长度，字节数 long fileLength = randomFile.length(); // 将写文件指针移到文件尾。 randomFile.seek(fileLength); randomFile.writeBytes(content); content = null; } catch (IOException e) { LOGGER.error("appendDataToFile is error!",e); } finally{ if(randomFile != null){ try { randomFile.close(); } catch (IOException e) { LOGGER.error("error",e); } } } } public static String readDataFromFile(File file,long offset,int length) throws IOException{ RandomAccessFile randomFile = null; try { // 打开一个随机访问文件流，按读写方式 randomFile = new RandomAccessFile(file, "rw"); randomFile.seek(offset); byte[] buffer = new byte[length]; randomFile.read(buffer); return new String(buffer).trim(); } catch (IOException e) { throw e; } finally{ if(randomFile != null){ try { randomFile.close(); } catch (IOException e) { LOGGER.error("error",e); } } } } /** * 读取逗号分隔的文件数据 * @param file * @param start 文件起始位置 * @param length 读取字节数 * @return * @throws IOException */ public static String readData(File file,long start,int length) throws IOException{ String data = readDataFromFile(file, start, length); if((start+length)<=file.length()){ data = data.substring(0, data.lastIndexOf(",")); } return data; } public static final int BUFSIZE = 1024 * 8; public static void mergeFiles(File outFile, File f) throws IOException { FileChannel outChannel = null; FileOutputStream fos = null; FileInputStream fis = null; try { fos = new FileOutputStream(outFile,true); fis = new FileInputStream(f); outChannel = fos.getChannel(); FileChannel fc = fis.getChannel(); ByteBuffer bb = ByteBuffer.allocate(BUFSIZE); while(fc.read(bb) != -1){ bb.flip(); outChannel.write(bb); bb.clear(); } fc.close(); } catch (IOException e) { throw e; } finally { try { if(fos != null){ fos.close(); } if(fis != null){ fis.close(); } if (outChannel != null){ outChannel.close(); } } catch (IOException e) { LOGGER.error("error",e); } } } /** * 统计文件有多少行 * @param file * @return */ public static long countLine(File file) throws IOException{ long count = 0L; RandomAccessFile randomFile = null; // 打开一个随机访问文件流，按读写方式 try { randomFile = new RandomAccessFile(file, "rw"); String s =""; while((s=randomFile.readLine())!=null && !s.trim().isEmpty()){ count++; } } catch (FileNotFoundException e) { throw e; }finally{ if(randomFile != null){ try { randomFile.close(); } catch (IOException e) { LOGGER.error("error",e); } } } return count; } /** * 递归删除目录下的所有文件及子目录下所有文件 * @param dir 将要删除的文件目录 * @return boolean Returns "true" if all deletions were successful. * If a deletion fails, the method stops attempting to * delete and returns "false". */ public static boolean deleteDir(File dir) { if (dir.isDirectory()) { String[] children = dir.list(); for (int i=0; i paramList= Arrays.asList(params); for(Object param:paramList){ cmd = cmd.replaceFirst("\\"+mark, Matcher.quoteReplacement(param.toString())); } return cmd; } public static Connection getMysqlConnection(DataNode dn) throws SQLException{ Connection con = null; con = DriverManager.getConnection(dn.getUrl(), dn.getUserName(), dn.getPwd()); return con; } public static List> executeQuery(Connection conn, String sql,Object... parameters) throws SQLException{ return JdbcUtils.executeQuery(conn, sql, Arrays.asList(parameters)); } //查询表数据量 public static long querySize(DataNode dn,String tableName) throws SQLException{ List> list=null; long size = 0L; Connection con = null; try { con = getMysqlConnection(dn); list = executeQuery(con, "select count(1) size from "+tableName); size = (long) list.get(0).get("size"); } catch (SQLException e) { throw e; }finally{ JdbcUtils.close(con); } return size; } public static void createTable(DataNode dn,String table) throws SQLException{ Connection con = null; try { con = getMysqlConnection(dn); JdbcUtils.execute(con, table, new ArrayList<>()); } catch (SQLException e) { throw e; }finally{ JdbcUtils.close(con); } } /** * 格式化数据迁移信息 * +---------title-------+ * |key1 = value1 | * |key2 = value2 | * |... | * +---------------------+ * @param title * @param map * @param mark * @return */ public static String printMigrateInfo(String title,Map map,String mark){ StringBuilder result = new StringBuilder(" "); List mergeList = new ArrayList<>(); Iterator> itor = map.entrySet().iterator(); int maxKeyLength = 0; int maxValueLength = 0; while(itor.hasNext()){ Entry entry = itor.next(); String key = entry.getKey(); String value = entry.getValue(); maxKeyLength = (key.length()>maxKeyLength)?key.length():maxKeyLength; maxValueLength = (value.length()>maxValueLength)?value.length():maxValueLength; } int maxLength=maxKeyLength+maxValueLength+2+mark.length(); if(maxLength<= title.length()){ maxLength = title.length()+8; } itor = map.entrySet().iterator(); //合并key和value，并找出长度最大的字符串 while(itor.hasNext()){ Entry entry = itor.next(); String key = entry.getKey(); String value = entry.getValue(); int keyLength = maxKeyLength-key.length(); StringBuilder keySb = new StringBuilder(key); for(int i=0;i maxLineLength){ maxLength = maxLineLength; } //拼第一行title StringBuilder titleSb = new StringBuilder("+"); int halfLength = (maxLength-title.length())/2; for(int i=0;i changeList = new ArrayList<>(); //调整内容 for(int i=0;i=maxLength){ String[] str = content.split(mark); String key = str[0]; String value =str[1]; String[] values = getValues(value,maxLength-maxKeyLength-1-mark.length()); for(int j=0;j 0){ StringBuilder keySb = new StringBuilder(); for(int k=0;k boolean isKeyExistIgnoreCase(Map map,String key){ return map.containsKey(key.toLowerCase()) || map.containsKey(key.toUpperCase()) || map.containsKey(key); } public static T getValueIgnoreCase(Map map,String key){ T result = map.get(key); if (result == null){ result = map.get(key.toLowerCase()); } if (result == null){ result = map.get(key.toUpperCase()); } return result; } public static Process exeCmdByOs(String cmd) throws IOException{ Process process = null; Runtime runtime = Runtime.getRuntime(); String osName = System.getProperty("os.name"); if(osName.toLowerCase().startsWith("win")){ process = runtime.exec((new String[]{"cmd","/C",cmd})); }else{ process = runtime.exec((new String[]{"sh","-c",cmd})); } return process; } private static String[] getValues(String value, int maxValueLength) { int length = value.length()/maxValueLength; if(value.length()%maxValueLength>0){ length+=1; } String[] result = new String[length]; for(int i=0;i tempFiles = new HashMap<>(); private TableMigrateInfo tableInfo; public DataNodeClearGroup(String ip, TableMigrateInfo tableInfo) { super(); this.ip = ip; this.tableInfo = tableInfo; } public String getIp() { return ip; } public void setIp(String ip) { this.ip = ip; } public Map getTempFiles() { return tempFiles; } public void setTempFiles(Map tempFiles) { this.tempFiles = tempFiles; } public TableMigrateInfo getTableInfo() { return tableInfo; } public void setTableInfo(TableMigrateInfo tableInfo) { this.tableInfo = tableInfo; } } ================================================ FILE: src/main/java/io/mycat/util/dataMigrator/DataNodeMigrateInfo.java ================================================ package io.mycat.util.dataMigrator; import java.io.File; /** * 数据迁移时数据节点间迁移信息 * @author haonan108 * */ public class DataNodeMigrateInfo { private DataNode src; private DataNode target; private File tempFile; private long size; private TableMigrateInfo table; public DataNodeMigrateInfo(TableMigrateInfo table, DataNode src, DataNode target, File tempFile, long size) { super(); this.table = table; this.src = src; this.target = target; this.tempFile = tempFile; this.size = size; } public TableMigrateInfo getTable() { return table; } public void setTable(TableMigrateInfo table) { this.table = table; } public DataNode getSrc() { return src; } public void setSrc(DataNode src) { this.src = src; } public DataNode getTarget() { return target; } public void setTarget(DataNode target) { this.target = target; } public File getTempFile() { return tempFile; } public void setTempFile(File tempFile) { this.tempFile = tempFile; } public long getSize() { return size; } public void setSize(long size) { this.size = size; } } ================================================ FILE: src/main/java/io/mycat/util/dataMigrator/MigratorConditonFilesMaker.java ================================================ package io.mycat.util.dataMigrator; import io.mycat.util.StringUtil; import java.io.File; import java.io.IOException; import java.sql.Connection; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.druid.util.JdbcUtils; import io.mycat.route.function.AbstractPartitionAlgorithm; import io.mycat.util.CollectionUtil; /** * 对具体某个节点重新路由 生成导出数据所依赖的中间文件 * @author haonan108 */ public class MigratorConditonFilesMaker implements Runnable{ private static final Logger LOGGER = LoggerFactory.getLogger(MigratorConditonFilesMaker.class); private DataNode srcDn; private List newDnList; private String column; private String tableName; private AbstractPartitionAlgorithm alg; private String tempFileDir; private TableMigrateInfo tableInfo; private int newDnSize; private int pageSize; private Map files = new HashMap<>(); Map map = new HashMap<>();//存放节点发生变化的拆分字段字符串数据 key:dn索引 value 拆分字段值,以逗号分隔 public MigratorConditonFilesMaker(TableMigrateInfo tableInfo,DataNode srcDn,String tempFileDir, int pageSize){ this.tableInfo = tableInfo; this.tempFileDir = tempFileDir; this.srcDn = srcDn; this.newDnList = tableInfo.getNewDataNodes(); this.column = tableInfo.getColumn(); this.tableName = tableInfo.getTableName(); this.alg = tableInfo.getNewRuleAlgorithm(); this.newDnSize = newDnList.size(); this.pageSize = pageSize; } @Override public void run() { if(tableInfo.isError()) { return; } long[] count = new long[newDnSize]; int page=0; List> list=null; Connection con = null; try { con = DataMigratorUtil.getMysqlConnection(srcDn); //创建空的中间临时文件 createTempFiles(); //暂时只实现mysql的分页查询 list = DataMigratorUtil.executeQuery(con, "select " + column+ " from " + tableName + " limit ?,?", page++ * pageSize, pageSize); int total = 0; //该节点表总数据量 while (!CollectionUtil.isEmpty(list)) { if(tableInfo.isError()) { return; } flushData(false); for(int i=0,l=list.size();i sf=list.get(i); Object objFieldVal = sf.get(column); String filedVal = objFieldVal.toString(); if (objFieldVal instanceof String){ filedVal = "'"+filedVal+"'"; } Integer newIndex=alg.calculate(StringUtil.removeBackquote(objFieldVal.toString())); total++; DataNode newDn = newDnList.get(newIndex); if(!srcDn.equals(newDn)){ count[newIndex]++; map.get(newDn).append(filedVal+","); } } list = DataMigratorUtil.executeQuery(con, "select " + column + " from " + tableName + " limit ?,?", page++ * pageSize, pageSize); } flushData(true); statisticalData(total,count); } catch (Exception e) { //发生错误，终止此拆分表所有节点线程任务，记录错误信息，退出此拆分表迁移任务 String message = "["+tableInfo.getSchemaName()+":"+tableName+"] src dataNode: "+srcDn.getUrl()+ " prepare temp files is failed! this table's migrator will exit! "+e.getMessage(); tableInfo.setError(true); tableInfo.setErrMessage(message); System.out.println(message); LOGGER.error(message, e); }finally{ JdbcUtils.close(con); } } //创建中间临时文件 private void createTempFiles() throws IOException{ File parentFile = createDirIfNotExist(); for(DataNode dn:newDnList){ if(!srcDn.equals(dn)){ map.put(dn, new StringBuilder()); createTempFile(parentFile,dn); } } } //中间临时文件 格式: srcDnName-targetDnName.txt 中间文件存在的话会被清除 private void createTempFile(File parentFile, DataNode dn) throws IOException { File f = new File(parentFile,srcDn.getName()+"(old)"+"-"+dn.getName()+"(new).txt"); if(f.exists()){ f.delete(); } f.createNewFile(); files.put(dn, f); } //统计各节点数据迁移信息,并移除空文件 private void statisticalData(int total, long[] count){ tableInfo.getSize().addAndGet(total); List list = tableInfo.getDataNodesDetail(); List sizeList = new ArrayList<>(); for(int i=0;i0){ DataNodeMigrateInfo info =new DataNodeMigrateInfo(tableInfo,srcDn, targetDn, files.get(targetDn), c); list.add(info); }else{ File f = files.get(targetDn); if(f != null && f.exists()){ f.delete(); } files.remove(targetDn); } } Map map = tableInfo.getDnMigrateSize(); map.put(srcDn.getName()+"["+total+"]", sizeList.toString()); } //将迁移字段值写入中间文件,数据超过1024或者要求强制才写入，避免重复打开关闭写入文件 private void flushData(boolean isForce) throws IOException { for(DataNode dn:newDnList){ StringBuilder sb = map.get(dn); if(sb == null) { continue; } if((isForce || sb.toString().getBytes().length>1024) && sb.length()>0){ String s = sb.toString(); if(isForce){//最后一次将末尾的','截掉 s = s.substring(0, s.length()-1); } DataMigratorUtil.appendDataToFile(files.get(dn),s); sb = new StringBuilder(); map.put(dn, sb); } } } //创建中间临时文件父目录 private File createDirIfNotExist() { File f = new File(tempFileDir,tableInfo.getSchemaName()+"-"+tableName); if(!f.exists()){ f.mkdirs(); } return f; } } ================================================ FILE: src/main/java/io/mycat/util/dataMigrator/TableMigrateInfo.java ================================================ package io.mycat.util.dataMigrator; import java.sql.Connection; import java.sql.SQLException; import java.util.ArrayList; import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicLong; import com.alibaba.druid.util.JdbcUtils; import io.mycat.route.function.AbstractPartitionAlgorithm; /** * 表迁移信息，包括: * 表名、迁移前后的数据节点、表数据量、迁移前后数据分布对比 * @author haonan108 * */ public class TableMigrateInfo { private String schemaName; private String tableName; private List oldDataNodes; private List newDataNodes; private AtomicLong size = new AtomicLong(); private List dataNodesDetail = new ArrayList<>();//节点间数据迁移详细信息 private AbstractPartitionAlgorithm newRuleAlgorithm; private String column; private boolean isExpantion; //true:扩容 false：缩容 private volatile boolean isError; private StringBuffer errMessage = new StringBuffer(); private String tableStructure = ""; //记录建表信息，迁移后的节点表不存在的话自动建表 private Map dnMigrateSize; public TableMigrateInfo(String schemaName, String tableName, List oldDataNodes, List newDataNodes, AbstractPartitionAlgorithm newRuleAlgorithm, String column) { super(); this.schemaName = schemaName; this.tableName = tableName; this.oldDataNodes = oldDataNodes; this.newDataNodes = newDataNodes; this.newRuleAlgorithm = newRuleAlgorithm; this.column = column; if(newDataNodes.size()>oldDataNodes.size()){ isExpantion = true; }else{ isExpantion = false; } dnMigrateSize = new TreeMap<>(new Comparator() { @Override public int compare(String o1, String o2) { return o1.compareTo(o2); } }); } //读取表结构 public void setTableStructure() throws SQLException{ DataNode dn = this.getOldDataNodes().get(0); Connection con = null; try { con = DataMigratorUtil.getMysqlConnection(dn); List> list = DataMigratorUtil.executeQuery(con, "show create table "+tableName); Map m = list.get(0); String str = m.get("Create Table").toString(); str = str.replaceAll("CREATE TABLE", "Create Table if not exists"); setTableStructure(str); } catch (SQLException e) { throw e; }finally { JdbcUtils.close(con); } } //缩容后，找出被移除的节点 public List getRemovedDataNodes(){ List list = new ArrayList<>(); list.addAll(oldDataNodes); list.removeAll(newDataNodes); return list; } //扩容后，找出除旧节点以外新增加的节点 public List getNewAddDataNodes(){ List list = new ArrayList<>(); list.addAll(newDataNodes); list.removeAll(oldDataNodes); return list; } //对新增的节点创建表：create table if not exists public void createTableToNewDataNodes() throws SQLException{ if(this.isExpantion){ List newDataNodes = getNewAddDataNodes(); for(DataNode dn:newDataNodes){ DataMigratorUtil.createTable(dn, this.tableStructure); } } } //打印迁移信息 @SuppressWarnings({ "unchecked", "rawtypes" }) public void printMigrateInfo(){ Map map = new LinkedHashMap(); map.put("tableSize", size.get()+""); map.put("migrate before", oldDataNodes.toString()); map.put("migrate after", newDataNodes.toString()); map.put("rule function", newRuleAlgorithm.getClass().getSimpleName()); String title = getSchemaAndTableName()+" migrate info"; System.out.println(DataMigratorUtil.printMigrateInfo(title, map, "=")); } public void printMigrateSchedule(){ String title = getSchemaAndTableName()+" migrate schedule"; System.out.println(DataMigratorUtil.printMigrateInfo(title, dnMigrateSize, "->")); } /** * 是否为扩容，true：扩容，false：缩容 * @return */ public boolean isExpantion(){ return isExpantion; } public List getDataNodesDetail() { return dataNodesDetail; } public void setDataNodesDetail(List dataNodesDetail) { this.dataNodesDetail = dataNodesDetail; } public String getSchemaName() { return schemaName; } public void setSchemaName(String schemaName) { this.schemaName = schemaName; } public String getTableName() { return tableName; } public void setTableName(String tableName) { this.tableName = tableName; } public List getOldDataNodes() { return oldDataNodes; } public void setOldDataNodes(List oldDataNodes) { this.oldDataNodes = oldDataNodes; } public List getNewDataNodes() { return newDataNodes; } public void setNewDataNodes(List newDataNodes) { this.newDataNodes = newDataNodes; } public AbstractPartitionAlgorithm getNewRuleAlgorithm() { return newRuleAlgorithm; } public void setNewRuleAlgorithm(AbstractPartitionAlgorithm newRuleAlgorithm) { this.newRuleAlgorithm = newRuleAlgorithm; } public String getColumn() { return column; } public void setColumn(String column) { this.column = column; } public String getSchemaAndTableName(){ return "["+schemaName+":"+tableName+"]"; } public StringBuffer getErrMessage() { return errMessage; } public void setErrMessage(String errMessage) { this.errMessage = new StringBuffer(errMessage); } public AtomicLong getSize() { return size; } public void setSize(long size){ this.size = new AtomicLong(size); } public boolean isError() { return isError; } public void setError(boolean isError) { this.isError = isError; } public String getTableStructure() { return tableStructure; } public void setTableStructure(String tableStructure) { this.tableStructure = tableStructure; } public void setSize(AtomicLong size) { this.size = size; } public Map getDnMigrateSize() { return dnMigrateSize; } public void setDnMigrateSize(Map dnMigrateSize) { this.dnMigrateSize = dnMigrateSize; } } ================================================ FILE: src/main/java/io/mycat/util/dataMigrator/dataIOImpl/MysqlDataIO.java ================================================ package io.mycat.util.dataMigrator.dataIOImpl; import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.Charset; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.mycat.util.dataMigrator.DataIO; import io.mycat.util.dataMigrator.DataMigrator; import io.mycat.util.dataMigrator.DataMigratorUtil; import io.mycat.util.dataMigrator.DataNode; import io.mycat.util.dataMigrator.TableMigrateInfo; import io.mycat.util.exception.DataMigratorException; /** * mysql导入导出实现类 * @author haonan108 * */ public class MysqlDataIO implements DataIO{ private static final Logger LOGGER = LoggerFactory.getLogger(MysqlDataIO.class); private String mysqlBin; private int cmdLength; private String charset; private boolean gtidPurged = true; public MysqlDataIO(){ cmdLength = DataMigrator.margs.getCmdLength(); charset = DataMigrator.margs.getCharSet(); mysqlBin = DataMigrator.margs.getMysqlBin(); gtidPurged = DataMigrator.margs.isGtidPurged(); } public boolean isWindows() { try { return System.getProperties().getProperty("os.name", "WINDOWS").toUpperCase().indexOf("WINDOWS") != -1; }catch (Throwable e){ LOGGER.error("",e); return true; } } @Override public void importData(TableMigrateInfo table,DataNode dn,String tableName, File file) throws IOException, InterruptedException { String ip = dn.getIp(); int port = dn.getPort(); String user = dn.getUserName(); String pwd = dn.getPwd(); String db = dn.getDb(); // String loadData ="?mysql -h? -P? -u? -p? -D? --local-infile=1 -e \"load data local infile '?' replace into table ? CHARACTER SET '?' FIELDS TERMINATED BY ',' LINES TERMINATED BY '\\r\\n'\""; String loadData = "?mysql -h? -P? -u? -p? -D? -f --default-character-set=? -e \"source ?\""; loadData = DataMigratorUtil.paramsAssignment(loadData,"?",mysqlBin,ip,port,user,pwd,db,charset,file.getAbsolutePath()); LOGGER.info(table.getSchemaAndTableName()+" "+loadData); Process process = DataMigratorUtil.exeCmdByOs(loadData); //获取错误信息 InputStreamReader in = new InputStreamReader(process.getErrorStream(),isWindows()?"GBK": Charset.defaultCharset().name()); BufferedReader br = new BufferedReader(in); String errMessage = null; while ((errMessage = br.readLine()) != null) { if(errMessage.trim().toLowerCase().contains("err")){ System.out.println(errMessage+" -> "+loadData); throw new DataMigratorException(errMessage+" -> "+loadData); } } process.waitFor(); } @Override public File exportData(TableMigrateInfo table,DataNode dn, String tableName, File export, File condition) throws IOException, InterruptedException { String ip = dn.getIp(); int port = dn.getPort(); String user = dn.getUserName(); String pwd = dn.getPwd(); String db = dn.getDb(); // String mysqlDump = "?mysqldump -h? -P? -u? -p? ? ? --no-create-info --default-character-set=? " // + "--add-locks=false --tab='?' --fields-terminated-by=',' --lines-terminated-by='\\r\\n' --where='? in(?)'"; //由于mysqldump导出csv格式文件只能导出到本地，暂时替换成导出insert形式的文件 String mysqlDump = "?mysqldump -h? -P? -u? -p? ? ? --compact --no-create-info --default-character-set=? --add-locks=false --where=\"? in (#)\" --result-file=\"?\" "; if (gtidPurged){ mysqlDump += " --set-gtid-purged=ON "; }else { mysqlDump += " --set-gtid-purged=OFF "; } String fileName = condition.getName(); File exportPath = new File(export,fileName.substring(0, fileName.indexOf(".txt"))); if(!exportPath.exists()){ exportPath.mkdirs(); } File exportFile = new File(exportPath,tableName.toLowerCase()+".txt"); //拼接mysqldump命令，不拼接where条件：--where=id in(?) mysqlDump = DataMigratorUtil.paramsAssignment(mysqlDump,"?",mysqlBin,ip,port,user,pwd,db,tableName,charset,table.getColumn(),exportFile); String data = ""; //由于操作系统对命令行长度的限制，导出过程被拆分成多次，最后需要将导出的数据文件合并 File mergedFile = new File(exportPath,tableName.toLowerCase()+".sql"); if(!mergedFile.exists()){ mergedFile.createNewFile(); } int offset = 0; while((data=DataMigratorUtil.readData(condition,offset,cmdLength)).length()>0){ offset += data.getBytes().length; if(data.startsWith(",")){ data = data.substring(1, data.length()); } if(data.endsWith(",")){ data = data.substring(0,data.length()-1); } String mysqlDumpCmd = DataMigratorUtil.paramsAssignment(mysqlDump,"#",data); LOGGER.info(table.getSchemaAndTableName()+mysqlDump); LOGGER.debug(table.getSchemaAndTableName()+" "+mysqlDumpCmd); Process process = DataMigratorUtil.exeCmdByOs(mysqlDumpCmd); //获取错误信息 InputStreamReader in = new InputStreamReader(process.getErrorStream()); BufferedReader br = new BufferedReader(in); String errMessage = null; while ((errMessage = br.readLine()) != null) { if(errMessage.trim().toLowerCase().contains("err")){ System.out.println(errMessage+" -> "+mysqlDump); throw new DataMigratorException(errMessage+" -> "+mysqlDump); }else{ LOGGER.info(table.getSchemaAndTableName()+mysqlDump+" exe info:"+errMessage); } } process.waitFor(); //合并文件 DataMigratorUtil.mergeFiles(mergedFile, exportFile); if(exportFile.exists()){ exportFile.delete(); } } return mergedFile; } } ================================================ FILE: src/main/java/io/mycat/util/exception/DataMigratorException.java ================================================ package io.mycat.util.exception; /** * * @author haonan108 * */ public class DataMigratorException extends RuntimeException{ private static final long serialVersionUID = -6706826479467595980L; public DataMigratorException() { super(); } public DataMigratorException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { super(message, cause, enableSuppression, writableStackTrace); } public DataMigratorException(String message, Throwable cause) { super(message, cause); } public DataMigratorException(String message) { super(message); } public DataMigratorException(Throwable cause) { super(cause); } } ================================================ FILE: src/main/java/io/mycat/util/exception/ErrorPacketException.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util.exception; /** * @author mycat */ public class ErrorPacketException extends RuntimeException { private static final long serialVersionUID = -2692093550257870555L; public ErrorPacketException() { super(); } public ErrorPacketException(String message, Throwable cause) { super(message, cause); } public ErrorPacketException(String message) { super(message); } public ErrorPacketException(Throwable cause) { super(cause); } } ================================================ FILE: src/main/java/io/mycat/util/exception/HeartbeatException.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util.exception; /** * @author mycat */ public class HeartbeatException extends RuntimeException { private static final long serialVersionUID = 7639414445868741580L; public HeartbeatException() { super(); } public HeartbeatException(String message, Throwable cause) { super(message, cause); } public HeartbeatException(String message) { super(message); } public HeartbeatException(Throwable cause) { super(cause); } } ================================================ FILE: src/main/java/io/mycat/util/exception/MurmurHashException.java ================================================ package io.mycat.util.exception; public class MurmurHashException extends RuntimeException{ /** * */ private static final long serialVersionUID = -8040964553491203562L; public MurmurHashException() { super(); } public MurmurHashException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { super(message, cause, enableSuppression, writableStackTrace); } public MurmurHashException(String message, Throwable cause) { super(message, cause); } public MurmurHashException(String message) { super(message); } public MurmurHashException(Throwable cause) { super(cause); } } ================================================ FILE: src/main/java/io/mycat/util/exception/RehashException.java ================================================ package io.mycat.util.exception; public class RehashException extends RuntimeException{ /** * */ private static final long serialVersionUID = 7562724429239862825L; public RehashException() { super(); } public RehashException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { super(message, cause, enableSuppression, writableStackTrace); } public RehashException(String message, Throwable cause) { super(message, cause); } public RehashException(String message) { super(message); } public RehashException(Throwable cause) { super(cause); } } ================================================ FILE: src/main/java/io/mycat/util/exception/UnknownCharsetException.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util.exception; /** * 未知字符集异常 * * @author mycat */ public class UnknownCharsetException extends RuntimeException { private static final long serialVersionUID = 552833416065882969L; public UnknownCharsetException() { super(); } public UnknownCharsetException(String message, Throwable cause) { super(message, cause); } public UnknownCharsetException(String message) { super(message); } public UnknownCharsetException(Throwable cause) { super(cause); } } ================================================ FILE: src/main/java/io/mycat/util/exception/UnknownDataNodeException.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util.exception; /** * @author mycat */ public class UnknownDataNodeException extends RuntimeException { private static final long serialVersionUID = -3752985849571697432L; public UnknownDataNodeException() { super(); } public UnknownDataNodeException(String message, Throwable cause) { super(message, cause); } public UnknownDataNodeException(String message) { super(message); } public UnknownDataNodeException(Throwable cause) { super(cause); } } ================================================ FILE: src/main/java/io/mycat/util/exception/UnknownPacketException.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util.exception; /** * 未知数据包异常 * * @author mycat */ public class UnknownPacketException extends RuntimeException { private static final long serialVersionUID = 3152986441780514147L; public UnknownPacketException() { super(); } public UnknownPacketException(String message, Throwable cause) { super(message, cause); } public UnknownPacketException(String message) { super(message); } public UnknownPacketException(Throwable cause) { super(cause); } } ================================================ FILE: src/main/java/io/mycat/util/exception/UnknownTxIsolationException.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util.exception; /** * 未知事物隔离级别异常 * * @author mycat */ public class UnknownTxIsolationException extends RuntimeException { private static final long serialVersionUID = -3911059999308980358L; public UnknownTxIsolationException() { super(); } public UnknownTxIsolationException(String message, Throwable cause) { super(message, cause); } public UnknownTxIsolationException(String message) { super(message); } public UnknownTxIsolationException(Throwable cause) { super(cause); } } ================================================ FILE: src/main/java/io/mycat/util/rehasher/HashType.java ================================================ package io.mycat.util.rehasher; public enum HashType { MURMUR,MOD; } ================================================ FILE: src/main/java/io/mycat/util/rehasher/RehashCmdArgs.java ================================================ package io.mycat.util.rehasher; import java.io.BufferedReader; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.util.ArrayList; import java.util.List; import io.mycat.util.StringUtil; import io.mycat.util.cmd.CmdArgs; public class RehashCmdArgs { public static final String JDBC_DRIVER="jdbcDriver"; public static final String JDBC_URL="jdbcUrl"; public static final String HOST="host"; public static final String USER="user"; public static final String DATABASE="database"; public static final String PASSWORD="password"; public static final String TABLES_FILE="tablesFile"; public static final String SHARDING_FIELD="shardingField"; public static final String REHASH_HOSTS_FILE="rehashHostsFile"; public static final String HASH_TYPE="hashType"; public static final String SEED="seed"; public static final String VIRTUAL_BUCKET_TIMES="virtualBucketTimes"; public static final String WEIGHT_MAP_FILE="weightMapFile"; public static final String REHASH_NODE_DIR="rehashNodeDir"; private CmdArgs cmdArgs; public RehashCmdArgs(String[] args){ cmdArgs=CmdArgs.getInstance(args); } public String getString(String name){ return cmdArgs.getString(name); } public String getJdbcDriver(){ return getString(JDBC_DRIVER); } public String getJdbcUrl(){ return getString(JDBC_URL); } /** * including host and port, which format is host:port * @return */ public String getHost(){ return getString(HOST); } public String getHostName(){ String host=getHost(); return host.substring(0,host.indexOf(':')); } public int getHostPort(){ String host=getHost(); return Integer.parseInt(host.substring(host.indexOf(':')+1)); } public String getDatabase(){ return getString(DATABASE); } public String getHostWithDatabase(){ return getHost()+'/'+getDatabase(); } public String getUser(){ return getString(USER); } public String getPassword(){ return getString(PASSWORD); } public String getTablesFile(){ return getString(TABLES_FILE); } public String[] getTables() throws IOException{ return readStringArrayFromFile(getTablesFile()); } public String getShardingField(){ return getString(SHARDING_FIELD); } public String getRehashHostsFile(){ return getString(REHASH_HOSTS_FILE); } public String[] getRehashHosts() throws IOException{ return readStringArrayFromFile(getRehashHostsFile()); } public HashType getHashType(){ return HashType.valueOf(getString(HASH_TYPE).toUpperCase()); } public int getMurmurHashSeed(){ return getIntWithDefaultValue(SEED, 0); } public int getMurmurHashVirtualBucketTimes(){ return getIntWithDefaultValue(VIRTUAL_BUCKET_TIMES, 160); } public String getMurmurWeightMapFile(){ return getString(WEIGHT_MAP_FILE); } public String getRehashNodeDir(){ return getString(REHASH_NODE_DIR); } private int getIntWithDefaultValue(String name,int defaultValue){ String val=getString(name); if(StringUtil.isEmpty(val)){ return defaultValue; }else{ return Integer.parseInt(val); } } private String[] readStringArrayFromFile(String file) throws IOException{ BufferedReader br=null; try{ List tableList=new ArrayList<>(); br=new BufferedReader(new InputStreamReader(new FileInputStream(file),"utf8")); String table=null; while((table=br.readLine())!=null){ tableList.add(table); } return tableList.toArray(new String[tableList.size()]); }finally{ if(br!=null){ br.close(); } } } } ================================================ FILE: src/main/java/io/mycat/util/rehasher/RehashLauncher.java ================================================ package io.mycat.util.rehasher; import io.mycat.util.StringUtil; import java.io.File; import java.io.IOException; import java.io.PrintStream; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.druid.pool.DruidDataSource; import com.alibaba.druid.util.JdbcUtils; import io.mycat.route.function.AbstractPartitionAlgorithm; import io.mycat.route.function.PartitionByMod; import io.mycat.route.function.PartitionByMurmurHash; import io.mycat.util.CollectionUtil; import io.mycat.util.exception.RehashException; /** * 本工具依赖druid，Mycat已经包含druid，druid配置请查阅相关文档。相关参数请看RehashCmdArgs * @author wujingrun * */ public class RehashLauncher { private final class RehashRunner implements Runnable { private final File output; private final String table; private RehashRunner(File output, String table) { this.output = output; this.table = table; } public void run(){ int pageSize=500; int page=0; List> list=null; int total=0; int rehashed=0; String hostWithDatabase=args.getHostWithDatabase(); PrintStream ps=null; try { ps=new PrintStream(output); list = JdbcUtils.executeQuery(dataSource, "select " + args.getShardingField() + " from " + table + " limit ?,?", page++ * pageSize, pageSize); while (!CollectionUtil.isEmpty(list)) { for(int i=0,l=list.size();i sf=list.get(i); Integer hash=alg.calculate(StringUtil.removeBackquote(sf.get(args.getShardingField()).toString())); String host=rehashHosts[hash]; total++; if(host.equals(hostWithDatabase)){ rehashed++; } ps.println(sf+"=>"+host); } list = JdbcUtils.executeQuery(dataSource, "select " + args.getShardingField() + " from " + table + " limit ?,?", page++ * pageSize, pageSize); } ps.println("rehashed ratio:"+(((double)rehashed)/total)); } catch (Exception e) { throw new RehashException(e); }finally{ if(ps!=null){ ps.close(); } latch.countDown(); } } } private RehashCmdArgs args; private DruidDataSource dataSource; private String[] rehashHosts; private AbstractPartitionAlgorithm alg; private ExecutorService executor; private CountDownLatch latch; private static final Logger LOGGER = LoggerFactory.getLogger(RehashLauncher.class); private RehashLauncher(String[] args) throws IOException{ this.args=new RehashCmdArgs(args); initDataSource(); this.rehashHosts=this.args.getRehashHosts(); initHashAlg(); executor=Executors.newCachedThreadPool(); } private void initHashAlg() throws IOException{ if (HashType.MURMUR.equals(args.getHashType())) { alg=new PartitionByMurmurHash(); PartitionByMurmurHash murmur=(PartitionByMurmurHash)alg; murmur.setCount(rehashHosts.length); murmur.setSeed(args.getMurmurHashSeed()); murmur.setVirtualBucketTimes(args.getMurmurHashVirtualBucketTimes()); murmur.setWeightMapFile(args.getMurmurWeightMapFile()); murmur.init(); } else if (HashType.MOD.equals(args.getHashType())) { alg=new PartitionByMod(); PartitionByMod mod=(PartitionByMod)alg; mod.setCount(rehashHosts.length); mod.init(); } } private void initDataSource(){ dataSource=new DruidDataSource(); dataSource.setAsyncCloseConnectionEnable(true); dataSource.setBreakAfterAcquireFailure(true); dataSource.setDefaultAutoCommit(true); dataSource.setDefaultReadOnly(true); dataSource.setDriverClassName(args.getJdbcDriver()); dataSource.setEnable(true); dataSource.setPassword(args.getPassword()); dataSource.setTestOnBorrow(true); dataSource.setTestOnReturn(true); dataSource.setTestWhileIdle(true); dataSource.setUrl(args.getJdbcUrl()); dataSource.setUsername(args.getUser()); } private RehashLauncher execute() throws IOException{ final String[] tables=args.getTables(); final File outputDir=new File(args.getRehashNodeDir()); if(!outputDir.exists()){ outputDir.mkdirs(); }else if(outputDir.isFile()){ throw new IllegalArgumentException("rehashNodeDir must be a directory"); }else if(outputDir.canWrite()){ throw new IllegalArgumentException("rehashNodeDir must be writable"); } latch=new CountDownLatch(tables.length); for(int i=0,l=tables.length;i ================================================ FILE: src/main/resources/index_to_charset.properties ================================================ 1=big5 2=latin2 3=dec8 4=cp850 5=latin1 6=hp8 7=koi8r 8=latin1 9=latin2 10=swe7 11=ascii 12=ujis 13=sjis 14=cp1251 15=latin1 16=hebrew 18=tis620 19=euckr 20=latin7 21=latin2 22=koi8u 23=cp1251 24=gb2312 25=greek 26=cp1250 27=latin2 28=gbk 29=cp1257 30=latin5 31=latin1 32=armscii8 33=utf8 34=cp1250 35=ucs2 36=cp866 37=keybcs2 38=macce 39=macroman 40=cp852 41=latin7 42=latin7 43=macce 44=cp1250 45=utf8mb4 46=utf8mb4 47=latin1 48=latin1 49=latin1 50=cp1251 51=cp1251 52=cp1251 53=macroman 54=utf16 55=utf16 56=utf16le 57=cp1256 58=cp1257 59=cp1257 60=utf32 61=utf32 62=utf16le 63=binary 64=armscii8 65=ascii 66=cp1250 67=cp1256 68=cp866 69=dec8 70=greek 71=hebrew 72=hp8 73=keybcs2 74=koi8r 75=koi8u 77=latin2 78=latin5 79=latin7 80=cp850 81=cp852 82=swe7 83=utf8 84=big5 85=euckr 86=gb2312 87=gbk 88=sjis 89=tis620 90=ucs2 91=ujis 92=geostd8 93=geostd8 94=latin1 95=cp932 96=cp932 97=eucjpms 98=eucjpms 99=cp1250 101=utf16 102=utf16 103=utf16 104=utf16 105=utf16 106=utf16 107=utf16 108=utf16 109=utf16 110=utf16 111=utf16 112=utf16 113=utf16 114=utf16 115=utf16 116=utf16 117=utf16 118=utf16 119=utf16 120=utf16 121=utf16 122=utf16 123=utf16 124=utf16 128=ucs2 129=ucs2 130=ucs2 131=ucs2 132=ucs2 133=ucs2 134=ucs2 135=ucs2 136=ucs2 137=ucs2 138=ucs2 139=ucs2 140=ucs2 141=ucs2 142=ucs2 143=ucs2 144=ucs2 145=ucs2 146=ucs2 147=ucs2 148=ucs2 149=ucs2 150=ucs2 151=ucs2 159=ucs2 160=utf32 161=utf32 162=utf32 163=utf32 164=utf32 165=utf32 166=utf32 167=utf32 168=utf32 169=utf32 170=utf32 171=utf32 172=utf32 173=utf32 174=utf32 175=utf32 176=utf32 177=utf32 178=utf32 179=utf32 180=utf32 181=utf32 182=utf32 183=utf32 192=utf8 193=utf8 194=utf8 195=utf8 196=utf8 197=utf8 198=utf8 199=utf8 200=utf8 201=utf8 202=utf8 203=utf8 204=utf8 205=utf8 206=utf8 207=utf8 208=utf8 209=utf8 210=utf8 211=utf8 212=utf8 213=utf8 214=utf8 215=utf8 223=utf8 224=utf8mb4 225=utf8mb4 226=utf8mb4 227=utf8mb4 228=utf8mb4 229=utf8mb4 230=utf8mb4 231=utf8mb4 232=utf8mb4 233=utf8mb4 234=utf8mb4 235=utf8mb4 236=utf8mb4 237=utf8mb4 238=utf8mb4 239=utf8mb4 240=utf8mb4 241=utf8mb4 242=utf8mb4 243=utf8mb4 244=utf8mb4 245=utf8mb4 246=utf8mb4 247=utf8mb4 248=utf8mb4 249=utf8mb4 250=utf8mb4 251=utf8mb4 252=utf8mb4 253=utf8mb4 254=utf8mb4 255=utf8mb4 ================================================ FILE: src/main/resources/log4j2.xml ================================================ %d{yyyy-MM-dd HH:mm:ss.SSS} %5p [%t] (%l) - %m%n ================================================ FILE: src/main/resources/migrateTables.properties ================================================ #schema1=tb1,tb2,... #schema2=all（写all或者不写将对此schema下拆分节点变化的拆分表全部进行重新路由） #... #sample #TESTDB=travelrecord,company,goods ================================================ FILE: src/main/resources/myid.properties ================================================ loadZk=false zkURL=127.0.0.1:2181 clusterId=mycat-cluster-1 myid=mycat_fz_01 clusterSize=3 clusterNodes=mycat_fz_01,mycat_fz_02,mycat_fz_04 #server booster ; booster install on db same server,will reset all minCon to 2 type=server boosterDataHosts=dataHost1 ================================================ FILE: src/main/resources/partition-hash-int.txt ================================================ 10000=0 10010=1 ================================================ FILE: src/main/resources/partition-range-mod.txt ================================================ # range start-end ,data node group size 0-200M=5 200M1-400M=1 400M1-600M=4 600M1-800M=4 800M1-1000M=6 ================================================ FILE: src/main/resources/rule.dtd ================================================ ================================================ FILE: src/main/resources/rule.xml ================================================ id func1 createTime partbyday user_id func1 sharding_id hash-int id rang-long id mod-long id murmur id crc32slot create_time partbymonth calldate latestMonth id rang-mod id jump-consistent-hash 0 2 160 2 partition-hash-int.txt autopartition-long.txt 3 8 128 24 yyyy-MM-dd 2015-01-01 yyyy-MM-dd 0 2014-01-01 2014-01-31 10 partition-range-mod.txt 3 ================================================ FILE: src/main/resources/schema.dtd ================================================ ================================================ FILE: src/main/resources/schema.xml ================================================

select user() ================================================ FILE: src/main/resources/sequence_conf.properties ================================================ #default global sequence GLOBAL.HISIDS= GLOBAL.MINID=10001 GLOBAL.MAXID=20000 GLOBAL.CURID=10000 # self define sequence COMPANY.HISIDS= COMPANY.MINID=1001 COMPANY.MAXID=2000 COMPANY.CURID=1000 CUSTOMER.HISIDS= CUSTOMER.MINID=1001 CUSTOMER.MAXID=2000 CUSTOMER.CURID=1000 ORDER.HISIDS= ORDER.MINID=1001 ORDER.MAXID=2000 ORDER.CURID=1000 HOTNEWS.HISIDS= HOTNEWS.MINID=1001 HOTNEWS.MAXID=2000 HOTNEWS.CURID=1000 ================================================ FILE: src/main/resources/sequence_db_conf.properties ================================================ #sequence stored in datanode GLOBAL=dn1 COMPANY=dn1 CUSTOMER=dn1 ORDERS=dn1 ================================================ FILE: src/main/resources/sequence_distributed_conf.properties ================================================ INSTANCEID=01 CLUSTERID=01 ================================================ FILE: src/main/resources/sequence_http_conf.properties ================================================ url = http://localhost:8067/ ================================================ FILE: src/main/resources/sequence_time_conf.properties ================================================ #sequence depend on TIME WORKID=01 DATAACENTERID=01 ================================================ FILE: src/main/resources/server.dtd ================================================ ================================================ FILE: src/main/resources/server.xml ================================================ 0 0 1 1 0 0 300 1 (?:(\s*next\s+value\s+for\s*MYCATSEQ_(\w+))(,|\)|\s)*)+ false io.mycat.route.sequence.handler.HttpIncrSequenceHandler 0 0 0 64k 1k 0 384m false true 0 2048 123456 TESTDB TESTDB user TESTDB true TESTDB ================================================ FILE: src/main/resources/sharding-by-enum.txt ================================================ 10000=0 10010=1 ================================================ FILE: src/main/resources/zkconf/auto-sharding-long.txt ================================================ 2000001-4000000=1 0-2000000=0 4000001-8000000=2 ================================================ FILE: src/main/resources/zkconf/auto-sharding-rang-mod.txt ================================================ 800M1-1000M=6 600M1-800M=4 200M1-400M=1 0-200M=5 400M1-600M=4 ================================================ FILE: src/main/resources/zkconf/autopartition-long.txt ================================================ # range start-end ,data node index # K=1000,M=10000. 0-500M=0 500M-1000M=1 1000M-1500M=2 ================================================ FILE: src/main/resources/zkconf/cacheservice.properties ================================================ #used for mycat cache service conf factory.encache=io.mycat.cache.impl.EnchachePooFactory #key is pool name ,value is type,max size, expire seconds pool.SQLRouteCache=encache,10000,1800 pool.ER_SQL2PARENTID=encache,1000,1800 layedpool.TableID2DataNodeCache=encache,10000,18000 layedpool.TableID2DataNodeCache.TESTDB_ORDERS=50000,18000 ================================================ FILE: src/main/resources/zkconf/ehcache.xml ================================================ ================================================ FILE: src/main/resources/zkconf/index_to_charset.properties ================================================ 1=big5 2=latin2 3=dec8 4=cp850 5=latin1 6=hp8 7=koi8r 8=latin1 9=latin2 10=swe7 11=ascii 12=ujis 13=sjis 14=cp1251 15=latin1 16=hebrew 18=tis620 19=euckr 20=latin7 21=latin2 22=koi8u 23=cp1251 24=gb2312 25=greek 26=cp1250 27=latin2 28=gbk 29=cp1257 30=latin5 31=latin1 32=armscii8 33=utf8 34=cp1250 35=ucs2 36=cp866 37=keybcs2 38=macce 39=macroman 40=cp852 41=latin7 42=latin7 43=macce 44=cp1250 45=utf8mb4 46=utf8mb4 47=latin1 48=latin1 49=latin1 50=cp1251 51=cp1251 52=cp1251 53=macroman 54=utf16 55=utf16 56=utf16le 57=cp1256 58=cp1257 59=cp1257 60=utf32 61=utf32 62=utf16le 63=binary 64=armscii8 65=ascii 66=cp1250 67=cp1256 68=cp866 69=dec8 70=greek 71=hebrew 72=hp8 73=keybcs2 74=koi8r 75=koi8u 77=latin2 78=latin5 79=latin7 80=cp850 81=cp852 82=swe7 83=utf8 84=big5 85=euckr 86=gb2312 87=gbk 88=sjis 89=tis620 90=ucs2 91=ujis 92=geostd8 93=geostd8 94=latin1 95=cp932 96=cp932 97=eucjpms 98=eucjpms 99=cp1250 101=utf16 102=utf16 103=utf16 104=utf16 105=utf16 106=utf16 107=utf16 108=utf16 109=utf16 110=utf16 111=utf16 112=utf16 113=utf16 114=utf16 115=utf16 116=utf16 117=utf16 118=utf16 119=utf16 120=utf16 121=utf16 122=utf16 123=utf16 124=utf16 128=ucs2 129=ucs2 130=ucs2 131=ucs2 132=ucs2 133=ucs2 134=ucs2 135=ucs2 136=ucs2 137=ucs2 138=ucs2 139=ucs2 140=ucs2 141=ucs2 142=ucs2 143=ucs2 144=ucs2 145=ucs2 146=ucs2 147=ucs2 148=ucs2 149=ucs2 150=ucs2 151=ucs2 159=ucs2 160=utf32 161=utf32 162=utf32 163=utf32 164=utf32 165=utf32 166=utf32 167=utf32 168=utf32 169=utf32 170=utf32 171=utf32 172=utf32 173=utf32 174=utf32 175=utf32 176=utf32 177=utf32 178=utf32 179=utf32 180=utf32 181=utf32 182=utf32 183=utf32 192=utf8 193=utf8 194=utf8 195=utf8 196=utf8 197=utf8 198=utf8 199=utf8 200=utf8 201=utf8 202=utf8 203=utf8 204=utf8 205=utf8 206=utf8 207=utf8 208=utf8 209=utf8 210=utf8 211=utf8 212=utf8 213=utf8 214=utf8 215=utf8 223=utf8 224=utf8mb4 225=utf8mb4 226=utf8mb4 227=utf8mb4 228=utf8mb4 229=utf8mb4 230=utf8mb4 231=utf8mb4 232=utf8mb4 233=utf8mb4 234=utf8mb4 235=utf8mb4 236=utf8mb4 237=utf8mb4 238=utf8mb4 239=utf8mb4 240=utf8mb4 241=utf8mb4 242=utf8mb4 243=utf8mb4 244=utf8mb4 245=utf8mb4 246=utf8mb4 247=utf8mb4 ================================================ FILE: src/main/resources/zkconf/partition-hash-int.txt ================================================ 10000=0 10010=1 ================================================ FILE: src/main/resources/zkconf/partition-range-mod.txt ================================================ # range start-end ,data node group size 0-200M=5 200M1-400M=1 400M1-600M=4 600M1-800M=4 800M1-1000M=6 ================================================ FILE: src/main/resources/zkconf/rule.xml ================================================ id func1 user_id func1 sharding_id hash-int id rang-long id mod-long id murmur create_date partbymonth calldate latestMonth id crc32slot id rang-mod id jump-consistent-hash 0 2 160 partition-hash-int.txt autopartition-long.txt 3 8 128 24 yyyy-MM-dd 2015-01-01 partition-range-mod.txt 3 ================================================ FILE: src/main/resources/zkconf/schema.xml ================================================
select user() ================================================ FILE: src/main/resources/zkconf/sequence_conf.properties ================================================ #default global sequence GLOBAL.HISIDS= GLOBAL.MINID=10001 GLOBAL.MAXID=20000 GLOBAL.CURID=10000 # self define sequence COMPANY.HISIDS= COMPANY.MINID=1001 COMPANY.MAXID=2000 COMPANY.CURID=1000 CUSTOMER.HISIDS= CUSTOMER.MINID=1001 CUSTOMER.MAXID=2000 CUSTOMER.CURID=1000 ORDER.HISIDS= ORDER.MINID=1001 ORDER.MAXID=2000 ORDER.CURID=1000 HOTNEWS.HISIDS= HOTNEWS.MINID=1001 HOTNEWS.MAXID=2000 HOTNEWS.CURID=1000 ================================================ FILE: src/main/resources/zkconf/sequence_db_conf.properties ================================================ #sequence stored in datanode GLOBAL=dn1 COMPANY=dn1 CUSTOMER=dn1 ORDERS=dn1 ================================================ FILE: src/main/resources/zkconf/sequence_distributed_conf-mycat_fz_01.properties ================================================ INSTANCEID=02 CLUSTERID=02 ================================================ FILE: src/main/resources/zkconf/sequence_distributed_conf.properties ================================================ INSTANCEID=01 CLUSTERID=01 ================================================ FILE: src/main/resources/zkconf/sequence_time_conf-mycat_fz_01.properties ================================================ #sequence depend on TIME WORKID=03 DATAACENTERID=03 ================================================ FILE: src/main/resources/zkconf/sequence_time_conf.properties ================================================ #sequence depend on TIME WORKID=01 DATAACENTERID=01 ================================================ FILE: src/main/resources/zkconf/server-mycat_fz_01.xml ================================================ 1 0 druidparser 2 0 0 1 1m 1k 0 389m digdeep TESTDB user TESTDB true ================================================ FILE: src/main/resources/zkconf/server.xml ================================================ 1 0 druidparser 2 0 0 1 1m 1k 0 384m digdeep TESTDB user TESTDB true ================================================ FILE: src/main/resources/zkconf/sharding-by-enum.txt ================================================ 10000=0 10010=1 ================================================ FILE: src/main/resources/zkdownload/auto-sharding-long.txt ================================================ 2000001-4000000=1 0-2000000=0 4000001-8000000=2 ================================================ FILE: src/test/java/demo/catlets/MyHellowJoin.java ================================================ package demo.catlets; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import io.mycat.cache.LayerCachePool; import io.mycat.catlets.Catlet; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.net.mysql.RowDataPacket; import io.mycat.server.ServerConnection; import io.mycat.sqlengine.AllJobFinishedListener; import io.mycat.sqlengine.EngineCtx; import io.mycat.sqlengine.SQLJobHandler; import io.mycat.util.ByteUtil; import io.mycat.util.ResultSetUtil; public class MyHellowJoin implements Catlet { public void processSQL(String sql, EngineCtx ctx) { DirectDBJoinHandler joinHandler = new DirectDBJoinHandler(ctx); String[] dataNodes = { "dn1", "dn2", "dn3" }; ctx.executeNativeSQLSequnceJob(dataNodes, sql, joinHandler); ctx.setAllJobFinishedListener(new AllJobFinishedListener() { @Override public void onAllJobFinished(EngineCtx ctx) { ctx.writeEof(); } }); } @Override public void route(SystemConfig sysConfig, SchemaConfig schema, int sqlType, String realSQL, String charset, ServerConnection sc, LayerCachePool cachePool) { } } class DirectDBJoinHandler implements SQLJobHandler { private List fields; private final EngineCtx ctx; public DirectDBJoinHandler(EngineCtx ctx) { super(); this.ctx = ctx; } private Map rows = new ConcurrentHashMap(); private ConcurrentLinkedQueue ids = new ConcurrentLinkedQueue(); @Override public void onHeader(String dataNode, byte[] header, List fields) { this.fields = fields; } private void createQryJob(int batchSize) { int count = 0; Map batchRows = new ConcurrentHashMap(); String theId = null; StringBuilder sb = new StringBuilder().append('('); while ((theId = ids.poll()) != null) { batchRows.put(theId, rows.remove(theId)); sb.append(theId).append(','); if (count++ > batchSize) { break; } } if (count == 0) { return; } sb.deleteCharAt(sb.length() - 1).append(')'); String querySQL = "select b.id, b.title from hotnews b where id in " + sb; ctx.executeNativeSQLParallJob(new String[] { "dn1", "dn2", "dn3" }, querySQL, new MyRowOutPutDataHandler(fields, ctx, batchRows)); } @Override public boolean onRowData(String dataNode, byte[] rowData) { String id = ResultSetUtil.getColumnValAsString(rowData, fields, 0); // 放入结果集 rows.put(id, rowData); ids.offer(id); int batchSize = 999; // 满1000条，发送一个查询请求 if (ids.size() > batchSize) { createQryJob(batchSize); } return false; } @Override public void finished(String dataNode, boolean failed, String errorMsg) { if (!failed) { createQryJob(Integer.MAX_VALUE); } // no more jobs ctx.endJobInput(); } } class MyRowOutPutDataHandler implements SQLJobHandler { private final List afields; private List bfields; private final EngineCtx ctx; private final Map arows; public MyRowOutPutDataHandler(List afields, EngineCtx ctx, Map arows) { super(); this.afields = afields; this.ctx = ctx; this.arows = arows; } @Override public void onHeader(String dataNode, byte[] header, List bfields) { this.bfields=bfields; ctx.writeHeader(afields, bfields); } @Override public boolean onRowData(String dataNode, byte[] rowData) { RowDataPacket rowDataPkg = ResultSetUtil.parseRowData(rowData, bfields); // 获取Id字段， String id = ByteUtil.getString(rowDataPkg.fieldValues.get(0)); byte[] bname = rowDataPkg.fieldValues.get(1); // 查找ID对应的A表的记录 byte[] arow = arows.remove(id); rowDataPkg = ResultSetUtil.parseRowData(arow, afields); // 设置b.name 字段 rowDataPkg.add(bname); ctx.writeRow(rowDataPkg); // EngineCtx.LOGGER.info("out put row "); return false; } @Override public void finished(String dataNode, boolean failed, String errorMsg) { } } ================================================ FILE: src/test/java/demo/test/ByteArrayToHexArray.java ================================================ package demo.test; import java.util.Arrays; public class ByteArrayToHexArray { public static String buqi(String hexStr){ if(hexStr.length() == 1){ return "0"+hexStr; } return hexStr; } public static void main(String[] args) { // byte [] array = {19, 0, 0, 8, 4, 49, 53, 53, 57, 1, 49, 4, 49, 49, 49, 53, 5, 52, 52, 51, 49, 55, -5}; byte [] array = {4, 0, 0, 0, 2, 100, 98, 50}; for(byte b : array){ int a=b&0xff; System.out.print( buqi(Integer.toHexString(b)) + ":"); } System.out.println(""); for(byte b : array){ int a=b&0xff; System.out.print( new String(new byte[]{b}) + " "); } } } ================================================ FILE: src/test/java/demo/test/TestClass1.java ================================================ package demo.test; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; /** * @author mycat * */ public class TestClass1 { public static void main( String args[] ) throws SQLException , ClassNotFoundException { String jdbcdriver="com.mysql.jdbc.Driver"; String jdbcurl="jdbc:mysql://127.0.0.1:8066/TESTDB?useUnicode=true&characterEncoding=utf-8"; String username="test"; String password="test"; System.out.println("开始连接mysql:"+jdbcurl); Class.forName(jdbcdriver); Connection c = DriverManager.getConnection(jdbcurl,username,password); Statement st = c.createStatement(); print( "test jdbc " , st.executeQuery("select count(*) from travelrecord ")); System.out.println("OK......"); } static void print( String name , ResultSet res ) throws SQLException { System.out.println( name); ResultSetMetaData meta=res.getMetaData(); //System.out.println( "\t"+res.getRow()+"条记录"); String str=""; for(int i=1;i<=meta.getColumnCount();i++){ str+=meta.getColumnName(i)+" "; //System.out.println( meta.getColumnName(i)+" "); } System.out.println("\t"+str); str=""; while ( res.next() ){ for(int i=1;i<=meta.getColumnCount();i++){ str+= res.getString(i)+" "; } System.out.println("\t"+str); str=""; } } } ================================================ FILE: src/test/java/io/mycat/BufferPerformanceMain.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat; import java.nio.ByteBuffer; /** * @author mycat */ public class BufferPerformanceMain { public void getAllocate() { ByteBuffer buffer = ByteBuffer.allocate(4096); byte[] b = new byte[1024]; int count = 1000000; System.currentTimeMillis(); long t1 = System.currentTimeMillis(); for (int i = 0; i < count; i++) { buffer.position(0); buffer.get(b); } long t2 = System.currentTimeMillis(); System.out.println("take time:" + (t2 - t1) + " ms.(Get:allocate)"); } public void getAllocateDirect() { ByteBuffer buffer = ByteBuffer.allocateDirect(4096); byte[] b = new byte[1024]; int count = 1000000; System.currentTimeMillis(); long t1 = System.currentTimeMillis(); for (int i = 0; i < count; i++) { buffer.position(0); buffer.get(b); } long t2 = System.currentTimeMillis(); System.out.println("take time:" + (t2 - t1) + " ms.(Get:allocateDirect)"); } public void putAllocate() { ByteBuffer buffer = ByteBuffer.allocate(4096); byte[] b = new byte[1024]; int count = 1000000; System.currentTimeMillis(); long t1 = System.currentTimeMillis(); for (int i = 0; i < count; i++) { buffer.position(0); buffer.put(b); } long t2 = System.currentTimeMillis(); System.out.println("take time:" + (t2 - t1) + " ms.(Put:allocate)"); } public void putAllocateDirect() { ByteBuffer buffer = ByteBuffer.allocateDirect(4096); byte[] b = new byte[1024]; int count = 1000000; System.currentTimeMillis(); long t1 = System.currentTimeMillis(); for (int i = 0; i < count; i++) { buffer.position(0); buffer.put(b); } long t2 = System.currentTimeMillis(); System.out.println("take time:" + (t2 - t1) + " ms.(Put:allocateDirect)"); } public void copyArrayDirect() { ByteBuffer buffer = ByteBuffer.allocateDirect(4096); while (buffer.hasRemaining()) { buffer.put((byte) 1); } byte[] b = new byte[1024]; int count = 10000000; System.currentTimeMillis(); long t1 = System.currentTimeMillis(); for (int i = 0; i < count; i++) { buffer.position(0); buffer.get(b, 0, b.length); } long t2 = System.currentTimeMillis(); System.out.println("take time:" + (t2 - t1) + " ms.(testCopyArrayDirect)"); } public void copyArray() { ByteBuffer buffer = ByteBuffer.allocate(4096); while (buffer.hasRemaining()) { buffer.put((byte) 1); } byte[] b = new byte[1024]; int count = 10000000; System.currentTimeMillis(); long t1 = System.currentTimeMillis(); for (int i = 0; i < count; i++) { buffer.position(0); buffer.get(b, 0, b.length); } long t2 = System.currentTimeMillis(); System.out.println("take time:" + (t2 - t1) + " ms.(testCopyArray)"); } public static void main(String[] args) { } } ================================================ FILE: src/test/java/io/mycat/ConfigInitializerTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat; import org.junit.Test; import io.mycat.config.ConfigInitializer; /** * @author mycat */ public class ConfigInitializerTest { @Test public void testConfigLoader() { new ConfigInitializer(true); } } ================================================ FILE: src/test/java/io/mycat/EchoBioServer.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.ServerSocket; import java.net.Socket; /** * @author mycat */ public class EchoBioServer implements Runnable { private static final byte[] FIRST_BYTES = "Welcome to MyCat Server.".getBytes(); private final ServerSocket serverSocket; public EchoBioServer(int port) throws IOException { serverSocket = new ServerSocket(port); } @Override public void run() { while (true) { try { Socket socket = serverSocket.accept(); new Thread(new BioConnection(socket)).start(); } catch (IOException e) { e.printStackTrace(); } } } private class BioConnection implements Runnable { private Socket socket; private InputStream input; private OutputStream output; private byte[] readBuffer; private byte[] writeBuffer; private BioConnection(Socket socket) throws IOException { this.socket = socket; this.input = socket.getInputStream(); this.output = socket.getOutputStream(); this.readBuffer = new byte[4096]; this.writeBuffer = new byte[4096]; } @Override public void run() { try { output.write(FIRST_BYTES); output.flush(); while (true) { int got = input.read(readBuffer); output.write(writeBuffer, 0, got); // output.flush(); } } catch (IOException e) { e.printStackTrace(); if (socket != null) { try { socket.close(); } catch (IOException e1) { e1.printStackTrace(); } } } } } public static void main(String[] args) throws Exception { new Thread(new EchoBioServer(8066)).start(); } } ================================================ FILE: src/test/java/io/mycat/ExecutorTestMain.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicLong; import io.mycat.util.ExecutorUtil; /** * @author mycat */ public class ExecutorTestMain { public static void main(String[] args) { final AtomicLong count = new AtomicLong(0L); final ThreadPoolExecutor executor = ExecutorUtil.create("TestExecutor", 5); new Thread() { @Override public void run() { for (;;) { long c = count.get(); try { Thread.sleep(5000L); } catch (InterruptedException e) { e.printStackTrace(); } System.out.println("count:" + (count.get() - c) / 5); System.out.println("active:" + executor.getActiveCount()); System.out.println("queue:" + executor.getQueue().size()); System.out.println("============================"); } } }.start(); new Thread() { @Override public void run() { for (;;) { executor.execute(new Runnable() { @Override public void run() { count.incrementAndGet(); } }); } } }.start(); new Thread() { @Override public void run() { for (;;) { executor.execute(new Runnable() { @Override public void run() { count.incrementAndGet(); } }); } } }.start(); } } ================================================ FILE: src/test/java/io/mycat/SimpleCachePool.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import io.mycat.cache.CacheStatic; import io.mycat.cache.LayerCachePool; public class SimpleCachePool implements LayerCachePool { private HashMap cacheMap; public SimpleCachePool() { long MAX_CACHE_SIZE = getMaxSize(); float factor = 0.75f; int capacity = (int)Math.ceil(MAX_CACHE_SIZE / factor) + 1; cacheMap = new LinkedHashMap(capacity, factor, true) { @Override protected boolean removeEldestEntry(Map.Entry eldest) { return size() > MAX_CACHE_SIZE; }}; } @Override public void putIfAbsent(Object key, Object value) { cacheMap.put(key, value); } @Override public Object get(Object key) { return cacheMap.get(key); } @Override public void clearCache() { cacheMap.clear(); } @Override public CacheStatic getCacheStatic() { return null; } @Override public void putIfAbsent(String primaryKey, Object secondKey, Object value) { putIfAbsent(primaryKey+"_"+secondKey,value); } @Override public Object get(String primaryKey, Object secondKey) { return get(primaryKey+"_"+secondKey); } @Override public Map getAllCacheStatic() { return null; } @Override public void clearCache(String cacheName) { if (cacheName != null) { cacheMap.remove(cacheName); } } @Override public long getMaxSize() { return 100; } }; ================================================ FILE: src/test/java/io/mycat/VolatileTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat; import org.junit.Test; /** * @author mycat */ public class VolatileTest { @Test public void testNoop() { } static class VolatileObject { volatile Object object = new Object(); } public static void main(String[] args) { final VolatileObject vo = new VolatileObject(); // set new Thread() { @Override public void run() { System.out.print("set..."); while (true) { vo.object = new Object(); } } }.start(); // get new Thread() { @Override public void run() { System.out.print("get..."); while (true) { Object oo = vo.object; oo.toString(); } } }.start(); } } ================================================ FILE: src/test/java/io/mycat/backend/jdbc/mongodb/MongoClientPropertyHelperTest.java ================================================ package io.mycat.backend.jdbc.mongodb; import org.junit.Test; import java.util.Properties; /** * @author liuxinsi * @mail akalxs@gmail.com */ public class MongoClientPropertyHelperTest { @Test public void testFormatProperties() { Properties pro = new Properties(); pro.put("authMechanism", "SCRAM-SHA-1"); pro.put("readPreference", "nearest"); pro.put("maxPoolSize", 10); pro.put("ssl", true); String options = MongoClientPropertyHelper.formatProperties(pro); System.out.println(options); } } ================================================ FILE: src/test/java/io/mycat/backend/jdbc/mongodb/MongoEmbeddedObjectProcessorTest.java ================================================ package io.mycat.backend.jdbc.mongodb; import com.google.common.collect.Lists; import com.mongodb.BasicDBObject; import org.bson.types.ObjectId; import org.junit.Assert; import org.junit.Test; import java.util.Date; import java.util.List; import java.util.Set; /** * @author liuxinsi * @mail akalxs@gmail.com */ public class MongoEmbeddedObjectProcessorTest { @Test public void testValueMapperWithObjectId() { String id = "5978776b8d69f75e091067ed"; Object obj = MongoEmbeddedObjectProcessor.valueMapper("_id", id, ObjectId.class); if (!(obj instanceof ObjectId)) { Assert.fail("not objectId"); } } @Test public void testValueMapperWithEmbeddedObject() { BasicDBObject dbObj = new BasicDBObject(); dbObj.put("str", "t1"); dbObj.put("inte", 1); dbObj.put("date", new Date()); dbObj.put("lon", 100L); dbObj.put("bool", true); dbObj.put("strs", new String[]{"a", "b", "c"}); dbObj.put("intes", new Integer[]{1, 2, 3}); dbObj.put("bytes", "ttt".getBytes()); dbObj.put("b", "a".getBytes()[0]); Object o = MongoEmbeddedObjectProcessor.valueMapper("embObj", dbObj, TestObject.class); if (!(o instanceof TestObject)) { Assert.fail("not emb obj"); } } @Test public void testValueMapperWithDeepEmbeddedObject() { BasicDBObject dbObj = new BasicDBObject(); dbObj.put("str", "t1"); dbObj.put("inte", 1); dbObj.put("date", new Date()); dbObj.put("lon", 100L); dbObj.put("bool", true); dbObj.put("strs", new String[]{"a", "b", "c"}); dbObj.put("intes", new Integer[]{1, 2, 3}); dbObj.put("bytes", "ttt".getBytes()); dbObj.put("b", "a".getBytes()[0]); BasicDBObject embedObj = new BasicDBObject(); embedObj.put("embeddedStr", "e1"); BasicDBObject deepEmbedObj1 = new BasicDBObject(); deepEmbedObj1.put("str", "aaa"); BasicDBObject deepEmbedObj2 = new BasicDBObject(); deepEmbedObj2.put("str", "bbb"); embedObj.put("testObjectList", Lists.newArrayList(deepEmbedObj1, deepEmbedObj2)); dbObj.put("embeddedObject", embedObj); Object o = MongoEmbeddedObjectProcessor.valueMapper("embObj", dbObj, TestObject.class); if (!(o instanceof TestObject)) { Assert.fail("not emb obj"); } System.out.println(o); } } class TestObject { private ObjectId _id; private String str; private Integer inte; private Date date; private Long lon; private Boolean bool; private String[] strs; private Integer[] intes; private byte[] bytes; private Byte b; private EmbeddedObject embeddedObject; public ObjectId get_id() { return _id; } public void set_id(ObjectId _id) { this._id = _id; } public String getStr() { return str; } public void setStr(String str) { this.str = str; } public Integer getInte() { return inte; } public void setInte(Integer inte) { this.inte = inte; } public Date getDate() { return date; } public void setDate(Date date) { this.date = date; } public Long getLon() { return lon; } public void setLon(Long lon) { this.lon = lon; } public Boolean getBool() { return bool; } public void setBool(Boolean bool) { this.bool = bool; } public String[] getStrs() { return strs; } public void setStrs(String[] strs) { this.strs = strs; } public Integer[] getIntes() { return intes; } public void setIntes(Integer[] intes) { this.intes = intes; } public byte[] getBytes() { return bytes; } public void setBytes(byte[] bytes) { this.bytes = bytes; } public Byte getB() { return b; } public void setB(Byte b) { this.b = b; } public EmbeddedObject getEmbeddedObject() { return embeddedObject; } public void setEmbeddedObject(EmbeddedObject embeddedObject) { this.embeddedObject = embeddedObject; } } class EmbeddedObject { private String embeddedStr; private List testObjectList; private Set someCodeSet; public String getEmbeddedStr() { return embeddedStr; } public void setEmbeddedStr(String embeddedStr) { this.embeddedStr = embeddedStr; } public List getTestObjectList() { return testObjectList; } public void setTestObjectList(List testObjectList) { this.testObjectList = testObjectList; } public Set getSomeCodeSet() { return someCodeSet; } public void setSomeCodeSet(Set someCodeSet) { this.someCodeSet = someCodeSet; } } ================================================ FILE: src/test/java/io/mycat/buffer/TestByteBufferArena.java ================================================ package io.mycat.buffer; import junit.framework.Assert; import org.junit.Test; import sun.nio.ch.DirectBuffer; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; /** * 仿照Netty的思路，针对MyCat内存缓冲策略优化 * 测试ByteBufferArena * * @author Hash Zhang * @version 1.0 * @time 17:19 2016/5/17 * @see @https://github.com/netty/netty */ public class TestByteBufferArena { int pageSize = 256; int chunkSize = 1024 * 8; int chunkCount = 8*128; @Test public void testAllocate() { int allocTimes = 1024 ; ByteBufferArena byteBufferArena = new ByteBufferArena(chunkSize,pageSize,chunkCount,8); long start = System.currentTimeMillis(); for (int i = 0; i < allocTimes; i++) { // System.out.println("allocate "+i); // long start=System.nanoTime(); int size = (i % 1024) + 1 ; ByteBuffer byteBufer = byteBufferArena.allocate(size); ByteBuffer byteBufer2 = byteBufferArena.allocate(size); ByteBuffer byteBufer3 = byteBufferArena.allocate(size); // System.out.println("alloc "+size+" usage "+(System.nanoTime()-start)); // start=System.nanoTime(); byteBufferArena.recycle(byteBufer); byteBufferArena.recycle(byteBufer3); // System.out.println("recycle usage "+(System.nanoTime()-start)); } long used = (System.currentTimeMillis() - start); System.out.println("ByteBufferArena total used time " + used + " avg speed " + allocTimes / used); } @Test public void testAllocateDirect() { int pageSize = 1024 ; int allocTimes = 100; DirectByteBufferPool pool = new DirectByteBufferPool(pageSize, (short) 256, (short) 8,0); long start = System.currentTimeMillis(); for (int i = 0; i < allocTimes; i++) { //System.out.println("allocate "+i); //long start=System.nanoTime(); int size = (i % 1024) + 1 ; ByteBuffer byteBufer = pool.allocate(size); ByteBuffer byteBufer2 = pool.allocate(size); ByteBuffer byteBufer3 = pool.allocate(size); //System.out.println("alloc "+size+" usage "+(System.nanoTime()-start)); //start=System.nanoTime(); pool.recycle(byteBufer); pool.recycle(byteBufer3); //System.out.println("recycle usage "+(System.nanoTime()-start)); } long used = (System.currentTimeMillis() - start); // System.out.println("DirectByteBufferPool total used time " + used + " avg speed " + allocTimes / used); } @Test public void testExpansion(){ ByteBufferArena byteBufferArena = new ByteBufferArena(1024,8,1,8); for (int i = 0; i < 1 ; i++) { ByteBuffer byteBufer = byteBufferArena.allocate(256); ByteBuffer byteBufer2 = byteBufferArena.allocate(256); ByteBuffer byteBufer3 = byteBufferArena.allocate(256); byteBufferArena.recycle(byteBufer); } } @Test public void testAllocateWithDifferentAddress() { int size = 256; int pageSize = size * 4; int allocTimes = 8; ByteBufferArena byteBufferArena = new ByteBufferArena(256*4,256,2,8); Map buffs = new HashMap(8); ByteBuffer byteBuffer = null; DirectBuffer directBuffer = null; ByteBuffer temp = null; long address; boolean failure = false; for (int i = 0; i < allocTimes; i++) { byteBuffer = byteBufferArena.allocate(size); if (byteBuffer == null) { Assert.fail("Should have enough memory"); } directBuffer = (DirectBuffer) byteBuffer; address = directBuffer.address(); System.out.println(address); temp = buffs.get(address); buffs.put(address, byteBuffer); if (null != temp) { failure = true; break; } } for (ByteBuffer buff : buffs.values()) { byteBufferArena.recycle(buff); } if (failure == true) { Assert.fail("Allocate with same address"); } } @Test public void testAllocateNullWhenOutOfMemory() { int size = 256; int pageSize = size * 4; int allocTimes = 9; ByteBufferArena pool = new ByteBufferArena(256*4,256,2,8);; long start = System.currentTimeMillis(); ByteBuffer byteBuffer = null; List buffs = new ArrayList(); int i = 0; for (; i < allocTimes; i++) { byteBuffer = pool.allocate(size); if (byteBuffer == null) { break; } buffs.add(byteBuffer); } for (ByteBuffer buff : buffs) { pool.recycle(buff); } } } ================================================ FILE: src/test/java/io/mycat/buffer/TestDirectByteBufferPool.java ================================================ package io.mycat.buffer; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import junit.framework.Assert; import org.junit.Test; import sun.nio.ch.DirectBuffer; public class TestDirectByteBufferPool { @Test public void testAllocate() { int pageSize = 1024 ; int allocTimes = 1024; DirectByteBufferPool pool = new DirectByteBufferPool(pageSize, (short) 256, (short) 8,0); long start = System.currentTimeMillis(); for (int i = 0; i < allocTimes; i++) { //System.out.println("allocate "+i); //long start=System.nanoTime(); int size = (i % 1024) + 1 ; ByteBuffer byteBufer = pool.allocate(size); ByteBuffer byteBufer2 = pool.allocate(size); ByteBuffer byteBufer3 = pool.allocate(size); //System.out.println("alloc "+size+" usage "+(System.nanoTime()-start)); //start=System.nanoTime(); pool.recycle(byteBufer); pool.recycle(byteBufer3); //System.out.println("recycle usage "+(System.nanoTime()-start)); } long used = (System.currentTimeMillis() - start); System.out.println("total used time " + used + " avg speed " + allocTimes / used); } @Test public void testAllocateWithDifferentAddress() { int size = 256; int pageSize = size * 4; int allocTimes = 8; DirectByteBufferPool pool = new DirectByteBufferPool(pageSize, (short) 256, (short) 2,0); Map buffs = new HashMap(8); ByteBuffer byteBuffer = null; DirectBuffer directBuffer = null; ByteBuffer temp = null; long address; boolean failure = false; for (int i = 0; i < allocTimes; i++) { byteBuffer = pool.allocate(size); if (byteBuffer == null) { Assert.fail("Should have enough memory"); } directBuffer = (DirectBuffer) byteBuffer; address = directBuffer.address(); System.out.println(address); temp = buffs.get(address); buffs.put(address, byteBuffer); if (null != temp) { failure = true; break; } } for (ByteBuffer buff : buffs.values()) { pool.recycle(buff); } if (failure == true) { Assert.fail("Allocate with same address"); } } @Test public void testAllocateNullWhenOutOfMemory() { int size = 256; int pageSize = size * 4; int allocTimes = 9; DirectByteBufferPool pool = new DirectByteBufferPool(pageSize, (short) 256, (short) 2,0); long start = System.currentTimeMillis(); ByteBuffer byteBuffer = null; List buffs = new ArrayList(); int i = 0; for (; i < allocTimes; i++) { byteBuffer = pool.allocate(size); if (byteBuffer == null||!(byteBuffer instanceof DirectBuffer) ) { break; } buffs.add(byteBuffer); } for (ByteBuffer buff : buffs) { pool.recycle(buff); } Assert.assertEquals("Should out of memory when i = " + 8, i, 8); } @Test public void testAllocateSign() { int size = 256; int pageSize = size * 4; int allocTimes = 9; DirectByteBufferPool pool = new DirectByteBufferPool(pageSize, (short) 256, (short) 2,0); long start = System.currentTimeMillis(); ByteBuffer byteBuffer = null; List buffs = new ArrayList(); int i = 0; for (; i < allocTimes; i++) { byteBuffer = pool.allocate(size); if (byteBuffer == null||!(byteBuffer instanceof DirectBuffer) ) { break; } buffs.add(byteBuffer); } for (ByteBuffer buff : buffs) { pool.recycle(buff); } Assert.assertEquals("Should out of memory when i = " + 8, i, 8); } @Test public void testExpandBuffer(){ int size = 512; int pageSize = 1024*1024; int allocTimes = 9; DirectByteBufferPool pool = new DirectByteBufferPool(pageSize, (short) 512, (short) 64,0); ByteBuffer byteBuffer = pool.allocate(1024); String str = "DirectByteBufferPool pool = new DirectByteBufferPool(pageSize, (short) 256, (short) 8)"; ByteBuffer newByteBuffer = null; int i = 0; while (i<10){ if(byteBuffer.remaining() freeMaps = new ConcurrentHashMap<>(); final MyCatMemoryAllocator memoryAllocator = new MyCatMemoryAllocator(Runtime.getRuntime().availableProcessors()*2); @Test public void testMemAlloc(){ for (int i = 0; i <10000/**20000000*/; i++) { ByteBuffer byteBuffer = getBuffer(8194); byteBuffer.put("helll world".getBytes()); byteBuffer.flip(); byte [] src= new byte[byteBuffer.remaining()]; byteBuffer.get(src); Assert.assertEquals("helll world",new String(src)); free(byteBuffer); } } public ByteBuffer getBuffer(int len) { ByteBuf byteBuf = memoryAllocator.directBuffer(len); ByteBuffer byteBuffer = byteBuf.nioBuffer(0,len); freeMaps.put(PlatformDependent.directBufferAddress(byteBuffer),byteBuf); return byteBuffer; } public void free(ByteBuffer byteBuffer) { ByteBuf byteBuf1 = freeMaps.get(PlatformDependent.directBufferAddress(byteBuffer)); byteBuf1.release(); Assert.assertEquals(0,byteBuf1.refCnt()); } public static String getString(ByteBuffer buffer) { Charset charset = null; CharsetDecoder decoder = null; CharBuffer charBuffer = null; try { charset = Charset.forName("UTF-8"); decoder = charset.newDecoder(); charBuffer = decoder.decode(buffer.asReadOnlyBuffer()); return charBuffer.toString(); } catch (Exception ex) { ex.printStackTrace(); return "error"; } } public static ByteBuffer getByteBuffer(String str) { return ByteBuffer.wrap(str.getBytes()); } } ================================================ FILE: src/test/java/io/mycat/cache/DefaultLayedCachePoolTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.cache; import junit.framework.Assert; import org.junit.Test; import io.mycat.cache.CacheStatic; import io.mycat.cache.DefaultLayedCachePool; import io.mycat.cache.impl.EnchachePooFactory; public class DefaultLayedCachePoolTest { private static DefaultLayedCachePool layedCachePool; static { layedCachePool=new DefaultLayedCachePool("defaultLayedPool",new EnchachePooFactory(),1000,1); } @Test public void testBasic() { layedCachePool.putIfAbsent("2", "dn2"); layedCachePool.putIfAbsent("1", "dn1"); layedCachePool.putIfAbsent("company", 1, "dn1"); layedCachePool.putIfAbsent("company", 2, "dn2"); layedCachePool.putIfAbsent("goods", "1", "dn1"); layedCachePool.putIfAbsent("goods", "2", "dn2"); Assert.assertEquals("dn2", layedCachePool.get("2")); Assert.assertEquals("dn1", layedCachePool.get("1")); Assert.assertEquals(null, layedCachePool.get("3")); Assert.assertEquals("dn1", layedCachePool.get("company", 1)); Assert.assertEquals("dn2", layedCachePool.get("company", 2)); Assert.assertEquals(null, layedCachePool.get("company", 3)); Assert.assertEquals("dn1", layedCachePool.get("goods", "1")); Assert.assertEquals("dn2", layedCachePool.get("goods", "2")); Assert.assertEquals(null, layedCachePool.get("goods", 3)); CacheStatic statics = layedCachePool.getCacheStatic(); Assert.assertEquals(statics.getItemSize(), 6); Assert.assertEquals(statics.getPutTimes(), 6); Assert.assertEquals(statics.getAccessTimes(), 9); Assert.assertEquals(statics.getHitTimes(), 6); Assert.assertTrue(statics.getLastAccesTime() > 0); Assert.assertTrue(statics.getLastPutTime() > 0); Assert.assertTrue(statics.getLastAccesTime() > 0); // wait expire try { Thread.sleep(2000); } catch (InterruptedException e) { } Assert.assertEquals(null, layedCachePool.get("2")); Assert.assertEquals(null, layedCachePool.get("1")); Assert.assertEquals(null, layedCachePool.get("goods", "2")); Assert.assertEquals(null, layedCachePool.get("company", 2)); } } ================================================ FILE: src/test/java/io/mycat/cache/EnCachePoolTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.cache; import junit.framework.Assert; import net.sf.ehcache.Cache; import net.sf.ehcache.CacheManager; import net.sf.ehcache.config.CacheConfiguration; import net.sf.ehcache.config.MemoryUnit; import org.junit.Test; import io.mycat.cache.CacheStatic; import io.mycat.cache.impl.EnchachePool; public class EnCachePoolTest { private static EnchachePool enCachePool; static { CacheConfiguration cacheConf = new CacheConfiguration(); cacheConf.setName("testcache"); cacheConf.maxBytesLocalHeap(50,MemoryUnit.MEGABYTES).timeToIdleSeconds(2); Cache cache=new Cache(cacheConf); CacheManager.create().addCache(cache); enCachePool = new EnchachePool(cacheConf.getName(),cache,50*10000); } @Test public void testBasic() { enCachePool.putIfAbsent("2", "dn2"); enCachePool.putIfAbsent("1", "dn1"); Assert.assertEquals("dn2", enCachePool.get("2")); Assert.assertEquals("dn1", enCachePool.get("1")); Assert.assertEquals(null, enCachePool.get("3")); CacheStatic statics = enCachePool.getCacheStatic(); Assert.assertEquals(statics.getItemSize(), 2); Assert.assertEquals(statics.getPutTimes(), 2); Assert.assertEquals(statics.getAccessTimes(), 3); Assert.assertEquals(statics.getHitTimes(), 2); Assert.assertTrue(statics.getLastAccesTime() > 0); Assert.assertTrue(statics.getLastPutTime() > 0); Assert.assertTrue(statics.getLastAccesTime() > 0); // wait expire try { Thread.sleep(4000); } catch (InterruptedException e) { } Assert.assertEquals(null, enCachePool.get("2")); Assert.assertEquals(null, enCachePool.get("1")); } } ================================================ FILE: src/test/java/io/mycat/cache/TestCachePoolPerformance.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.cache; import io.mycat.cache.CachePool; import io.mycat.cache.CacheStatic; import io.mycat.cache.impl.EnchachePool; import io.mycat.cache.impl.MapDBCachePooFactory; /** * test cache performance ,for encache test set VM param -server -Xms1100M -Xmx1100M * for mapdb set vm param -server -Xms100M -Xmx100M -XX:MaxPermSize=1G */ import net.sf.ehcache.Cache; import net.sf.ehcache.CacheManager; import net.sf.ehcache.config.CacheConfiguration; import net.sf.ehcache.config.MemoryUnit; public class TestCachePoolPerformance { private CachePool pool; private int maxCacheCount = 100 * 10000; public static CachePool createEnCachePool() { CacheConfiguration cacheConf = new CacheConfiguration(); cacheConf.setName("testcache"); cacheConf.maxBytesLocalHeap(400, MemoryUnit.MEGABYTES) .timeToIdleSeconds(3600); Cache cache = new Cache(cacheConf); CacheManager.create().addCache(cache); EnchachePool enCachePool = new EnchachePool(cacheConf.getName(),cache,400*10000); return enCachePool; } public static CachePool createMapDBCachePool() { MapDBCachePooFactory fact = new MapDBCachePooFactory(); return fact.createCachePool("mapdbcache", 100 * 10000, 3600); } public void test() { testSwarm(); testInsertSpeed(); testSelectSpeed(); } private void testSwarm() { System.out.println("prepare ........"); for (int i = 0; i < 100000; i++) { pool.putIfAbsent(i % 100, "dn1"); } for (int i = 0; i < 100000; i++) { pool.get(i % 100); } pool.clearCache(); } private void testSelectSpeed() { System.out.println("test select speed for " + this.pool + " count:" + this.maxCacheCount); long startTime = System.currentTimeMillis(); for (int i = 0; i < maxCacheCount; i++) { pool.get(i + ""); } double used = (System.currentTimeMillis() - startTime) / 1000.0; CacheStatic statics = pool.getCacheStatic(); System.out.println("used time:" + used + " tps:" + maxCacheCount / used + " cache hit:" + 100 * statics.getHitTimes() / statics.getAccessTimes()); } private void GC() { for (int i = 0; i < 5; i++) { System.gc(); } } private void testInsertSpeed() { this.GC(); long freeMem = Runtime.getRuntime().freeMemory(); System.out.println("test insert speed for " + this.pool + " with insert count:" + this.maxCacheCount); long start = System.currentTimeMillis(); for (int i = 0; i < maxCacheCount; i++) { try { pool.putIfAbsent(i + "", "dn" + i % 100); } catch (Error e) { System.out.println("insert " + i + " error"); e.printStackTrace(); break; } } long used = (System.currentTimeMillis() - start) / 1000; long count = pool.getCacheStatic().getItemSize(); this.GC(); long usedMem = freeMem - Runtime.getRuntime().freeMemory(); System.out.println(" cache size is " + count + " ,all in cache :" + (count == maxCacheCount) + " ,used time:" + used + " ,tps:" + count / used + " used memory:" + usedMem / 1024 / 1024 + "M"); } public static void main(String[] args) { if (args.length < 1) { System.out .println("usage : \r\n cache: 1 for encache 2 for mapdb\r\n"); return; } TestCachePoolPerformance tester = new TestCachePoolPerformance(); int cacheType = Integer.parseInt(args[0]); if (cacheType == 1) { tester.pool = createEnCachePool(); tester.test(); } else if (cacheType == 2) { tester.pool = createMapDBCachePool(); tester.test(); } else { System.out.println("not valid input "); } } } ================================================ FILE: src/test/java/io/mycat/classload/TestDynClassLoad.java ================================================ package io.mycat.classload; import org.junit.Assert; import org.junit.Test; import io.mycat.config.classloader.DynaClassLoader; public class TestDynClassLoad { @Test public void testLoadClass() throws Exception { String path=this.getClass().getResource("/").getPath(); String clsName="demo.test.TestClass1"; System.out.println("class load path "+path); DynaClassLoader loader =new DynaClassLoader(path,1); Object obj=loader.getInstanceofClass(clsName); Assert.assertEquals(obj.getClass().getSimpleName(),"TestClass1"); Assert.assertEquals(true,loader.getInstanceofClass(clsName)==obj); } } ================================================ FILE: src/test/java/io/mycat/config/ConfigTest.java ================================================ package io.mycat.config; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import org.junit.Test; import io.mycat.backend.datasource.PhysicalDBPool; import io.mycat.backend.datasource.PhysicalDatasource; import io.mycat.backend.jdbc.JDBCDatasource; import io.mycat.backend.mysql.nio.MySQLDataSource; import io.mycat.config.loader.ConfigLoader; import io.mycat.config.loader.xml.XMLConfigLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.DBHostConfig; import io.mycat.config.model.DataHostConfig; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.config.model.TableConfig; import io.mycat.config.model.UserConfig; import io.mycat.config.util.ConfigException; import junit.framework.Assert; public class ConfigTest { private SystemConfig system; private final Map users; private Map schemas; private Map dataHosts; public ConfigTest() { String schemaFile = "/config/schema.xml"; String ruleFile = "/config/rule.xml"; XMLSchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); XMLConfigLoader configLoader = new XMLConfigLoader(schemaLoader); this.system = configLoader.getSystemConfig(); this.users = configLoader.getUserConfigs(); this.schemas = configLoader.getSchemaConfigs(); this.dataHosts = initDataHosts(configLoader); } /** * 测试 临时读可用 配置 */ @Test public void testTempReadHostAvailable() { PhysicalDBPool pool = this.dataHosts.get("localhost2"); DataHostConfig hostConfig = pool.getSource().getHostConfig(); Assert.assertTrue( hostConfig.isTempReadHostAvailable() == true ); } /** * 测试 用户服务降级 拒连 配置 */ @Test public void testReadUserBenchmark() { UserConfig userConfig = this.users.get("test"); int benchmark = userConfig.getBenchmark(); Assert.assertTrue( benchmark == 11111 ); } /** * 测试 读服务的 权重 * * @throws Exception */ @Test public void testReadHostWeight() throws Exception { ArrayList okSources = new ArrayList(); PhysicalDBPool pool = this.dataHosts.get("localhost2"); okSources.addAll(pool.getAllDataSources()); PhysicalDatasource source = pool.randomSelect( okSources ); Assert.assertTrue( source != null ); } /** * 测试 动态日期表 * * @throws Exception */ @Test public void testDynamicYYYYMMTable() throws Exception { SchemaConfig sc = this.schemas.get("dbtest1"); Map tbm = sc.getTables(); Assert.assertTrue( tbm.size() == 32); } private Map initDataHosts(ConfigLoader configLoader) { Map nodeConfs = configLoader.getDataHosts(); Map nodes = new HashMap( nodeConfs.size()); for (DataHostConfig conf : nodeConfs.values()) { PhysicalDBPool pool = getPhysicalDBPool(conf, configLoader); nodes.put(pool.getHostName(), pool); } return nodes; } private PhysicalDatasource[] createDataSource(DataHostConfig conf, String hostName, String dbType, String dbDriver, DBHostConfig[] nodes, boolean isRead) { PhysicalDatasource[] dataSources = new PhysicalDatasource[nodes.length]; if (dbType.equals("mysql") && dbDriver.equals("native")) { for (int i = 0; i < nodes.length; i++) { nodes[i].setIdleTimeout(system.getIdleTimeout()); MySQLDataSource ds = new MySQLDataSource(nodes[i], conf, isRead); dataSources[i] = ds; } } else if(dbDriver.equals("jdbc")) { for (int i = 0; i < nodes.length; i++) { nodes[i].setIdleTimeout(system.getIdleTimeout()); JDBCDatasource ds = new JDBCDatasource(nodes[i], conf, isRead); dataSources[i] = ds; } } else { throw new ConfigException("not supported yet !" + hostName); } return dataSources; } private PhysicalDBPool getPhysicalDBPool(DataHostConfig conf, ConfigLoader configLoader) { String name = conf.getName(); String dbType = conf.getDbType(); String dbDriver = conf.getDbDriver(); PhysicalDatasource[] writeSources = createDataSource(conf, name, dbType, dbDriver, conf.getWriteHosts(), false); Map readHostsMap = conf.getReadHosts(); Map readSourcesMap = new HashMap( readHostsMap.size()); for (Map.Entry entry : readHostsMap.entrySet()) { PhysicalDatasource[] readSources = createDataSource(conf, name, dbType, dbDriver, entry.getValue(), true); readSourcesMap.put(entry.getKey(), readSources); } PhysicalDBPool pool = new PhysicalDBPool(conf.getName(),conf, writeSources, readSourcesMap, conf.getBalance(), conf.getWriteType()); return pool; } } ================================================ FILE: src/test/java/io/mycat/heartbeat/HeartbeatConfigForTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.heartbeat; import org.junit.Test; /** * @author mycat */ public class HeartbeatConfigForTest { @Test public void testNoop() { } // public static DataNodeConfig[] getOfferNodes(int offset, int length) { // DataNodeConfig[] nodes = new DataNodeConfig[length]; // for (int i = 0; i < length; i++) { // DataNodeConfig node = new DataNodeConfig(); // node.name = "offer" + (offset + i); // node.activedIndex = 0; // node.dataSource = getOfferDataSource(node.name); // nodes[i] = node; // } // return nodes; // } // // private static DataSourceConfig[] getOfferDataSource(String schema) { // DataSourceConfig ds1 = new DataSourceConfig(); // ds1.host = "10.20.132.17"; // ds1.port = 3306; // ds1.schema = schema; // ds1.user = "offer"; // ds1.password = "offer"; // ds1.statement = "update xdual set x=now()"; // // DataSourceConfig ds2 = new DataSourceConfig(); // ds2.host = "10.20.153.177"; // ds2.port = 3316; // ds2.schema = schema; // ds2.user = "offer"; // ds2.password = "offer"; // ds2.statement = "update xdual set x=now()"; // // return new DataSourceConfig[] { ds1, ds2 }; // } // // public static DataNodeConfig getNodeErrorConfig() { // // 数据源1（IP错误） // DataSourceConfig ds1 = new DataSourceConfig(); // ds1.host = "100.20.132.17"; // ds1.port = 3306; // ds1.schema = "offer1"; // ds1.user = "offer"; // ds1.password = "offer"; // ds1.statement = "update xdual set x=now()"; // // // 数据源2（端口错误） // DataSourceConfig ds2 = new DataSourceConfig(); // ds2.host = "10.20.132.17"; // ds2.port = 3316; // ds2.schema = "offer1"; // ds2.user = "offer"; // ds2.password = "offer"; // ds2.statement = "update xdual set x=now()"; // // // 数据源3（SCHEMA错误） // DataSourceConfig ds3 = new DataSourceConfig(); // ds3.host = "10.20.132.17"; // ds3.port = 3306; // ds3.schema = "offer1_x"; // ds3.user = "offer"; // ds3.password = "offer"; // ds3.statement = "update xdual set x=now()"; // // // 数据源4（用户错误） // DataSourceConfig ds4 = new DataSourceConfig(); // ds4.host = "10.20.132.17"; // ds4.port = 3306; // ds4.schema = "offer1"; // ds4.user = "offer_x"; // ds4.password = "offer"; // ds4.statement = "update xdual set x=now()"; // // // 数据源5（密码错误） // DataSourceConfig ds5 = new DataSourceConfig(); // ds5.host = "10.20.132.17"; // ds5.port = 3306; // ds5.schema = "offer1"; // ds5.user = "offer"; // ds5.password = "offer_x"; // ds5.statement = "update xdual set x=now()"; // // // 数据源6（语句错误） // DataSourceConfig ds6 = new DataSourceConfig(); // ds6.host = "10.20.132.17"; // ds6.port = 3306; // ds6.schema = "offer1"; // ds6.user = "offer"; // ds6.password = "offer"; // ds6.statement = "update xdual_x set x=now()"; // // // 数据源（正确配置） // DataSourceConfig ds = new DataSourceConfig(); // ds.host = "10.20.132.17"; // ds.port = 3306; // ds.schema = "offer1"; // ds.user = "offer"; // ds.password = "offer"; // ds.statement = "update xdual set x=now()"; // // DataNodeConfig node = new DataNodeConfig(); // node.name = "offer1"; // node.activedIndex = 0; // node.dataSource = new DataSourceConfig[] { ds1, ds2, ds3, ds4, ds5, ds6, // ds }; // return node; // } } ================================================ FILE: src/test/java/io/mycat/heartbeat/HeartbeatContext.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.heartbeat; /** * @author mycat */ public class HeartbeatContext { // private final static long TIMER_PERIOD = 1000L; // // private String name; // private Timer timer; // private NIOProcessor[] processors; // private NIOConnector connector; // // public HeartbeatContext(String name) throws IOException { // this.name = name; // this.init(); // } // // public void startup() { // // startup timer // timer.schedule(new TimerTask() { // @Override // public void run() { // TimeUtil.update(); // } // }, 0L, TimeUtil.UPDATE_PERIOD); // // // startup processors // for (int i = 0; i < processors.length; i++) { // processors[i].startup(); // } // // // startup connector // connector.start(); // } // // public void doHeartbeat(HeartbeatConfig heartbeat) { // timer.schedule(new MySQLHeartbeatTask(connector, heartbeat), 0L, // TIMER_PERIOD); // } // // private void init() throws IOException { // // init timer // this.timer = new Timer(name + "Timer", false); // // // init processors // processors = new // NIOProcessor[Runtime.getRuntime().availableProcessors()]; // for (int i = 0; i < processors.length; i++) { // processors[i] = new NIOProcessor(name + "Processor" + i); // } // // // init connector // connector = new NIOConnector(name + "Connector"); // connector.setProcessors(processors); // } } ================================================ FILE: src/test/java/io/mycat/heartbeat/HeartbeatStartup.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.heartbeat; /** * @author mycat */ public class HeartbeatStartup { // public static void main(String[] args) throws Exception { // // 初始化心跳运行环境 // HeartbeatContext ctx = new HeartbeatContext("HB"); // // // 启动心跳运行环境 // ctx.startup(); // // // 执行心跳任务 // doHeartbeat(ctx); // } // // static void doHeartbeat(HeartbeatContext ctx) { // DataNodeConfig[] nodes = HeartbeatConfigForTest.getOfferNodes(1, 32); // for (int i = 0; i < nodes.length; i++) { // HeartbeatConfig config = new HeartbeatConfig(); // config.node = nodes[i]; // ctx.doHeartbeat(config); // } // } // // static void doZookeeper(HeartbeatContext ctx) { // } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/PlatformUtilSuite.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe; import org.junit.Assert; import org.junit.Test; public class PlatformUtilSuite { @Test public void overlappingCopyMemory() { byte[] data = new byte[3 * 1024 * 1024]; int size = 2 * 1024 * 1024; for (int i = 0; i < data.length; ++i) { data[i] = (byte)i; } Platform.copyMemory(data, Platform.BYTE_ARRAY_OFFSET, data, Platform.BYTE_ARRAY_OFFSET, size); for (int i = 0; i < data.length; ++i) { Assert.assertEquals((byte)i, data[i]); } Platform.copyMemory( data, Platform.BYTE_ARRAY_OFFSET + 1, data, Platform.BYTE_ARRAY_OFFSET, size); for (int i = 0; i < size; ++i) { Assert.assertEquals((byte)(i + 1), data[i]); } for (int i = 0; i < data.length; ++i) { data[i] = (byte)i; } Platform.copyMemory( data, Platform.BYTE_ARRAY_OFFSET, data, Platform.BYTE_ARRAY_OFFSET + 1, size); for (int i = 0; i < size; ++i) { Assert.assertEquals((byte)i, data[i + 1]); } } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/array/LongArraySuite.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.array; import io.mycat.memory.unsafe.memory.MemoryBlock; import org.junit.Assert; import org.junit.Test; public class LongArraySuite { @Test public void basicTest() { long[] bytes = new long[2]; LongArray arr = new LongArray(MemoryBlock.fromLongArray(bytes)); arr.set(0, 1L); arr.set(1, 2L); arr.set(1, 3L); Assert.assertEquals(2, arr.size()); Assert.assertEquals(1L, arr.get(0)); Assert.assertEquals(3L, arr.get(1)); arr.zeroOut(); Assert.assertEquals(0L, arr.get(0)); Assert.assertEquals(0L, arr.get(1)); } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/hash/Murmur3_x86_32Suite.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.hash; import io.mycat.memory.unsafe.Platform; import org.junit.Assert; import org.junit.Test; import java.nio.charset.StandardCharsets; import java.util.HashSet; import java.util.Random; import java.util.Set; /** * Test file based on Guava's Murmur3Hash32Test. */ public class Murmur3_x86_32Suite { private static final Murmur3_x86_32 hasher = new Murmur3_x86_32(0); @Test public void testKnownIntegerInputs() { Assert.assertEquals(593689054, hasher.hashInt(0)); Assert.assertEquals(-189366624, hasher.hashInt(-42)); Assert.assertEquals(-1134849565, hasher.hashInt(42)); Assert.assertEquals(-1718298732, hasher.hashInt(Integer.MIN_VALUE)); Assert.assertEquals(-1653689534, hasher.hashInt(Integer.MAX_VALUE)); } @Test public void testKnownLongInputs() { Assert.assertEquals(1669671676, hasher.hashLong(0L)); Assert.assertEquals(-846261623, hasher.hashLong(-42L)); Assert.assertEquals(1871679806, hasher.hashLong(42L)); Assert.assertEquals(1366273829, hasher.hashLong(Long.MIN_VALUE)); Assert.assertEquals(-2106506049, hasher.hashLong(Long.MAX_VALUE)); } @Test public void randomizedStressTest() { int size = 65536; Random rand = new Random(); // A set used to track collision rate. Set hashcodes = new HashSet(); for (int i = 0; i < size; i++) { int vint = rand.nextInt(); long lint = rand.nextLong(); Assert.assertEquals(hasher.hashInt(vint), hasher.hashInt(vint)); Assert.assertEquals(hasher.hashLong(lint), hasher.hashLong(lint)); hashcodes.add(hasher.hashLong(lint)); } // A very loose bound. Assert.assertTrue(hashcodes.size() > size * 0.95); } @Test public void randomizedStressTestBytes() { int size = 65536; Random rand = new Random(); // A set used to track collision rate. Set hashcodes = new HashSet(); for (int i = 0; i < size; i++) { int byteArrSize = rand.nextInt(100) * 8; byte[] bytes = new byte[byteArrSize]; rand.nextBytes(bytes); Assert.assertEquals( hasher.hashUnsafeWords(bytes, Platform.BYTE_ARRAY_OFFSET, byteArrSize), hasher.hashUnsafeWords(bytes, Platform.BYTE_ARRAY_OFFSET, byteArrSize)); hashcodes.add(hasher.hashUnsafeWords( bytes, Platform.BYTE_ARRAY_OFFSET, byteArrSize)); } // A very loose bound. Assert.assertTrue(hashcodes.size() > size * 0.95); } @Test public void randomizedStressTestPaddedStrings() { int size = 64000; // A set used to track collision rate. Set hashcodes = new HashSet(); for (int i = 0; i < size; i++) { int byteArrSize = 8; byte[] strBytes = String.valueOf(i).getBytes(StandardCharsets.UTF_8); byte[] paddedBytes = new byte[byteArrSize]; System.arraycopy(strBytes, 0, paddedBytes, 0, strBytes.length); Assert.assertEquals( hasher.hashUnsafeWords(paddedBytes, Platform.BYTE_ARRAY_OFFSET, byteArrSize), hasher.hashUnsafeWords(paddedBytes, Platform.BYTE_ARRAY_OFFSET, byteArrSize)); hashcodes.add(hasher.hashUnsafeWords( paddedBytes, Platform.BYTE_ARRAY_OFFSET, byteArrSize)); } // A very loose bound. Assert.assertTrue(hashcodes.size() > size * 0.95); } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/map/AbstractBytesToBytesMapSuite.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.map; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.array.ByteArrayMethods; import io.mycat.memory.unsafe.memory.TestMemoryManager; import io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager; import io.mycat.memory.unsafe.storage.DataNodeDiskManager; import io.mycat.memory.unsafe.storage.SerializerManager; import io.mycat.memory.unsafe.utils.JavaUtils; import io.mycat.memory.unsafe.utils.MycatPropertyConf; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.util.*; import static org.hamcrest.Matchers.greaterThan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.mockito.Answers.RETURNS_SMART_NULLS; public abstract class AbstractBytesToBytesMapSuite { private final Random rand = new Random(42); MycatPropertyConf conf = new MycatPropertyConf() .set("mycat.memory.offHeap.enabled", "" + useOffHeapMemoryAllocator()) .set("mycat.memory.offHeap.size", "256mb"); private TestMemoryManager memoryManager = new TestMemoryManager(conf ); private DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager,0); private SerializerManager serializerManager = new SerializerManager(); private static final long PAGE_SIZE_BYTES = 1L << 26; // 64 megabytes final LinkedList spillFilesCreated = new LinkedList(); File tempDir; DataNodeDiskManager blockManager = new DataNodeDiskManager(conf,true,serializerManager); /* private static final class CompressStream extends AbstractFunction1 { @Override public OutputStream apply(OutputStream stream) { return stream; } } */ @Before public void setup() { } @After public void tearDown() throws IOException { //Utils.deleteRecursively(tempDir); //tempDir = null; if (dataNodeMemoryManager != null) { Assert.assertEquals(0L, dataNodeMemoryManager.cleanUpAllAllocatedMemory()); long leakedMemory = dataNodeMemoryManager.getMemoryConsumptionForThisConnection(); dataNodeMemoryManager = null; Assert.assertEquals(0L, leakedMemory); } } protected abstract boolean useOffHeapMemoryAllocator(); private static byte[] getByteArray(Object base, long offset, int size) { final byte[] arr = new byte[size]; Platform.copyMemory(base, offset, arr, Platform.BYTE_ARRAY_OFFSET, size); return arr; } private byte[] getRandomByteArray(int numWords) { Assert.assertTrue(numWords >= 0); final int lengthInBytes = numWords * 8; final byte[] bytes = new byte[lengthInBytes]; rand.nextBytes(bytes); return bytes; } /** * Fast equality checking for byte arrays, since these comparisons are a bottleneck * in our stress tests. */ private static boolean arrayEquals( byte[] expected, Object base, long offset, long actualLengthBytes) { return (actualLengthBytes == expected.length) && ByteArrayMethods.arrayEquals( expected, Platform.BYTE_ARRAY_OFFSET, base, offset, expected.length ); } @Test public void emptyMap() { BytesToBytesMap map = new BytesToBytesMap(dataNodeMemoryManager, 64, PAGE_SIZE_BYTES); try { Assert.assertEquals(0, map.numKeys()); final int keyLengthInWords = 10; final int keyLengthInBytes = keyLengthInWords * 8; final byte[] key = getRandomByteArray(keyLengthInWords); Assert.assertFalse(map.lookup(key, Platform.BYTE_ARRAY_OFFSET, keyLengthInBytes).isDefined()); Assert.assertFalse(map.iterator().hasNext()); } finally { map.free(); } } @Test public void setAndRetrieveAKey() { BytesToBytesMap map = new BytesToBytesMap(dataNodeMemoryManager, 64, PAGE_SIZE_BYTES); final int recordLengthWords = 10; final int recordLengthBytes = recordLengthWords * 8; final byte[] keyData = getRandomByteArray(recordLengthWords); final byte[] valueData = getRandomByteArray(recordLengthWords); try { final BytesToBytesMap.Location loc = map.lookup(keyData, Platform.BYTE_ARRAY_OFFSET, recordLengthBytes); Assert.assertFalse(loc.isDefined()); Assert.assertTrue(loc.append( keyData, Platform.BYTE_ARRAY_OFFSET, recordLengthBytes, valueData, Platform.BYTE_ARRAY_OFFSET, recordLengthBytes )); // After storing the key and value, the other location methods should return results that // reflect the result of this store without us having to call lookup() again on the same key. Assert.assertEquals(recordLengthBytes, loc.getKeyLength()); Assert.assertEquals(recordLengthBytes, loc.getValueLength()); Assert.assertArrayEquals(keyData, getByteArray(loc.getKeyBase(), loc.getKeyOffset(), recordLengthBytes)); Assert.assertArrayEquals(valueData, getByteArray(loc.getValueBase(), loc.getValueOffset(), recordLengthBytes)); // After calling lookup() the location should still point to the correct data. Assert.assertTrue( map.lookup(keyData, Platform.BYTE_ARRAY_OFFSET, recordLengthBytes).isDefined()); Assert.assertEquals(recordLengthBytes, loc.getKeyLength()); Assert.assertEquals(recordLengthBytes, loc.getValueLength()); Assert.assertArrayEquals(keyData, getByteArray(loc.getKeyBase(), loc.getKeyOffset(), recordLengthBytes)); Assert.assertArrayEquals(valueData, getByteArray(loc.getValueBase(), loc.getValueOffset(), recordLengthBytes)); try { Assert.assertTrue(loc.append( keyData, Platform.BYTE_ARRAY_OFFSET, recordLengthBytes, valueData, Platform.BYTE_ARRAY_OFFSET, recordLengthBytes )); Assert.fail("Should not be able to set a new value for a key"); } catch (AssertionError e) { // Expected exception; do nothing. } } finally { map.free(); } } private void iteratorTestBase(boolean destructive) throws Exception { final int size = 4096; BytesToBytesMap map = new BytesToBytesMap(dataNodeMemoryManager, size / 2, PAGE_SIZE_BYTES); try { for (long i = 0; i < size; i++) { final long[] value = new long[] { i }; final BytesToBytesMap.Location loc = map.lookup(value, Platform.LONG_ARRAY_OFFSET, 8); Assert.assertFalse(loc.isDefined()); // Ensure that we store some zero-length keys if (i % 5 == 0) { Assert.assertTrue(loc.append( null, Platform.LONG_ARRAY_OFFSET, 0, value, Platform.LONG_ARRAY_OFFSET, 8 )); } else { Assert.assertTrue(loc.append( value, Platform.LONG_ARRAY_OFFSET, 8, value, Platform.LONG_ARRAY_OFFSET, 8 )); } } final BitSet valuesSeen = new BitSet(size); final Iterator iter; if (destructive) { iter = map.destructiveIterator(); } else { iter = map.iterator(); } int numPages = map.getNumDataPages(); int countFreedPages = 0; while (iter.hasNext()) { final BytesToBytesMap.Location loc = iter.next(); Assert.assertTrue(loc.isDefined()); final long value = Platform.getLong(loc.getValueBase(), loc.getValueOffset()); final long keyLength = loc.getKeyLength(); if (keyLength == 0) { Assert.assertTrue("value " + value + " was not divisible by 5", value % 5 == 0); } else { final long key = Platform.getLong(loc.getKeyBase(), loc.getKeyOffset()); Assert.assertEquals(value, key); } valuesSeen.set((int) value); if (destructive) { // The iterator moves onto next page and frees previous page if (map.getNumDataPages() < numPages) { numPages = map.getNumDataPages(); countFreedPages++; } } } if (destructive) { // Latest page is not freed by iterator but by map itself Assert.assertEquals(countFreedPages, numPages - 1); } Assert.assertEquals(size, valuesSeen.cardinality()); } finally { map.free(); } } @Test public void iteratorTest() throws Exception { iteratorTestBase(false); } @Test public void destructiveIteratorTest() throws Exception { iteratorTestBase(true); } @Test public void iteratingOverDataPagesWithWastedSpace() throws Exception { final int NUM_ENTRIES = 1000 * 1000; final int KEY_LENGTH = 24; final int VALUE_LENGTH = 40; final BytesToBytesMap map = new BytesToBytesMap(dataNodeMemoryManager, NUM_ENTRIES, PAGE_SIZE_BYTES); // Each record will take 8 + 24 + 40 = 72 bytes of space in the data page. Our 64-megabyte // pages won't be evenly-divisible by records of this size, which will cause us to waste some // space at the end of the page. This is necessary in order for us to take the end-of-record // handling branch in iterator(). try { for (int i = 0; i < NUM_ENTRIES; i++) { final long[] key = new long[] { i, i, i }; // 3 * 8 = 24 bytes final long[] value = new long[] { i, i, i, i, i }; // 5 * 8 = 40 bytes final BytesToBytesMap.Location loc = map.lookup( key, Platform.LONG_ARRAY_OFFSET, KEY_LENGTH ); Assert.assertFalse(loc.isDefined()); Assert.assertTrue(loc.append( key, Platform.LONG_ARRAY_OFFSET, KEY_LENGTH, value, Platform.LONG_ARRAY_OFFSET, VALUE_LENGTH )); } Assert.assertEquals(2, map.getNumDataPages()); final BitSet valuesSeen = new BitSet(NUM_ENTRIES); final Iterator iter = map.iterator(); final long[] key = new long[KEY_LENGTH / 8]; final long[] value = new long[VALUE_LENGTH / 8]; while (iter.hasNext()) { final BytesToBytesMap.Location loc = iter.next(); Assert.assertTrue(loc.isDefined()); Assert.assertEquals(KEY_LENGTH, loc.getKeyLength()); Assert.assertEquals(VALUE_LENGTH, loc.getValueLength()); Platform.copyMemory( loc.getKeyBase(), loc.getKeyOffset(), key, Platform.LONG_ARRAY_OFFSET, KEY_LENGTH ); Platform.copyMemory( loc.getValueBase(), loc.getValueOffset(), value, Platform.LONG_ARRAY_OFFSET, VALUE_LENGTH ); for (long j : key) { Assert.assertEquals(key[0], j); } for (long j : value) { Assert.assertEquals(key[0], j); } valuesSeen.set((int) key[0]); } Assert.assertEquals(NUM_ENTRIES, valuesSeen.cardinality()); } finally { map.free(); } } @Test public void randomizedStressTest() { final int size = 65536; // Java arrays' hashCodes() aren't based on the arrays' contents, so we need to wrap arrays // into ByteBuffers in order to use them as keys here. final Map expected = new HashMap(); final BytesToBytesMap map = new BytesToBytesMap(dataNodeMemoryManager, size, PAGE_SIZE_BYTES); try { // Fill the map to 90% full so that we can trigger probing for (int i = 0; i < size * 0.9; i++) { final byte[] key = getRandomByteArray(rand.nextInt(10) + 1); final byte[] value = getRandomByteArray(rand.nextInt(10) + 1); if (!expected.containsKey(ByteBuffer.wrap(key))) { expected.put(ByteBuffer.wrap(key), value); final BytesToBytesMap.Location loc = map.lookup( key, Platform.BYTE_ARRAY_OFFSET, key.length ); Assert.assertFalse(loc.isDefined()); Assert.assertTrue(loc.append( key, Platform.BYTE_ARRAY_OFFSET, key.length, value, Platform.BYTE_ARRAY_OFFSET, value.length )); // After calling putNewKey, the following should be true, even before calling // lookup(): Assert.assertTrue(loc.isDefined()); Assert.assertEquals(key.length, loc.getKeyLength()); Assert.assertEquals(value.length, loc.getValueLength()); Assert.assertTrue(arrayEquals(key, loc.getKeyBase(), loc.getKeyOffset(), key.length)); Assert.assertTrue( arrayEquals(value, loc.getValueBase(), loc.getValueOffset(), value.length)); } } /** for (Map.Entry entry : expected.entrySet()) { final byte[] key = JavaUtils.bufferToArray(entry.getKey()); final byte[] value = entry.getValue(); final BytesToBytesMap.Location loc = map.lookup(key, Platform.BYTE_ARRAY_OFFSET, key.length); Assert.assertTrue(loc.isDefined()); Assert.assertTrue( arrayEquals(key, loc.getKeyBase(), loc.getKeyOffset(), loc.getKeyLength())); Assert.assertTrue( arrayEquals(value, loc.getValueBase(), loc.getValueOffset(), loc.getValueLength())); } */ } finally { map.free(); } } @Test public void randomizedTestWithRecordsLargerThanPageSize() { final long pageSizeBytes = 128; final BytesToBytesMap map = new BytesToBytesMap(dataNodeMemoryManager, 64, pageSizeBytes); // Java arrays' hashCodes() aren't based on the arrays' contents, so we need to wrap arrays // into ByteBuffers in order to use them as keys here. final Map expected = new HashMap(); try { for (int i = 0; i < 1000; i++) { final byte[] key = getRandomByteArray(rand.nextInt(128)); final byte[] value = getRandomByteArray(rand.nextInt(128)); if (!expected.containsKey(ByteBuffer.wrap(key))) { expected.put(ByteBuffer.wrap(key), value); final BytesToBytesMap.Location loc = map.lookup( key, Platform.BYTE_ARRAY_OFFSET, key.length ); Assert.assertFalse(loc.isDefined()); Assert.assertTrue(loc.append( key, Platform.BYTE_ARRAY_OFFSET, key.length, value, Platform.BYTE_ARRAY_OFFSET, value.length )); // After calling putNewKey, the following should be true, even before calling // lookup(): Assert.assertTrue(loc.isDefined()); Assert.assertEquals(key.length, loc.getKeyLength()); Assert.assertEquals(value.length, loc.getValueLength()); Assert.assertTrue(arrayEquals(key, loc.getKeyBase(), loc.getKeyOffset(), key.length)); Assert.assertTrue( arrayEquals(value, loc.getValueBase(), loc.getValueOffset(), value.length)); } } /** for (Map.Entry entry : expected.entrySet()) { final byte[] key = JavaUtils.bufferToArray(entry.getKey()); final byte[] value = entry.getValue(); final BytesToBytesMap.Location loc = map.lookup(key, Platform.BYTE_ARRAY_OFFSET, key.length); Assert.assertTrue(loc.isDefined()); Assert.assertTrue( arrayEquals(key, loc.getKeyBase(), loc.getKeyOffset(), loc.getKeyLength())); Assert.assertTrue( arrayEquals(value, loc.getValueBase(), loc.getValueOffset(), loc.getValueLength())); } */ } finally { map.free(); } } @Test public void failureToAllocateFirstPage() { memoryManager.limit(1024); // longArray BytesToBytesMap map = new BytesToBytesMap(dataNodeMemoryManager, 1, PAGE_SIZE_BYTES); try { final long[] emptyArray = new long[0]; final BytesToBytesMap.Location loc = map.lookup(emptyArray, Platform.LONG_ARRAY_OFFSET, 0); Assert.assertFalse(loc.isDefined()); Assert.assertFalse(loc.append( emptyArray, Platform.LONG_ARRAY_OFFSET, 0, emptyArray, Platform.LONG_ARRAY_OFFSET, 0)); } finally { map.free(); } } @Test public void failureToGrow() { BytesToBytesMap map = new BytesToBytesMap(dataNodeMemoryManager, 1, 1024); try { boolean success = true; int i; for (i = 0; i < 127; i++) { if (i > 0) { memoryManager.limit(0); } final long[] arr = new long[]{i}; final BytesToBytesMap.Location loc = map.lookup(arr, Platform.LONG_ARRAY_OFFSET, 8); success = loc.append(arr, Platform.LONG_ARRAY_OFFSET, 8, arr, Platform.LONG_ARRAY_OFFSET, 8); if (!success) { break; } } Assert.assertThat(i, greaterThan(0)); Assert.assertFalse(success); } finally { map.free(); } } @Test public void spillInIterator() throws IOException { BytesToBytesMap map = new BytesToBytesMap( dataNodeMemoryManager, blockManager, serializerManager, 1, 0.75, 1024, false); try { int i; for (i = 0; i < 1024; i++) { final long[] arr = new long[]{i}; final BytesToBytesMap.Location loc = map.lookup(arr, Platform.LONG_ARRAY_OFFSET, 8); loc.append(arr, Platform.LONG_ARRAY_OFFSET, 8, arr, Platform.LONG_ARRAY_OFFSET, 8); } BytesToBytesMap.MapIterator iter = map.iterator(); for (i = 0; i < 100; i++) { iter.next(); } // Non-destructive iterator is not spillable Assert.assertEquals(0, iter.spill(1024L * 10)); for (i = 100; i < 1024; i++) { iter.next(); } BytesToBytesMap.MapIterator iter2 = map.destructiveIterator(); for (i = 0; i < 100; i++) { iter2.next(); } Assert.assertTrue(iter2.spill(1024) >= 1024); for (i = 100; i < 1024; i++) { iter2.next(); } assertFalse(iter2.hasNext()); } finally { map.free(); for (File spillFile : spillFilesCreated) { assertFalse("Spill file " + spillFile.getPath() + " was not cleaned up", spillFile.exists()); } } } @Test public void multipleValuesForSameKey() { BytesToBytesMap map = new BytesToBytesMap(dataNodeMemoryManager, blockManager, serializerManager, 1, 0.75, 1024, false); try { int i; for (i = 0; i < 1024; i++) { final long[] arr = new long[]{i}; map.lookup(arr, Platform.LONG_ARRAY_OFFSET, 8) .append(arr, Platform.LONG_ARRAY_OFFSET, 8, arr, Platform.LONG_ARRAY_OFFSET, 8); } assert map.numKeys() == 1024; assert map.numValues() == 1024; for (i = 0; i < 1024; i++) { final long[] arr = new long[]{i}; map.lookup(arr, Platform.LONG_ARRAY_OFFSET, 8) .append(arr, Platform.LONG_ARRAY_OFFSET, 8, arr, Platform.LONG_ARRAY_OFFSET, 8); } assert map.numKeys() == 1024; assert map.numValues() == 2048; for (i = 0; i < 1024; i++) { final long[] arr = new long[]{i}; final BytesToBytesMap.Location loc = map.lookup(arr, Platform.LONG_ARRAY_OFFSET, 8); assert loc.isDefined(); assert loc.nextValue(); assert !loc.nextValue(); } BytesToBytesMap.MapIterator iter = map.iterator(); for (i = 0; i < 2048; i++) { assert iter.hasNext(); final BytesToBytesMap.Location loc = iter.next(); assert loc.isDefined(); } } finally { map.free(); } } @Test public void initialCapacityBoundsChecking() { try { new BytesToBytesMap(dataNodeMemoryManager, 0, PAGE_SIZE_BYTES); Assert.fail("Expected IllegalArgumentException to be thrown"); } catch (IllegalArgumentException e) { // expected exception } try { new BytesToBytesMap( dataNodeMemoryManager, BytesToBytesMap.MAX_CAPACITY + 1, PAGE_SIZE_BYTES); Assert.fail("Expected IllegalArgumentException to be thrown"); } catch (IllegalArgumentException e) { // expected exception } } @Test public void testPeakMemoryUsed() { final long recordLengthBytes = 32; final long pageSizeBytes = 256 + 8; // 8 bytes for end-of-page marker final long numRecordsPerPage = (pageSizeBytes - 8) / recordLengthBytes; final BytesToBytesMap map = new BytesToBytesMap(dataNodeMemoryManager, 1024, pageSizeBytes); // Since BytesToBytesMap is append-only, we expect the total memory consumption to be // monotonically increasing. More specifically, every time we allocate a new page it // should increase by exactly the size of the page. In this regard, the memory usage // at any given time is also the peak memory used. long previousPeakMemory = map.getPeakMemoryUsedBytes(); long newPeakMemory; try { for (long i = 0; i < numRecordsPerPage * 10; i++) { final long[] value = new long[]{i}; map.lookup(value, Platform.LONG_ARRAY_OFFSET, 8).append( value, Platform.LONG_ARRAY_OFFSET, 8, value, Platform.LONG_ARRAY_OFFSET, 8); newPeakMemory = map.getPeakMemoryUsedBytes(); if (i % numRecordsPerPage == 0) { // We allocated a new page for this record, so peak memory should change assertEquals(previousPeakMemory + pageSizeBytes, newPeakMemory); } else { assertEquals(previousPeakMemory, newPeakMemory); } previousPeakMemory = newPeakMemory; } // Freeing the map should not change the peak memory map.free(); newPeakMemory = map.getPeakMemoryUsedBytes(); assertEquals(previousPeakMemory, newPeakMemory); } finally { map.free(); } } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/map/BytesToBytesMapOffHeapSuite.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.map; public class BytesToBytesMapOffHeapSuite extends AbstractBytesToBytesMapSuite { @Override protected boolean useOffHeapMemoryAllocator() { return true; } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/map/BytesToBytesMapOnHeapSuite.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.map; public class BytesToBytesMapOnHeapSuite extends AbstractBytesToBytesMapSuite { @Override protected boolean useOffHeapMemoryAllocator() { return false; } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/map/MapSorterByValueTest.java ================================================ package io.mycat.memory.unsafe.map; import org.junit.Test; import java.util.*; /** * Created by znix on 2016/7/4. */ public class MapSorterByValueTest { @Test public void testMapSorterByValue(){ Map map = new HashMap(); map.put("q",23); map.put("b",4); map.put("c",5); map.put("d",6); Map resultMap = mapSorterByValue(map); //按Value进行排序 for (Map.Entry entry : resultMap.entrySet()) { System.out.println(entry.getKey() + " " + entry.getValue()); } } private Map mapSorterByValue(Map map) { if (map == null || map.isEmpty()) { return null; } Map sortedMap = new LinkedHashMap(); List> entryList = new ArrayList< Map.Entry>( map.entrySet()); Collections.sort(entryList, new Comparator>() { @Override public int compare(Map.Entry o1, Map.Entry o2) { return o1.getValue().compareTo(o2.getValue()); } }); Iterator> iter = entryList.iterator(); Map.Entry tmpEntry = null; while (iter.hasNext()) { tmpEntry = iter.next(); sortedMap.put(tmpEntry.getKey(), tmpEntry.getValue()); } return sortedMap; } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/map/UnsafeFixedWidthAggregationMapSuite.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.map; import io.mycat.memory.MyCatMemory; import io.mycat.memory.unsafe.KVIterator; import io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager; import io.mycat.memory.unsafe.memory.mm.MemoryManager; import io.mycat.memory.unsafe.row.BufferHolder; import io.mycat.memory.unsafe.row.StructType; import io.mycat.memory.unsafe.row.UnsafeRow; import io.mycat.memory.unsafe.row.UnsafeRowWriter; import io.mycat.memory.unsafe.utils.BytesTools; import io.mycat.sqlengine.mpp.ColMeta; import io.mycat.sqlengine.mpp.OrderCol; import org.apache.log4j.Logger; import org.junit.Assert; import org.junit.Test; import java.io.IOException; import java.util.*; /** * Created by zagnix on 2016/6/4. */ public class UnsafeFixedWidthAggregationMapSuite { private StructType groupKeySchema ; private StructType aggBufferSchema; private UnsafeRow emptyAggregationBuffer; private long PAGE_SIZE_BYTES = 1L << 20; private final Random rand = new Random(42); private static Logger LOGGER = Logger.getLogger(UnsafeFixedWidthAggregationMapSuite.class); @Test public void testAggregateMap() throws NoSuchFieldException, IllegalAccessException, IOException { /** * 创造上文环境 */ MyCatMemory myCatMemory = new MyCatMemory(); MemoryManager memoryManager = myCatMemory.getResultMergeMemoryManager(); DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId()); /** * 构造数据字段group key */ int fieldCount = 2; ColMeta colMeta = null; Map colMetaMap = new HashMap(fieldCount); colMeta = new ColMeta(0,ColMeta.COL_TYPE_STRING); colMetaMap.put("id",colMeta); colMeta = new ColMeta(1,ColMeta.COL_TYPE_STRING); colMetaMap.put("name",colMeta); OrderCol[] orderCols = new OrderCol[1]; OrderCol orderCol = new OrderCol(colMetaMap.get("id"),OrderCol.COL_ORDER_TYPE_DESC); orderCols[0] = orderCol; groupKeySchema = new StructType(colMetaMap,fieldCount); groupKeySchema.setOrderCols(orderCols); /** * 构造数据字段value key */ fieldCount = 4; colMeta = null; colMetaMap = new HashMap(fieldCount); colMeta = new ColMeta(0,ColMeta.COL_TYPE_STRING); colMetaMap.put("id",colMeta); colMeta = new ColMeta(1,ColMeta.COL_TYPE_STRING); colMetaMap.put("name",colMeta); colMeta = new ColMeta(2,ColMeta.COL_TYPE_INT); colMetaMap.put("age",colMeta); colMeta = new ColMeta(3,ColMeta.COL_TYPE_LONGLONG); colMetaMap.put("score",colMeta); orderCols = new OrderCol[1]; orderCol = new OrderCol(colMetaMap.get("id"),OrderCol.COL_ORDER_TYPE_DESC); orderCols[0] = orderCol; aggBufferSchema = new StructType(colMetaMap,fieldCount); aggBufferSchema.setOrderCols(orderCols); /** *emtpy Row value */ BufferHolder bufferHolder ; emptyAggregationBuffer = new UnsafeRow(4); bufferHolder = new BufferHolder(emptyAggregationBuffer,0); UnsafeRowWriter unsafeRowWriter = new UnsafeRowWriter(bufferHolder,4); bufferHolder.reset(); String value = "o"; unsafeRowWriter.write(0,value.getBytes()); unsafeRowWriter.write(1,value.getBytes()); emptyAggregationBuffer.setInt(2,0); emptyAggregationBuffer.setLong(3,0); emptyAggregationBuffer.setTotalSize(bufferHolder.totalSize()); UnsafeFixedWidthAggregationMap map = new UnsafeFixedWidthAggregationMap( emptyAggregationBuffer, aggBufferSchema, groupKeySchema, dataNodeMemoryManager, 2*1024, PAGE_SIZE_BYTES, true); /** * 造数据 */ int i; List rows = new ArrayList(); for ( i = 0; i < 100000; i++) { /** * key */ UnsafeRow groupKey = new UnsafeRow(2); bufferHolder = new BufferHolder(groupKey,0); unsafeRowWriter = new UnsafeRowWriter(bufferHolder,2); bufferHolder.reset(); unsafeRowWriter.write(0, BytesTools.toBytes(rand.nextInt(10000000))); unsafeRowWriter.write(1,BytesTools.toBytes(rand.nextInt(10000000))); groupKey.setTotalSize(bufferHolder.totalSize()); UnsafeRow valueKey = new UnsafeRow(4); bufferHolder = new BufferHolder(valueKey,0); unsafeRowWriter = new UnsafeRowWriter(bufferHolder,4); bufferHolder.reset(); unsafeRowWriter.write(0, BytesTools.toBytes(rand.nextInt(10))); unsafeRowWriter.write(1,BytesTools.toBytes(rand.nextInt(10))); valueKey.setInt(2,i); valueKey.setLong(3,1); valueKey.setTotalSize(bufferHolder.totalSize()); if(map.find(groupKey)){ UnsafeRow rs = map.getAggregationBuffer(groupKey); rs.setLong(3,i+valueKey.getLong(3)); rs.setInt(2,100+valueKey.getInt(2)); }else { map.put(groupKey,valueKey); } rows.add(valueKey); } KVIterator iter = map.iterator(); int j = 0; while (iter.next()){ Assert.assertEquals(j,iter.getValue().getInt(2)); j++; iter.getValue().setInt(2,5000000); iter.getValue().setLong(3,600000); } Assert.assertEquals(rows.size(),j); int k = 0; KVIterator iter1 = map.iterator(); while (iter1.next()){ k++; // LOGGER.error("(" + BytesTools.toInt(iter1.getKey().getBinary(0)) + "," + // iter1.getValue().getInt(2) +"," +iter1.getValue().getLong(3)+")"); Assert.assertEquals(5000000,iter1.getValue().getInt(2)); Assert.assertEquals(600000,iter1.getValue().getLong(3)); } Assert.assertEquals(j,k); map.free(); } @Test public void testWithMemoryLeakDetection() throws IOException, NoSuchFieldException, IllegalAccessException { MyCatMemory myCatMemory = new MyCatMemory(); MemoryManager memoryManager = myCatMemory.getResultMergeMemoryManager(); DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId()); int fieldCount = 3; ColMeta colMeta = null; Map colMetaMap = new HashMap(fieldCount); colMeta = new ColMeta(0,ColMeta.COL_TYPE_STRING); colMetaMap.put("id",colMeta); colMeta = new ColMeta(1,ColMeta.COL_TYPE_STRING); colMetaMap.put("name",colMeta); colMeta = new ColMeta(2,ColMeta.COL_TYPE_STRING); colMetaMap.put("age",colMeta); OrderCol[] orderCols = new OrderCol[1]; OrderCol orderCol = new OrderCol(colMetaMap.get("id"),OrderCol.COL_ORDER_TYPE_DESC); orderCols[0] = orderCol; groupKeySchema = new StructType(colMetaMap,fieldCount); groupKeySchema.setOrderCols(orderCols); fieldCount = 3; colMeta = null; colMetaMap = new HashMap(fieldCount); colMeta = new ColMeta(0,ColMeta.COL_TYPE_LONGLONG); colMetaMap.put("age",colMeta); colMeta = new ColMeta(1,ColMeta.COL_TYPE_LONGLONG); colMetaMap.put("age1",colMeta); colMeta = new ColMeta(2,ColMeta.COL_TYPE_STRING); colMetaMap.put("name",colMeta); orderCols = new OrderCol[1]; orderCol = new OrderCol(colMetaMap.get("id"),OrderCol.COL_ORDER_TYPE_DESC); orderCols[0] = orderCol; aggBufferSchema = new StructType(colMetaMap,fieldCount); aggBufferSchema.setOrderCols(orderCols); /** * value */ BufferHolder bufferHolder ; emptyAggregationBuffer = new UnsafeRow(3); bufferHolder = new BufferHolder(emptyAggregationBuffer,0); UnsafeRowWriter unsafeRowWriter = new UnsafeRowWriter(bufferHolder,3); bufferHolder.reset(); String value = "ok,hello"; emptyAggregationBuffer.setLong(0,0); emptyAggregationBuffer.setLong(1,0); unsafeRowWriter.write(2,value.getBytes()); emptyAggregationBuffer.setTotalSize(bufferHolder.totalSize()); UnsafeFixedWidthAggregationMap map = new UnsafeFixedWidthAggregationMap( emptyAggregationBuffer, aggBufferSchema, groupKeySchema, dataNodeMemoryManager, 2*1024, PAGE_SIZE_BYTES, false ); int i; List rows = new ArrayList(); for ( i = 0; i < 1000; i++) { String line = "testUnsafeRow" + i; /** * key */ UnsafeRow groupKey = new UnsafeRow(3); bufferHolder = new BufferHolder(groupKey,0); unsafeRowWriter = new UnsafeRowWriter(bufferHolder,3); bufferHolder.reset(); final byte[] key = getRandomByteArray(rand.nextInt(8)); String age = "5"+i; unsafeRowWriter.write(0,key); unsafeRowWriter.write(1,line.getBytes()); unsafeRowWriter.write(2,age.getBytes()); groupKey.setTotalSize(bufferHolder.totalSize()); map.getAggregationBuffer(groupKey); rows.add(groupKey); } Assert.assertEquals(i ,rows.size() ); UnsafeRow row = rows.get(12); UnsafeRow rs = map.getAggregationBuffer(row); rs.setLong(0,12); rs = map.getAggregationBuffer(row); Assert.assertEquals(12,rs.getLong(0)); map.free(); } private byte[] getRandomByteArray(int numWords) { Assert.assertTrue(numWords >= 0); final int lengthInBytes = numWords * 8; final byte[] bytes = new byte[lengthInBytes]; rand.nextBytes(bytes); return bytes; } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/memory/MemoryManagerSuite.java ================================================ package io.mycat.memory.unsafe.memory; /** * Created by zagnix on 2016/6/6. */ public interface MemoryManagerSuite { } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/memory/MycatMemoryTest.java ================================================ package io.mycat.memory.unsafe.memory; import io.mycat.memory.MyCatMemory; import io.mycat.memory.unsafe.Platform; import org.junit.Test; /** * Created by zagnix on 2016/6/12. */ public class MycatMemoryTest { /** * -Xmx1024m -XX:MaxDirectMemorySize=1G */ @Test public void testMycatMemory() throws NoSuchFieldException, IllegalAccessException { MyCatMemory myCatMemory = new MyCatMemory(); System.out.println(myCatMemory.getResultSetBufferSize()); System.out.println(Platform.getMaxHeapMemory()); System.out.println(Platform.getMaxDirectMemory()); } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/memory/TaskMemoryManagerSuite.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.memory; import io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager; import io.mycat.memory.unsafe.memory.mm.MemoryMode; import io.mycat.memory.unsafe.memory.mm.ResultMergeMemoryManager; import io.mycat.memory.unsafe.utils.MycatPropertyConf; import org.junit.Assert; import org.junit.Test; public class TaskMemoryManagerSuite { @Test public void leakedPageMemoryIsDetected() { final DataNodeMemoryManager manager = new DataNodeMemoryManager( new ResultMergeMemoryManager( new MycatPropertyConf().set("mycat.memory.offHeap.enabled", "false") .set("mycat.memory.offHeap.size","32768"), 1, Long.MAX_VALUE ), 0); manager.allocatePage(4096, null); // leak memory Assert.assertEquals(4096, manager.getMemoryConsumptionForThisConnection()); Assert.assertEquals(4096, manager.cleanUpAllAllocatedMemory()); } @Test public void encodePageNumberAndOffsetOffHeap() { final MycatPropertyConf conf = new MycatPropertyConf() .set("mycat.memory.offHeap.enabled", "true") .set("mycat.memory.offHeap.size", "1000"); final DataNodeMemoryManager manager = new DataNodeMemoryManager(new TestMemoryManager(conf), 0); final MemoryBlock dataPage = manager.allocatePage(256, null); // In off-heap mode, an offset is an absolute address that may require more than 51 bits to // encode. This map exercises that corner-case: final long offset = ((1L << DataNodeMemoryManager.OFFSET_BITS) + 10); final long encodedAddress = manager.encodePageNumberAndOffset(dataPage, offset); Assert.assertEquals(null, manager.getPage(encodedAddress)); Assert.assertEquals(offset, manager.getOffsetInPage(encodedAddress)); } @Test public void encodePageNumberAndOffsetOnHeap() { final DataNodeMemoryManager manager = new DataNodeMemoryManager( new TestMemoryManager(new MycatPropertyConf().set("mycat.memory.offHeap.enabled", "false")), 0); final MemoryBlock dataPage = manager.allocatePage(256, null); final long encodedAddress = manager.encodePageNumberAndOffset(dataPage, 64); Assert.assertEquals(dataPage.getBaseObject(), manager.getPage(encodedAddress)); Assert.assertEquals(64, manager.getOffsetInPage(encodedAddress)); } @Test public void cooperativeSpilling() throws InterruptedException { final TestMemoryManager memoryManager = new TestMemoryManager(new MycatPropertyConf()); memoryManager.limit(100); final DataNodeMemoryManager manager = new DataNodeMemoryManager(memoryManager, 0); TestMemoryConsumer c1 = new TestMemoryConsumer(manager); TestMemoryConsumer c2 = new TestMemoryConsumer(manager); c1.use(100); Assert.assertEquals(100, c1.getUsed()); c2.use(100); Assert.assertEquals(100, c2.getUsed()); Assert.assertEquals(0, c1.getUsed()); // spilled c1.use(100); Assert.assertEquals(100, c1.getUsed()); Assert.assertEquals(0, c2.getUsed()); // spilled c1.use(50); Assert.assertEquals(50, c1.getUsed()); // spilled Assert.assertEquals(0, c2.getUsed()); c2.use(50); Assert.assertEquals(50, c1.getUsed()); Assert.assertEquals(50, c2.getUsed()); c1.use(100); Assert.assertEquals(100, c1.getUsed()); Assert.assertEquals(0, c2.getUsed()); // spilled c1.free(20); Assert.assertEquals(80, c1.getUsed()); c2.use(10); Assert.assertEquals(80, c1.getUsed()); Assert.assertEquals(10, c2.getUsed()); c2.use(100); Assert.assertEquals(100, c2.getUsed()); Assert.assertEquals(0, c1.getUsed()); // spilled c1.free(0); c2.free(100); Assert.assertEquals(0, manager.cleanUpAllAllocatedMemory()); } @Test public void offHeapConfigurationBackwardsCompatibility() { final MycatPropertyConf conf = new MycatPropertyConf() .set("mycat.memory.offHeap.enabled", "true") .set("mycat.memory.offHeap.size","1000"); final DataNodeMemoryManager manager = new DataNodeMemoryManager(new TestMemoryManager(conf), 0); Assert.assertSame(MemoryMode.OFF_HEAP, manager.tungstenMemoryMode); } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/memory/TestMemoryConsumer.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.memory; import io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager; import io.mycat.memory.unsafe.memory.mm.MemoryConsumer; import java.io.IOException; public class TestMemoryConsumer extends MemoryConsumer { public TestMemoryConsumer(DataNodeMemoryManager memoryManager) { super(memoryManager); } @Override public long spill(long size, MemoryConsumer trigger) throws IOException { long used = getUsed(); free(used); return used; } void use(long size) throws InterruptedException { long got = dataNodeMemoryManager.acquireExecutionMemory( size, dataNodeMemoryManager.tungstenMemoryMode, this); used += got; } void free(long size) { used -= size; dataNodeMemoryManager.releaseExecutionMemory( size, dataNodeMemoryManager.tungstenMemoryMode, this); } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/memory/TestMemoryManager.java ================================================ package io.mycat.memory.unsafe.memory; import io.mycat.memory.unsafe.memory.mm.MemoryManager; import io.mycat.memory.unsafe.memory.mm.MemoryMode; import io.mycat.memory.unsafe.utils.MycatPropertyConf; public class TestMemoryManager extends MemoryManager { public TestMemoryManager(MycatPropertyConf conf){ super(conf,1, Long.MAX_VALUE); } private boolean oomOnce = false; private long available = Long.MAX_VALUE; @Override protected long acquireExecutionMemory( long numBytes, long taskAttemptId, MemoryMode memoryMode){ if (oomOnce) { oomOnce = false; return 0; } else if (available >= numBytes) { available -= numBytes; return numBytes; } else { long grant = available; available = 0; return grant; } } @Override public void releaseExecutionMemory( long numBytes, long taskAttemptId, MemoryMode memoryMode){ available += numBytes; } public void markExecutionAsOutOfMemoryOnce(){ oomOnce = true; } public void limit(long avail){ available = avail; } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/row/UnsafeRowListTest.java ================================================ package io.mycat.memory.unsafe.row; import org.junit.Assert; import org.junit.Test; import java.util.ArrayList; /** * Created by zagnix on 2016/6/27. */ public class UnsafeRowListTest { @Test public void testUnsafeRowList(){ ArrayList list = new ArrayList(); UnsafeRow unsafeRow ; BufferHolder bufferHolder ; UnsafeRowWriter unsafeRowWriter; String line = "testUnsafeRow"; for (int i = 0; i <10; i++) { unsafeRow = new UnsafeRow(3); bufferHolder = new BufferHolder(unsafeRow); unsafeRowWriter = new UnsafeRowWriter(bufferHolder,3); bufferHolder.reset(); unsafeRow.setInt(0,89); unsafeRowWriter.write(1,line.getBytes(),0,line.length()); unsafeRow.setInt(2,23); unsafeRow.setTotalSize(bufferHolder.totalSize()); list.add(unsafeRow); } for (int i = 0; i <10; i++) { UnsafeRow row = list.get(i); row.setInt(0,1000+i); } for (int i = 0; i <10; i++) { UnsafeRow row = list.get(i); Assert.assertEquals(1000+i,row.getInt(0)); } } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/row/UnsafeRowSuite.java ================================================ package io.mycat.memory.unsafe.row; import junit.framework.Assert; import static org.junit.Assert.assertEquals; import java.math.BigDecimal; import org.junit.Test; /** * Created by zagnix on 2016/6/10. */ public class UnsafeRowSuite { @Test public void testUnsafeRowSingle(){ UnsafeRow unsafeRow = new UnsafeRow(5); BufferHolder bufferHolder = new BufferHolder(unsafeRow,64); UnsafeRowWriter unsafeRowWriter = new UnsafeRowWriter(bufferHolder,5); bufferHolder.reset(); String line2 = "testUnsafeRow3"; unsafeRow.setFloat(0, 7.4f); unsafeRow.setInt(1, 7); unsafeRow.setLong(2,455555); unsafeRowWriter.write(3,line2.getBytes(),0, line2.length()); unsafeRow.setNullAt(4); unsafeRow.setInt(1, 9); assert(unsafeRow.getFloat(0) == 7.4f); assert(unsafeRow.getInt(1) == 9); assert(unsafeRow.getLong(2) == 455555); Assert.assertEquals("testUnsafeRow3",new String(unsafeRow.getBinary(3))); assert (false==unsafeRow.isNullAt(3)); assert (true==unsafeRow.isNullAt(4)); } public void testUnsafeRowWithDecimal() { int fieldCount = 4; String value = "12345678901234567890123456789.0123456789"; String value1 = "100"; BigDecimal decimal = new BigDecimal(value); BigDecimal decimal1 = new BigDecimal(value1); System.out.println("decimal precision : " + decimal.precision() + ", scale : " + decimal.scale()); UnsafeRow unsafeRow = new UnsafeRow(fieldCount); BufferHolder bufferHolder = new BufferHolder(unsafeRow,64); UnsafeRowWriter unsafeRowWriter = new UnsafeRowWriter(bufferHolder,fieldCount); bufferHolder.reset(); unsafeRow.setInt(0, 100); unsafeRow.setDouble(1, 0.99); unsafeRow.setLong(2, 1000); unsafeRowWriter.write(3, decimal); assertEquals(100, unsafeRow.getInt(0)); assertEquals("0.99", String.valueOf(unsafeRow.getDouble(1))); assertEquals(1000, unsafeRow.getLong(2)); assertEquals(decimal, unsafeRow.getDecimal(3, decimal.scale())); unsafeRow.updateDecimal(3, decimal1); assertEquals(decimal1, unsafeRow.getDecimal(3, decimal1.scale())); // update null decimal BigDecimal nullDecimal = null; unsafeRow.updateDecimal(3, nullDecimal); assertEquals(nullDecimal, unsafeRow.getDecimal(3, 0)); unsafeRow.updateDecimal(3, decimal); assertEquals(decimal, unsafeRow.getDecimal(3, decimal.scale())); } // @Test // public void testUnsafeRowInsert(){ // UnsafeRow unsafeRow = new UnsafeRow(4); // // assert(unsafeRow.getFloat(0) == 7.4f); // assert(unsafeRow.getInt(1) == 9); // assert(unsafeRow.getLong(2) == 455555); // Assert.assertEquals("testUnsafeRow3",new String(unsafeRow.getBinary(3))); // } }; ================================================ FILE: src/test/java/io/mycat/memory/unsafe/sort/HashPartitioner.java ================================================ package io.mycat.memory.unsafe.sort; import io.mycat.memory.unsafe.utils.JavaUtils; /** * Created by zagnix on 2016/6/6. */ public class HashPartitioner { private int index =0; public HashPartitioner(int i) { this.index = i; } public int getPartition(String key){ return JavaUtils.nonNegativeMod(key.hashCode(), index); } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/sort/TestTimSort.java ================================================ /** * Copyright 2015 Stijn de Gouw * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ package io.mycat.memory.unsafe.sort; import java.util.ArrayList; import java.util.Arrays; import java.util.List; /** * This codes generates a int array which fails the standard TimSort. * * The blog that reported the bug * http://www.envisage-project.eu/timsort-specification-and-verification/ * * This codes was originally wrote by Stijn de Gouw, modified by Evan Yu to adapt to * our test suite. * * https://github.com/abstools/java-timsort-bug * https://github.com/abstools/java-timsort-bug/blob/master/LICENSE */ public class TestTimSort { private static final int MIN_MERGE = 32; /** * Returns an array of integers that demonstrate the bug in TimSort */ public static int[] getTimSortBugTestSet(int length) { int minRun = minRunLength(length); List runs = runsJDKWorstCase(minRun, length); return createArray(runs, length); } private static int minRunLength(int n) { int r = 0; // Becomes 1 if any 1 bits are shifted off while (n >= MIN_MERGE) { r |= (n & 1); n >>= 1; } return n + r; } private static int[] createArray(List runs, int length) { int[] a = new int[length]; Arrays.fill(a, 0); int endRun = -1; for (long len : runs) { a[endRun += len] = 1; } a[length - 1] = 0; return a; } /** * Fills runs with a sequence of run lengths of the form
* Y_n x_{n,1} x_{n,2} ... x_{n,l_n}
* Y_{n-1} x_{n-1,1} x_{n-1,2} ... x_{n-1,l_{n-1}}
* ...
* Y_1 x_{1,1} x_{1,2} ... x_{1,l_1}
* The Y_i's are chosen to satisfy the invariant throughout execution, * but the x_{i,j}'s are merged (by TimSort.mergeCollapse) * into an X_i that violates the invariant. * * @param length The sum of all run lengths that will be added to runs. */ private static List runsJDKWorstCase(int minRun, int length) { List runs = new ArrayList<>(); long runningTotal = 0, Y = minRun + 4, X = minRun; while (runningTotal + Y + X <= length) { runningTotal += X + Y; generateJDKWrongElem(runs, minRun, X); runs.add(0, Y); // X_{i+1} = Y_i + x_{i,1} + 1, since runs.get(1) = x_{i,1} X = Y + runs.get(1) + 1; // Y_{i+1} = X_{i+1} + Y_i + 1 Y += X + 1; } if (runningTotal + X <= length) { runningTotal += X; generateJDKWrongElem(runs, minRun, X); } runs.add(length - runningTotal); return runs; } /** * Adds a sequence x_1, ..., x_n of run lengths to runs such that:
* 1. X = x_1 + ... + x_n
* 2. x_j >= minRun for all j
* 3. x_1 + ... + x_{j-2} < x_j < x_1 + ... + x_{j-1} for all j
* These conditions guarantee that TimSort merges all x_j's one by one * (resulting in X) using only merges on the second-to-last element. * * @param X The sum of the sequence that should be added to runs. */ private static void generateJDKWrongElem(List runs, int minRun, long X) { for (long newTotal; X >= 2 * minRun + 1; X = newTotal) { //Default strategy newTotal = X / 2 + 1; //Specialized strategies if (3 * minRun + 3 <= X && X <= 4 * minRun + 1) { // add x_1=MIN+1, x_2=MIN, x_3=X-newTotal to runs newTotal = 2 * minRun + 1; } else if (5 * minRun + 5 <= X && X <= 6 * minRun + 5) { // add x_1=MIN+1, x_2=MIN, x_3=MIN+2, x_4=X-newTotal to runs newTotal = 3 * minRun + 3; } else if (8 * minRun + 9 <= X && X <= 10 * minRun + 9) { // add x_1=MIN+1, x_2=MIN, x_3=MIN+2, x_4=2MIN+2, x_5=X-newTotal to runs newTotal = 5 * minRun + 5; } else if (13 * minRun + 15 <= X && X <= 16 * minRun + 17) { // add x_1=MIN+1, x_2=MIN, x_3=MIN+2, x_4=2MIN+2, x_5=3MIN+4, x_6=X-newTotal to runs newTotal = 8 * minRun + 9; } runs.add(0, X - newTotal); } runs.add(0, X); } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/sort/UnsafeExternalRowSorterTest.java ================================================ package io.mycat.memory.unsafe.sort; import io.mycat.memory.MyCatMemory; import io.mycat.memory.unsafe.array.ByteArrayMethods; import io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager; import io.mycat.memory.unsafe.memory.mm.MemoryManager; import io.mycat.memory.unsafe.row.BufferHolder; import io.mycat.memory.unsafe.row.StructType; import io.mycat.memory.unsafe.row.UnsafeRow; import io.mycat.memory.unsafe.row.UnsafeRowWriter; import io.mycat.memory.unsafe.storage.DataNodeDiskManager; import io.mycat.memory.unsafe.storage.SerializerManager; import io.mycat.memory.unsafe.utils.BytesTools; import io.mycat.memory.unsafe.utils.MycatPropertyConf; import io.mycat.memory.unsafe.utils.sort.PrefixComparator; import io.mycat.memory.unsafe.utils.sort.PrefixComparators; import io.mycat.memory.unsafe.utils.sort.RowPrefixComputer; import io.mycat.memory.unsafe.utils.sort.UnsafeExternalRowSorter; import io.mycat.sqlengine.mpp.ColMeta; import io.mycat.sqlengine.mpp.OrderCol; import io.mycat.util.ExecutorUtil; import io.mycat.util.NameableExecutor; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.lang.reflect.Array; import java.util.*; import java.util.concurrent.Future; /** * Created by zagnix on 2016/6/19. */ public class UnsafeExternalRowSorterTest { private static final int TEST_SIZE = 100000; public static final Logger LOGGER = LoggerFactory.getLogger(UnsafeExternalRowSorterTest.class); /** * 测试类型 LONG，INT，SHORT,Float，Double，String，Binary * 经测试基数排序可以适用上述数据类型，大大提高排序速度 */ @Test public void testUnsafeExternalRowSorter() throws NoSuchFieldException, IllegalAccessException, IOException { MyCatMemory myCatMemory = new MyCatMemory(); MemoryManager memoryManager = myCatMemory.getResultMergeMemoryManager(); DataNodeDiskManager blockManager = myCatMemory.getBlockManager(); SerializerManager serializerManager = myCatMemory.getSerializerManager(); MycatPropertyConf conf = myCatMemory.getConf(); DataNodeMemoryManager dataNodeMemoryManager = new DataNodeMemoryManager(memoryManager, Thread.currentThread().getId()); /** * 1.schema ,模拟一个field字段值 * */ int fieldCount = 3; ColMeta colMeta = null; Map colMetaMap = new HashMap(fieldCount); colMeta = new ColMeta(0, ColMeta.COL_TYPE_STRING); colMetaMap.put("id", colMeta); colMeta = new ColMeta(1, ColMeta.COL_TYPE_STRING); colMetaMap.put("name", colMeta); colMeta = new ColMeta(2, ColMeta.COL_TYPE_STRING); colMetaMap.put("age", colMeta); OrderCol[] orderCols = new OrderCol[1]; OrderCol orderCol = new OrderCol(colMetaMap.get("id"), OrderCol.COL_ORDER_TYPE_ASC); orderCols[0] = orderCol; /** * 2 .PrefixComputer */ StructType schema = new StructType(colMetaMap, fieldCount); schema.setOrderCols(orderCols); UnsafeExternalRowSorter.PrefixComputer prefixComputer = new RowPrefixComputer(schema); /** * 3 .PrefixComparator 默认是ASC，可以选择DESC */ final PrefixComparator prefixComparator = PrefixComparators.LONG; UnsafeExternalRowSorter sorter = new UnsafeExternalRowSorter(dataNodeMemoryManager, myCatMemory, schema, prefixComparator, prefixComputer, conf.getSizeAsBytes("mycat.buffer.pageSize","1m"), true, /**使用基数排序？true or false*/ true); UnsafeRow unsafeRow; BufferHolder bufferHolder; UnsafeRowWriter unsafeRowWriter; String line = "testUnsafeRow"; // List floats = new ArrayList(); List longs = new ArrayList(); final Random rand = new Random(42); for (int i = 0; i < TEST_SIZE; i++) { unsafeRow = new UnsafeRow(3); bufferHolder = new BufferHolder(unsafeRow); unsafeRowWriter = new UnsafeRowWriter(bufferHolder,3); bufferHolder.reset(); String key = getRandomString(rand.nextInt(300)+100); //long v = rand.nextLong(); // longs.add(v); unsafeRowWriter.write(0,key.getBytes()); // unsafeRowWriter.write(0, BytesTools.toBytes(v)); unsafeRowWriter.write(1, line.getBytes()); unsafeRowWriter.write(2, ("35" + 1).getBytes()); unsafeRow.setTotalSize(bufferHolder.totalSize()); sorter.insertRow(unsafeRow); } Iterator iter = sorter.sort(); /* float [] com = new float[floats.size()]; for (int i = 0; i = 2); UnsafeExternalSorter.SpillableIterator iter = (UnsafeExternalSorter.SpillableIterator) sorter.getSortedIterator(); int lastv = 0; for (int i = 0; i < n / 3; i++) { iter.hasNext(); iter.loadNext(); assertTrue(Platform.getLong(iter.getBaseObject(), iter.getBaseOffset()) == i); lastv = i; } assertTrue(iter.spill() > 0); assertEquals(0, iter.spill()); assertTrue(Platform.getLong(iter.getBaseObject(), iter.getBaseOffset()) == lastv); for (int i = n / 3; i < n; i++) { iter.hasNext(); iter.loadNext(); assertEquals(i, Platform.getLong(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); } @Test public void forcedSpillingWithNotReadIterator() throws Exception { final UnsafeExternalSorter sorter = newSorter(); long[] record = new long[100]; int recordSize = record.length * 8; int n = (int) pageSizeBytes / recordSize * 3; for (int i = 0; i < n; i++) { record[0] = (long) i; sorter.insertRecord(record, Platform.LONG_ARRAY_OFFSET, recordSize, 0); } assertTrue(sorter.getNumberOfAllocatedPages() >= 2); UnsafeExternalSorter.SpillableIterator iter = (UnsafeExternalSorter.SpillableIterator) sorter.getSortedIterator(); assertTrue(iter.spill() > 0); assertEquals(0, iter.spill()); for (int i = 0; i < n; i++) { iter.hasNext(); iter.loadNext(); assertEquals(i, Platform.getLong(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); } @Test public void forcedSpillingWithoutComparator() throws Exception { final UnsafeExternalSorter sorter = UnsafeExternalSorter.create( DATA_NODE_MEMORY_MANAGER, blockManager, serializerManager, null, null, /* initialSize */ 1024, pageSizeBytes, shouldUseRadixSort(),true); long[] record = new long[100]; int recordSize = record.length * 8; int n = (int) pageSizeBytes / recordSize * 3; int batch = n / 4; for (int i = 0; i < n; i++) { record[0] = (long) i; sorter.insertRecord(record, Platform.LONG_ARRAY_OFFSET, recordSize, 0); if (i % batch == batch - 1) { sorter.spill(); } } UnsafeSorterIterator iter = sorter.getIterator(); for (int i = 0; i < n; i++) { iter.hasNext(); iter.loadNext(); assertEquals(i, Platform.getLong(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); } @Test public void testPeakMemoryUsed() throws Exception { final long recordLengthBytes = 8; final long pageSizeBytes = 256; final long numRecordsPerPage = pageSizeBytes / recordLengthBytes; final UnsafeExternalSorter sorter = UnsafeExternalSorter.create( DATA_NODE_MEMORY_MANAGER, blockManager, serializerManager, recordComparator, prefixComparator, 1024, pageSizeBytes, shouldUseRadixSort(),true); // Peak memory should be monotonically increasing. More specifically, every time // we allocate a new page it should increase by exactly the size of the page. long previousPeakMemory = sorter.getPeakMemoryUsedBytes(); long newPeakMemory; try { for (int i = 0; i < numRecordsPerPage * 10; i++) { insertNumber(sorter, i); newPeakMemory = sorter.getPeakMemoryUsedBytes(); if (i % numRecordsPerPage == 0) { // We allocated a new page for this record, so peak memory should change assertEquals(previousPeakMemory + pageSizeBytes, newPeakMemory); } else { assertEquals(previousPeakMemory, newPeakMemory); } previousPeakMemory = newPeakMemory; } // Spilling should not change peak memory sorter.spill(); newPeakMemory = sorter.getPeakMemoryUsedBytes(); assertEquals(previousPeakMemory, newPeakMemory); for (int i = 0; i < numRecordsPerPage; i++) { insertNumber(sorter, i); } newPeakMemory = sorter.getPeakMemoryUsedBytes(); assertEquals(previousPeakMemory, newPeakMemory); } finally { sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); } } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/sort/UnsafeInMemorySorterRadixSortSuite.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.sort; public class UnsafeInMemorySorterRadixSortSuite extends UnsafeInMemorySorterSuite { @Override protected boolean shouldUseRadixSort() { return true; } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/sort/UnsafeInMemorySorterSuite.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.sort; import io.mycat.memory.unsafe.Platform; import io.mycat.memory.unsafe.memory.MemoryBlock; import io.mycat.memory.unsafe.memory.TestMemoryConsumer; import io.mycat.memory.unsafe.memory.TestMemoryManager; import io.mycat.memory.unsafe.memory.mm.DataNodeMemoryManager; import io.mycat.memory.unsafe.utils.MycatPropertyConf; import io.mycat.memory.unsafe.utils.sort.*; import org.junit.Assert; import org.junit.Test; import java.nio.charset.StandardCharsets; import java.util.Arrays; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.isIn; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.mockito.Mockito.mock; public class UnsafeInMemorySorterSuite { protected boolean shouldUseRadixSort() { return true; } private static String getStringFromDataPage(Object baseObject,long baseOffset,int length) { final byte[] strBytes = new byte[length]; Platform.copyMemory(baseObject,baseOffset,strBytes, Platform.BYTE_ARRAY_OFFSET,length); return new String(strBytes,StandardCharsets.UTF_8); } @Test public void testSortingEmptyInput() { final DataNodeMemoryManager memoryManager = new DataNodeMemoryManager( new TestMemoryManager(new MycatPropertyConf().set("mycat.memory.offHeap.enabled", "false")), 0); final TestMemoryConsumer consumer = new TestMemoryConsumer(memoryManager); final UnsafeInMemorySorter sorter = new UnsafeInMemorySorter(consumer, memoryManager, mock(RecordComparator.class), mock(PrefixComparator.class), 100, shouldUseRadixSort(),true); final UnsafeSorterIterator iter = sorter.getSortedIterator(); Assert.assertFalse(iter.hasNext()); } @Test public void testSortingOnlyByIntegerPrefix() throws Exception { final String[] dataToSort = new String[] { "Boba", "Pearls", "Tapioca", "Taho", "Condensed Milk", "Jasmine", "Milk Tea", "Lychee", "Mango" }; final DataNodeMemoryManager memoryManager = new DataNodeMemoryManager( new TestMemoryManager(new MycatPropertyConf().set("mycat.memory.offHeap.enabled","false")), 0); final TestMemoryConsumer consumer = new TestMemoryConsumer(memoryManager); final MemoryBlock dataPage = memoryManager.allocatePage(2048, null); final Object baseObject = dataPage.getBaseObject(); // Write the records into the data page: long position = dataPage.getBaseOffset(); for (String str : dataToSort) { final byte[] strBytes = str.getBytes(StandardCharsets.UTF_8); Platform.putInt(baseObject, position, strBytes.length); position += 4; Platform.copyMemory(strBytes,Platform.BYTE_ARRAY_OFFSET,baseObject, position, strBytes.length); position += strBytes.length; } // Since the key fits within the 8-byte prefix, we don't need to do any record comparison, so // use a dummy comparator final RecordComparator recordComparator = new RecordComparator() { @Override public int compare( Object leftBaseObject, long leftBaseOffset, Object rightBaseObject, long rightBaseOffset) { return 0; } }; // Compute key prefixes based on the records' partition ids final HashPartitioner hashPartitioner = new HashPartitioner(4); // Use integer comparison for comparing prefixes (which are partition ids, in this case) final PrefixComparator prefixComparator = PrefixComparators.LONG; UnsafeInMemorySorter sorter = new UnsafeInMemorySorter( consumer,memoryManager,recordComparator, prefixComparator, dataToSort.length, shouldUseRadixSort(),true); // Given a page of records, insert those records into the sorter one-by-one: position = dataPage.getBaseOffset(); System.out.println("(0)address = " + position); for (int i = 0; i < dataToSort.length; i++) { if (!sorter.hasSpaceForAnotherRecord()) { sorter.expandPointerArray(consumer.allocateLongArray(sorter.getMemoryUsage() / 8 * 2)); } // position now points to the start of a record (which holds its length). final int recordLength = Platform.getInt(baseObject,position); final long address = memoryManager.encodePageNumberAndOffset(dataPage,position); final String str = getStringFromDataPage(baseObject,position+4,recordLength); final int partitionId = hashPartitioner.getPartition(str); System.out.println("(" + partitionId + "," + str + ")"); sorter.insertRecord(address,partitionId); position += 4 + recordLength; } final UnsafeSorterIterator iter = sorter.getSortedIterator(); int iterLength = 0; long prevPrefix = -1; Arrays.sort(dataToSort); while (iter.hasNext()) { iter.loadNext(); final String str = getStringFromDataPage(iter.getBaseObject(), iter.getBaseOffset(), iter.getRecordLength()); final long keyPrefix = iter.getKeyPrefix(); assertThat(str, isIn(Arrays.asList(dataToSort))); assertThat(keyPrefix, greaterThanOrEqualTo(prevPrefix)); prevPrefix = keyPrefix; iterLength++; } assertEquals(dataToSort.length, iterLength); } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/storage/BlockManagerTest.java ================================================ package io.mycat.memory.unsafe.storage; import com.google.common.io.Closeables; import io.mycat.memory.unsafe.utils.MycatPropertyConf; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; /** * Created by zagnix on 2016/6/4. */ public class BlockManagerTest { private static final Logger logger = LoggerFactory.getLogger(BlockManagerTest.class); @Test public void testNewDiskBlockManager() throws IOException { MycatPropertyConf conf = new MycatPropertyConf(); SerializerManager serializerManager = new SerializerManager(); DataNodeDiskManager blockManager = new DataNodeDiskManager(conf,true,serializerManager); DataNodeFileManager diskBlockManager = blockManager.diskBlockManager(); /** * 生成一个文本文件 */ File file = diskBlockManager.getFile("mycat1"); FileOutputStream fos = new FileOutputStream(file); BufferedOutputStream bos = new BufferedOutputStream(fos); bos.write("KOKKKKKK".getBytes()); bos.flush(); bos.close(); fos.close(); /** * 读刚刚写入的文件 */ File file1 = diskBlockManager.getFile("mycat1"); FileInputStream ios = new FileInputStream(file1); BufferedInputStream bin = new BufferedInputStream(ios); byte[] str = new byte["KOKKKKKK".getBytes().length]; int size = bin.read(str); bin.close(); ios.close(); Assert.assertEquals("KOKKKKKK",new String(str)); File file2 = diskBlockManager.getFile("mycat1"); DiskRowWriter writer = blockManager. getDiskWriter(null,file2,DummySerializerInstance.INSTANCE,1024*1024); byte [] writeBuffer = new byte[4]; int v =4; writeBuffer[0] = (byte)(v >>> 24); writeBuffer[1] = (byte)(v >>> 16); writeBuffer[2] = (byte)(v >>> 8); writeBuffer[3] = (byte)(v >>> 0); writer.write(writeBuffer,0,4); writer.write("you are ok? 1111111111111".getBytes(),0,"you are ok? 1111111111111".getBytes().length); writer.write("you are ok? 1111111111111".getBytes(),0,"you are ok? 1111111111111".getBytes().length); writer.write("you are ok? 1111111111111".getBytes(),0,"you are ok? 1111111111111".getBytes().length); writer.write("you are ok? 1111111111111".getBytes(),0,"you are ok? 1111111111111".getBytes().length); writer.write("you are ok? 1111111111111".getBytes(),0,"you are ok? 1111111111111".getBytes().length); writer.write("you are ok? 1111111111111".getBytes(),0,"you are ok? 1111111111111".getBytes().length); writer.write("you are ok? 1111111111111".getBytes(),0,"you are ok? 1111111111111".getBytes().length); writer.write("you are ok? 1111111111111".getBytes(),0,"you are ok? 1111111111111".getBytes().length); writer.close(); try { Thread.sleep(100); } catch (InterruptedException e) { logger.error(e.getMessage()); } assert (file2.length() > 0); final BufferedInputStream bs = new BufferedInputStream(new FileInputStream(file2)); try { InputStream in = serializerManager.wrapForCompression(null,bs); DataInputStream din= new DataInputStream(in); int numRecords = din.readInt(); Assert.assertEquals(4,numRecords); din.close(); in.close(); bs.close(); } catch (IOException e) { Closeables.close(bs, /* swallowIOException = */ true); throw e; } } @Test public void testNewDiskBlockWriter(){ MycatPropertyConf conf = new MycatPropertyConf(); SerializerManager serializerManager = new SerializerManager(); } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/storage/SerializerManagerTest.java ================================================ package io.mycat.memory.unsafe.storage; import org.junit.Assert; import org.junit.Test; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; /** * Created by zagnix on 2016/6/4. */ public class SerializerManagerTest { @Test public void testNewSerializerManager() throws IOException { SerializerManager serializerManager = new SerializerManager(); final int[] value = new int[1]; OutputStream s = serializerManager.wrapForCompression(null, new OutputStream() { @Override public void write(int b) throws IOException { value[0] = b; } }); s.write(10); Assert.assertEquals(10,value[0]); InputStream in = serializerManager.wrapForCompression(null, new InputStream() { @Override public int read() throws IOException { return 10; } }); Assert.assertEquals(10,in.read()); } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/types/CalendarIntervalSuite.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.mycat.memory.unsafe.types; import org.junit.Test; import static org.junit.Assert.*; public class CalendarIntervalSuite { @Test public void equalsTest() { CalendarInterval i1 = new CalendarInterval(3, 123); CalendarInterval i2 = new CalendarInterval(3, 321); CalendarInterval i3 = new CalendarInterval(1, 123); CalendarInterval i4 = new CalendarInterval(3, 123); assertNotSame(i1, i2); assertNotSame(i1, i3); assertNotSame(i2, i3); assertEquals(i1, i4); } @Test public void toStringTest() { CalendarInterval i; i = new CalendarInterval(34, 0); assertEquals("interval 2 years 10 months", i.toString()); i = new CalendarInterval(-34, 0); assertEquals("interval -2 years -10 months", i.toString()); i = new CalendarInterval(0, 3 * CalendarInterval.MICROS_PER_WEEK + 13 * CalendarInterval.MICROS_PER_HOUR + 123); assertEquals("interval 3 weeks 13 hours 123 microseconds", i.toString()); i = new CalendarInterval(0, -3 * CalendarInterval.MICROS_PER_WEEK - 13 * CalendarInterval.MICROS_PER_HOUR - 123); assertEquals("interval -3 weeks -13 hours -123 microseconds", i.toString()); i = new CalendarInterval(34, 3 * CalendarInterval.MICROS_PER_WEEK + 13 * CalendarInterval.MICROS_PER_HOUR + 123); assertEquals("interval 2 years 10 months 3 weeks 13 hours 123 microseconds", i.toString()); } @Test public void fromStringTest() { testSingleUnit("year", 3, 36, 0); testSingleUnit("month", 3, 3, 0); testSingleUnit("week", 3, 0, 3 * CalendarInterval.MICROS_PER_WEEK); testSingleUnit("day", 3, 0, 3 * CalendarInterval.MICROS_PER_DAY); testSingleUnit("hour", 3, 0, 3 * CalendarInterval.MICROS_PER_HOUR); testSingleUnit("minute", 3, 0, 3 *CalendarInterval. MICROS_PER_MINUTE); testSingleUnit("second", 3, 0, 3 * CalendarInterval.MICROS_PER_SECOND); testSingleUnit("millisecond", 3, 0, 3 *CalendarInterval. MICROS_PER_MILLI); testSingleUnit("microsecond", 3, 0, 3); String input; input = "interval -5 years 23 month"; CalendarInterval result = new CalendarInterval(-5 * 12 + 23, 0); assertEquals(CalendarInterval.fromString(input), result); input = "interval -5 years 23 month "; assertEquals(CalendarInterval.fromString(input), result); input = " interval -5 years 23 month "; assertEquals(CalendarInterval.fromString(input), result); // Error cases input = "interval 3month 1 hour"; assertNull(CalendarInterval.fromString(input)); input = "interval 3 moth 1 hour"; assertNull(CalendarInterval.fromString(input)); input = "interval"; assertNull(CalendarInterval.fromString(input)); input = "int"; assertNull(CalendarInterval.fromString(input)); input = ""; assertNull(CalendarInterval.fromString(input)); input = null; assertNull(CalendarInterval.fromString(input)); } @Test public void fromYearMonthStringTest() { String input; CalendarInterval i; input = "99-10"; i = new CalendarInterval(99 * 12 + 10, 0L); assertEquals(CalendarInterval.fromYearMonthString(input), i); input = "-8-10"; i = new CalendarInterval(-8 * 12 - 10, 0L); assertEquals(CalendarInterval.fromYearMonthString(input), i); try { input = "99-15"; CalendarInterval.fromYearMonthString(input); fail("Expected to throw an exception for the invalid input"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("month 15 outside range")); } } @Test public void fromDayTimeStringTest() { String input; CalendarInterval i; input = "5 12:40:30.999999999"; i = new CalendarInterval(0, 5 * CalendarInterval.MICROS_PER_DAY + 12 * CalendarInterval.MICROS_PER_HOUR + 40 *CalendarInterval. MICROS_PER_MINUTE + 30 *CalendarInterval. MICROS_PER_SECOND + 999999L); assertEquals(CalendarInterval.fromDayTimeString(input), i); input = "10 0:12:0.888"; i = new CalendarInterval(0, 10 * CalendarInterval.MICROS_PER_DAY + 12 * CalendarInterval.MICROS_PER_MINUTE); assertEquals(CalendarInterval.fromDayTimeString(input), i); input = "-3 0:0:0"; i = new CalendarInterval(0, -3 * CalendarInterval.MICROS_PER_DAY); assertEquals(CalendarInterval.fromDayTimeString(input), i); try { input = "5 30:12:20"; CalendarInterval.fromDayTimeString(input); fail("Expected to throw an exception for the invalid input"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("hour 30 outside range")); } try { input = "5 30-12"; CalendarInterval.fromDayTimeString(input); fail("Expected to throw an exception for the invalid input"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("not match day-time format")); } } @Test public void fromSingleUnitStringTest() { String input; CalendarInterval i; input = "12"; i = new CalendarInterval(12 * 12, 0L); assertEquals(CalendarInterval.fromSingleUnitString("year", input), i); input = "100"; i = new CalendarInterval(0, 100 * CalendarInterval.MICROS_PER_DAY); assertEquals(CalendarInterval.fromSingleUnitString("day", input), i); input = "1999.38888"; i = new CalendarInterval(0, 1999 *CalendarInterval. MICROS_PER_SECOND + 38); assertEquals(CalendarInterval.fromSingleUnitString("second", input), i); try { input = String.valueOf(Integer.MAX_VALUE); CalendarInterval.fromSingleUnitString("year", input); fail("Expected to throw an exception for the invalid input"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("outside range")); } try { input = String.valueOf(Long.MAX_VALUE / CalendarInterval.MICROS_PER_HOUR + 1); CalendarInterval.fromSingleUnitString("hour", input); fail("Expected to throw an exception for the invalid input"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("outside range")); } } @Test public void addTest() { String input = "interval 3 month 1 hour"; String input2 = "interval 2 month 100 hour"; CalendarInterval interval = CalendarInterval.fromString(input); CalendarInterval interval2 = CalendarInterval.fromString(input2); assertEquals(interval.add(interval2), new CalendarInterval(5, 101 * CalendarInterval.MICROS_PER_HOUR)); input = "interval -10 month -81 hour"; input2 = "interval 75 month 200 hour"; interval = CalendarInterval.fromString(input); interval2 = CalendarInterval.fromString(input2); assertEquals(interval.add(interval2), new CalendarInterval(65, 119 * CalendarInterval.MICROS_PER_HOUR)); } @Test public void subtractTest() { String input = "interval 3 month 1 hour"; String input2 = "interval 2 month 100 hour"; CalendarInterval interval = CalendarInterval.fromString(input); CalendarInterval interval2 = CalendarInterval.fromString(input2); assertEquals(interval.subtract(interval2), new CalendarInterval(1, -99 * CalendarInterval.MICROS_PER_HOUR)); input = "interval -10 month -81 hour"; input2 = "interval 75 month 200 hour"; interval = CalendarInterval.fromString(input); interval2 = CalendarInterval.fromString(input2); assertEquals(interval.subtract(interval2), new CalendarInterval(-85, -281 * CalendarInterval.MICROS_PER_HOUR)); } private static void testSingleUnit(String unit, int number, int months, long microseconds) { String input1 = "interval " + number + " " + unit; String input2 = "interval " + number + " " + unit + "s"; CalendarInterval result = new CalendarInterval(months, microseconds); assertEquals(CalendarInterval.fromString(input1), result); assertEquals(CalendarInterval.fromString(input2), result); } } ================================================ FILE: src/test/java/io/mycat/memory/unsafe/types/UTF8StringSuite.java ================================================ package io.mycat.memory.unsafe.types; /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import com.google.common.collect.ImmutableMap; import org.junit.Test; import java.io.UnsupportedEncodingException; import java.util.Arrays; import java.util.HashMap; import static org.junit.Assert.*; public class UTF8StringSuite { private static void checkBasic(String str, int len) throws UnsupportedEncodingException { UTF8String s1 = UTF8String.fromString(str); UTF8String s2 = UTF8String.fromBytes(str.getBytes("utf8")); assertEquals(s1.numChars(), len); assertEquals(s2.numChars(), len); assertEquals(s1.toString(), str); assertEquals(s2.toString(), str); assertEquals(s1, s2); assertEquals(s1.hashCode(), s2.hashCode()); assertEquals(0, s1.compareTo(s2)); assertTrue(s1.contains(s2)); assertTrue(s2.contains(s1)); assertTrue(s1.startsWith(s1)); assertTrue(s1.endsWith(s1)); } @Test public void basicTest() throws UnsupportedEncodingException { checkBasic("", 0); checkBasic("hello", 5); checkBasic("大 千 世 界", 7); } @Test public void emptyStringTest() { assertEquals(UTF8String.EMPTY_UTF8, UTF8String.fromString("")); assertEquals(UTF8String.EMPTY_UTF8, UTF8String.fromBytes(new byte[0])); assertEquals(0, UTF8String.EMPTY_UTF8.numChars()); assertEquals(0, UTF8String.EMPTY_UTF8.numBytes()); } @Test public void prefix() { assertTrue(UTF8String.fromString("a").getPrefix() - UTF8String.fromString("b").getPrefix() < 0); assertTrue(UTF8String.fromString("ab").getPrefix() - UTF8String.fromString("b").getPrefix() < 0); assertTrue( UTF8String.fromString("abbbbbbbbbbbasdf").getPrefix() - UTF8String.fromString("bbbbbbbbbbbbasdf").getPrefix() < 0); assertTrue(UTF8String.fromString("").getPrefix() - UTF8String.fromString("a").getPrefix() < 0); assertTrue(UTF8String.fromString("你好").getPrefix() - UTF8String.fromString("世界").getPrefix() > 0); byte[] buf1 = {1, 2, 3, 4, 5, 6, 7, 8, 9}; byte[] buf2 = {1, 2, 3}; UTF8String str1 = UTF8String.fromBytes(buf1, 0, 3); UTF8String str2 = UTF8String.fromBytes(buf1, 0, 8); UTF8String str3 = UTF8String.fromBytes(buf2); assertTrue(str1.getPrefix() - str2.getPrefix() < 0); assertEquals(str1.getPrefix(), str3.getPrefix()); } @Test public void compareTo() { assertTrue(UTF8String.fromString("").compareTo(UTF8String.fromString("a")) < 0); assertTrue(UTF8String.fromString("abc").compareTo(UTF8String.fromString("ABC")) > 0); assertTrue(UTF8String.fromString("abc0").compareTo(UTF8String.fromString("abc")) > 0); assertTrue(UTF8String.fromString("abcabcabc").compareTo(UTF8String.fromString("abcabcabc")) == 0); assertTrue(UTF8String.fromString("aBcabcabc").compareTo(UTF8String.fromString("Abcabcabc")) > 0); assertTrue(UTF8String.fromString("Abcabcabc").compareTo(UTF8String.fromString("abcabcabC")) < 0); assertTrue(UTF8String.fromString("abcabcabc").compareTo(UTF8String.fromString("abcabcabC")) > 0); assertTrue(UTF8String.fromString("abc").compareTo(UTF8String.fromString("世界")) < 0); assertTrue(UTF8String.fromString("你好").compareTo(UTF8String.fromString("世界")) > 0); assertTrue(UTF8String.fromString("你好123").compareTo(UTF8String.fromString("你好122")) > 0); } protected static void testUpperandLower(String upper, String lower) { UTF8String us = UTF8String.fromString(upper); UTF8String ls = UTF8String.fromString(lower); assertEquals(ls, us.toLowerCase()); assertEquals(us, ls.toUpperCase()); assertEquals(us, us.toUpperCase()); assertEquals(ls, ls.toLowerCase()); } @Test public void upperAndLower() { testUpperandLower("", ""); testUpperandLower("0123456", "0123456"); testUpperandLower("ABCXYZ", "abcxyz"); testUpperandLower("ЀЁЂѺΏỀ", "ѐёђѻώề"); testUpperandLower("大千世界 数据砖头", "大千世界 数据砖头"); } @Test public void titleCase() { assertEquals(UTF8String.fromString(""), UTF8String.fromString("").toTitleCase()); assertEquals(UTF8String.fromString("Ab Bc Cd"), UTF8String.fromString("ab bc cd").toTitleCase()); assertEquals(UTF8String.fromString("Ѐ Ё Ђ Ѻ Ώ Ề"), UTF8String.fromString("ѐ ё ђ ѻ ώ ề").toTitleCase()); assertEquals(UTF8String.fromString("大千世界 数据砖头"), UTF8String.fromString("大千世界 数据砖头").toTitleCase()); } @Test public void concatTest() { assertEquals(UTF8String.EMPTY_UTF8, UTF8String.concat()); assertNull(UTF8String.concat((UTF8String) null)); assertEquals(UTF8String.EMPTY_UTF8, UTF8String.concat(UTF8String.EMPTY_UTF8)); assertEquals(UTF8String.fromString("ab"), UTF8String.concat(UTF8String.fromString("ab"))); assertEquals(UTF8String.fromString("ab"), UTF8String.concat(UTF8String.fromString("a"), UTF8String.fromString("b"))); assertEquals(UTF8String.fromString("abc"), UTF8String.concat(UTF8String.fromString("a"), UTF8String.fromString("b"), UTF8String.fromString("c"))); assertNull(UTF8String.concat(UTF8String.fromString("a"), null, UTF8String.fromString("c"))); assertNull(UTF8String.concat(UTF8String.fromString("a"), null, null)); assertNull(UTF8String.concat(null, null, null)); assertEquals(UTF8String.fromString("数据砖头"), UTF8String.concat(UTF8String.fromString("数据"), UTF8String.fromString("砖头"))); } @Test public void concatWsTest() { // Returns null if the separator is null assertNull(UTF8String.concatWs(null, (UTF8String) null)); assertNull(UTF8String.concatWs(null, UTF8String.fromString("a"))); // If separator is null, concatWs should skip all null inputs and never return null. UTF8String sep = UTF8String.fromString("哈哈"); assertEquals( UTF8String.EMPTY_UTF8, UTF8String.concatWs(sep, UTF8String.EMPTY_UTF8)); assertEquals( UTF8String.fromString("ab"), UTF8String.concatWs(sep, UTF8String.fromString("ab"))); assertEquals( UTF8String.fromString("a哈哈b"), UTF8String.concatWs(sep, UTF8String.fromString("a"), UTF8String.fromString("b"))); assertEquals( UTF8String.fromString("a哈哈b哈哈c"), UTF8String.concatWs(sep, UTF8String.fromString("a"), UTF8String.fromString("b"), UTF8String.fromString("c"))); assertEquals( UTF8String.fromString("a哈哈c"), UTF8String.concatWs(sep, UTF8String.fromString("a"), null, UTF8String.fromString("c"))); assertEquals( UTF8String.fromString("a"), UTF8String.concatWs(sep, UTF8String.fromString("a"), null, null)); assertEquals( UTF8String.EMPTY_UTF8, UTF8String.concatWs(sep, null, null, null)); assertEquals( UTF8String.fromString("数据哈哈砖头"), UTF8String.concatWs(sep, UTF8String.fromString("数据"), UTF8String.fromString("砖头"))); } @Test public void contains() { assertTrue(UTF8String.EMPTY_UTF8.contains(UTF8String.EMPTY_UTF8)); assertTrue(UTF8String.fromString("hello").contains(UTF8String.fromString("ello"))); assertFalse(UTF8String.fromString("hello").contains(UTF8String.fromString("vello"))); assertFalse(UTF8String.fromString("hello").contains(UTF8String.fromString("hellooo"))); assertTrue(UTF8String.fromString("大千世界").contains(UTF8String.fromString("千世界"))); assertFalse(UTF8String.fromString("大千世界").contains(UTF8String.fromString("世千"))); assertFalse(UTF8String.fromString("大千世界").contains(UTF8String.fromString("大千世界好"))); } @Test public void startsWith() { assertTrue(UTF8String.EMPTY_UTF8.startsWith(UTF8String.EMPTY_UTF8)); assertTrue(UTF8String.fromString("hello").startsWith(UTF8String.fromString("hell"))); assertFalse(UTF8String.fromString("hello").startsWith(UTF8String.fromString("ell"))); assertFalse(UTF8String.fromString("hello").startsWith(UTF8String.fromString("hellooo"))); assertTrue(UTF8String.fromString("数据砖头").startsWith(UTF8String.fromString("数据"))); assertFalse(UTF8String.fromString("大千世界").startsWith(UTF8String.fromString("千"))); assertFalse(UTF8String.fromString("大千世界").startsWith(UTF8String.fromString("大千世界好"))); } @Test public void endsWith() { assertTrue(UTF8String.EMPTY_UTF8.endsWith(UTF8String.EMPTY_UTF8)); assertTrue(UTF8String.fromString("hello").endsWith(UTF8String.fromString("ello"))); assertFalse(UTF8String.fromString("hello").endsWith(UTF8String.fromString("ellov"))); assertFalse(UTF8String.fromString("hello").endsWith(UTF8String.fromString("hhhello"))); assertTrue(UTF8String.fromString("大千世界").endsWith(UTF8String.fromString("世界"))); assertFalse(UTF8String.fromString("大千世界").endsWith(UTF8String.fromString("世"))); assertFalse(UTF8String.fromString("数据砖头").endsWith(UTF8String.fromString("我的数据砖头"))); } @Test public void substring() { assertEquals(UTF8String.EMPTY_UTF8, UTF8String.fromString("hello").substring(0, 0)); assertEquals(UTF8String.fromString("el"), UTF8String.fromString("hello").substring(1, 3)); assertEquals(UTF8String.fromString("数"), UTF8String.fromString("数据砖头").substring(0, 1)); assertEquals(UTF8String.fromString("据砖"), UTF8String.fromString("数据砖头").substring(1, 3)); assertEquals(UTF8String.fromString("头"), UTF8String.fromString("数据砖头").substring(3, 5)); assertEquals(UTF8String.fromString("ߵ梷"), UTF8String.fromString("ߵ梷").substring(0, 2)); } @Test public void trims() { assertEquals(UTF8String.fromString("hello"), UTF8String.fromString(" hello ").trim()); assertEquals(UTF8String.fromString("hello "), UTF8String.fromString(" hello ").trimLeft()); assertEquals(UTF8String.fromString(" hello"), UTF8String.fromString(" hello ").trimRight()); assertEquals(UTF8String.EMPTY_UTF8, UTF8String.fromString(" ").trim()); assertEquals(UTF8String.EMPTY_UTF8, UTF8String.fromString(" ").trimLeft()); assertEquals(UTF8String.EMPTY_UTF8, UTF8String.fromString(" ").trimRight()); assertEquals(UTF8String.fromString("数据砖头"), UTF8String.fromString(" 数据砖头 ").trim()); assertEquals(UTF8String.fromString("数据砖头 "), UTF8String.fromString(" 数据砖头 ").trimLeft()); assertEquals(UTF8String.fromString(" 数据砖头"), UTF8String.fromString(" 数据砖头 ").trimRight()); assertEquals(UTF8String.fromString("数据砖头"), UTF8String.fromString("数据砖头").trim()); assertEquals(UTF8String.fromString("数据砖头"), UTF8String.fromString("数据砖头").trimLeft()); assertEquals(UTF8String.fromString("数据砖头"), UTF8String.fromString("数据砖头").trimRight()); } @Test public void indexOf() { assertEquals(0, UTF8String.EMPTY_UTF8.indexOf(UTF8String.EMPTY_UTF8, 0)); assertEquals(-1, UTF8String.EMPTY_UTF8.indexOf(UTF8String.fromString("l"), 0)); assertEquals(0, UTF8String.fromString("hello").indexOf(UTF8String.EMPTY_UTF8, 0)); assertEquals(2, UTF8String.fromString("hello").indexOf(UTF8String.fromString("l"), 0)); assertEquals(3, UTF8String.fromString("hello").indexOf(UTF8String.fromString("l"), 3)); assertEquals(-1, UTF8String.fromString("hello").indexOf(UTF8String.fromString("a"), 0)); assertEquals(2, UTF8String.fromString("hello").indexOf(UTF8String.fromString("ll"), 0)); assertEquals(-1, UTF8String.fromString("hello").indexOf(UTF8String.fromString("ll"), 4)); assertEquals(1, UTF8String.fromString("数据砖头").indexOf(UTF8String.fromString("据砖"), 0)); assertEquals(-1, UTF8String.fromString("数据砖头").indexOf(UTF8String.fromString("数"), 3)); assertEquals(0, UTF8String.fromString("数据砖头").indexOf(UTF8String.fromString("数"), 0)); assertEquals(3, UTF8String.fromString("数据砖头").indexOf(UTF8String.fromString("头"), 0)); } @Test public void substring_index() { assertEquals(UTF8String.fromString("www.apache.org"), UTF8String.fromString("www.apache.org").subStringIndex(UTF8String.fromString("."), 3)); assertEquals(UTF8String.fromString("www.apache"), UTF8String.fromString("www.apache.org").subStringIndex(UTF8String.fromString("."), 2)); assertEquals(UTF8String.fromString("www"), UTF8String.fromString("www.apache.org").subStringIndex(UTF8String.fromString("."), 1)); assertEquals(UTF8String.fromString(""), UTF8String.fromString("www.apache.org").subStringIndex(UTF8String.fromString("."), 0)); assertEquals(UTF8String.fromString("org"), UTF8String.fromString("www.apache.org").subStringIndex(UTF8String.fromString("."), -1)); assertEquals(UTF8String.fromString("apache.org"), UTF8String.fromString("www.apache.org").subStringIndex(UTF8String.fromString("."), -2)); assertEquals(UTF8String.fromString("www.apache.org"), UTF8String.fromString("www.apache.org").subStringIndex(UTF8String.fromString("."), -3)); // str is empty string assertEquals(UTF8String.fromString(""), UTF8String.fromString("").subStringIndex(UTF8String.fromString("."), 1)); // empty string delim assertEquals(UTF8String.fromString(""), UTF8String.fromString("www.apache.org").subStringIndex(UTF8String.fromString(""), 1)); // delim does not exist in str assertEquals(UTF8String.fromString("www.apache.org"), UTF8String.fromString("www.apache.org").subStringIndex(UTF8String.fromString("#"), 2)); // delim is 2 chars assertEquals(UTF8String.fromString("www||apache"), UTF8String.fromString("www||apache||org").subStringIndex(UTF8String.fromString("||"), 2)); assertEquals(UTF8String.fromString("apache||org"), UTF8String.fromString("www||apache||org").subStringIndex(UTF8String.fromString("||"), -2)); // non ascii chars assertEquals(UTF8String.fromString("大千世界大"), UTF8String.fromString("大千世界大千世界").subStringIndex(UTF8String.fromString("千"), 2)); // overlapped delim assertEquals(UTF8String.fromString("||"), UTF8String.fromString("||||||").subStringIndex(UTF8String.fromString("|||"), 3)); assertEquals(UTF8String.fromString("|||"), UTF8String.fromString("||||||").subStringIndex(UTF8String.fromString("|||"), -4)); } @Test public void reverse() { assertEquals(UTF8String.fromString("olleh"), UTF8String.fromString("hello").reverse()); assertEquals(UTF8String.EMPTY_UTF8, UTF8String.EMPTY_UTF8.reverse()); assertEquals(UTF8String.fromString("者行孙"), UTF8String.fromString("孙行者").reverse()); assertEquals(UTF8String.fromString("者行孙 olleh"), UTF8String.fromString("hello 孙行者").reverse()); } @Test public void repeat() { assertEquals(UTF8String.fromString("数d数d数d数d数d"), UTF8String.fromString("数d").repeat(5)); assertEquals(UTF8String.fromString("数d"), UTF8String.fromString("数d").repeat(1)); assertEquals(UTF8String.EMPTY_UTF8, UTF8String.fromString("数d").repeat(-1)); } @Test public void pad() { assertEquals(UTF8String.fromString("hel"), UTF8String.fromString("hello").lpad(3, UTF8String.fromString("????"))); assertEquals(UTF8String.fromString("hello"), UTF8String.fromString("hello").lpad(5, UTF8String.fromString("????"))); assertEquals(UTF8String.fromString("?hello"), UTF8String.fromString("hello").lpad(6, UTF8String.fromString("????"))); assertEquals(UTF8String.fromString("???????hello"), UTF8String.fromString("hello").lpad(12, UTF8String.fromString("????"))); assertEquals(UTF8String.fromString("?????hello"), UTF8String.fromString("hello").lpad(10, UTF8String.fromString("?????"))); assertEquals(UTF8String.fromString("???????"), UTF8String.EMPTY_UTF8.lpad(7, UTF8String.fromString("?????"))); assertEquals(UTF8String.fromString("hel"), UTF8String.fromString("hello").rpad(3, UTF8String.fromString("????"))); assertEquals(UTF8String.fromString("hello"), UTF8String.fromString("hello").rpad(5, UTF8String.fromString("????"))); assertEquals(UTF8String.fromString("hello?"), UTF8String.fromString("hello").rpad(6, UTF8String.fromString("????"))); assertEquals(UTF8String.fromString("hello???????"), UTF8String.fromString("hello").rpad(12, UTF8String.fromString("????"))); assertEquals(UTF8String.fromString("hello?????"), UTF8String.fromString("hello").rpad(10, UTF8String.fromString("?????"))); assertEquals(UTF8String.fromString("???????"), UTF8String.EMPTY_UTF8.rpad(7, UTF8String.fromString("?????"))); assertEquals(UTF8String.fromString("数据砖"), UTF8String.fromString("数据砖头").lpad(3, UTF8String.fromString("????"))); assertEquals(UTF8String.fromString("?数据砖头"), UTF8String.fromString("数据砖头").lpad(5, UTF8String.fromString("????"))); assertEquals(UTF8String.fromString("??数据砖头"), UTF8String.fromString("数据砖头").lpad(6, UTF8String.fromString("????"))); assertEquals(UTF8String.fromString("孙行数据砖头"), UTF8String.fromString("数据砖头").lpad(6, UTF8String.fromString("孙行者"))); assertEquals(UTF8String.fromString("孙行者数据砖头"), UTF8String.fromString("数据砖头").lpad(7, UTF8String.fromString("孙行者"))); assertEquals( UTF8String.fromString("孙行者孙行者孙行数据砖头"), UTF8String.fromString("数据砖头").lpad(12, UTF8String.fromString("孙行者"))); assertEquals(UTF8String.fromString("数据砖"), UTF8String.fromString("数据砖头").rpad(3, UTF8String.fromString("????"))); assertEquals(UTF8String.fromString("数据砖头?"), UTF8String.fromString("数据砖头").rpad(5, UTF8String.fromString("????"))); assertEquals(UTF8String.fromString("数据砖头??"), UTF8String.fromString("数据砖头").rpad(6, UTF8String.fromString("????"))); assertEquals(UTF8String.fromString("数据砖头孙行"), UTF8String.fromString("数据砖头").rpad(6, UTF8String.fromString("孙行者"))); assertEquals(UTF8String.fromString("数据砖头孙行者"), UTF8String.fromString("数据砖头").rpad(7, UTF8String.fromString("孙行者"))); assertEquals( UTF8String.fromString("数据砖头孙行者孙行者孙行"), UTF8String.fromString("数据砖头").rpad(12, UTF8String.fromString("孙行者"))); assertEquals(UTF8String.EMPTY_UTF8, UTF8String.fromString("数据砖头").lpad(-10, UTF8String.fromString("孙行者"))); assertEquals(UTF8String.EMPTY_UTF8, UTF8String.fromString("数据砖头").lpad(-10, UTF8String.EMPTY_UTF8)); assertEquals(UTF8String.fromString("数据砖头"), UTF8String.fromString("数据砖头").lpad(5, UTF8String.EMPTY_UTF8)); assertEquals(UTF8String.fromString("数据砖"), UTF8String.fromString("数据砖头").lpad(3, UTF8String.EMPTY_UTF8)); assertEquals(UTF8String.EMPTY_UTF8, UTF8String.EMPTY_UTF8.lpad(3, UTF8String.EMPTY_UTF8)); assertEquals(UTF8String.EMPTY_UTF8, UTF8String.fromString("数据砖头").rpad(-10, UTF8String.fromString("孙行者"))); assertEquals(UTF8String.EMPTY_UTF8, UTF8String.fromString("数据砖头").rpad(-10, UTF8String.EMPTY_UTF8)); assertEquals(UTF8String.fromString("数据砖头"), UTF8String.fromString("数据砖头").rpad(5, UTF8String.EMPTY_UTF8)); assertEquals(UTF8String.fromString("数据砖"), UTF8String.fromString("数据砖头").rpad(3, UTF8String.EMPTY_UTF8)); assertEquals(UTF8String.EMPTY_UTF8, UTF8String.EMPTY_UTF8.rpad(3, UTF8String.EMPTY_UTF8)); } @Test public void substringSQL() { UTF8String e = UTF8String.fromString("example"); assertEquals(e.substringSQL(0, 2), UTF8String.fromString("ex")); assertEquals(e.substringSQL(1, 2), UTF8String.fromString("ex")); assertEquals(e.substringSQL(0, 7), UTF8String.fromString("example")); assertEquals(e.substringSQL(1, 2), UTF8String.fromString("ex")); assertEquals(e.substringSQL(0, 100), UTF8String.fromString("example")); assertEquals(e.substringSQL(1, 100), UTF8String.fromString("example")); assertEquals(e.substringSQL(2, 2), UTF8String.fromString("xa")); assertEquals(e.substringSQL(1, 6), UTF8String.fromString("exampl")); assertEquals(e.substringSQL(2, 100), UTF8String.fromString("xample")); assertEquals(e.substringSQL(0, 0), UTF8String.fromString("")); assertEquals(e.substringSQL(100, 4), UTF8String.EMPTY_UTF8); assertEquals(e.substringSQL(0, Integer.MAX_VALUE), UTF8String.fromString("example")); assertEquals(e.substringSQL(1, Integer.MAX_VALUE), UTF8String.fromString("example")); assertEquals(e.substringSQL(2, Integer.MAX_VALUE), UTF8String.fromString("xample")); } @Test public void split() { assertTrue(Arrays.equals(UTF8String.fromString("ab,def,ghi").split(UTF8String.fromString(","), -1), new UTF8String[]{UTF8String.fromString("ab"), UTF8String.fromString("def"), UTF8String.fromString("ghi")})); assertTrue(Arrays.equals(UTF8String.fromString("ab,def,ghi").split(UTF8String.fromString(","), 2), new UTF8String[]{UTF8String.fromString("ab"), UTF8String.fromString("def,ghi")})); assertTrue(Arrays.equals(UTF8String.fromString("ab,def,ghi").split(UTF8String.fromString(","), 2), new UTF8String[]{UTF8String.fromString("ab"), UTF8String.fromString("def,ghi")})); } @Test public void levenshteinDistance() { assertEquals(0, UTF8String.EMPTY_UTF8.levenshteinDistance(UTF8String.EMPTY_UTF8)); assertEquals(1, UTF8String.EMPTY_UTF8.levenshteinDistance(UTF8String.fromString("a"))); assertEquals(7, UTF8String.fromString("aaapppp").levenshteinDistance(UTF8String.EMPTY_UTF8)); assertEquals(1, UTF8String.fromString("frog").levenshteinDistance(UTF8String.fromString("fog"))); assertEquals(3, UTF8String.fromString("fly").levenshteinDistance(UTF8String.fromString("ant"))); assertEquals(7, UTF8String.fromString("elephant").levenshteinDistance(UTF8String.fromString("hippo"))); assertEquals(7, UTF8String.fromString("hippo").levenshteinDistance(UTF8String.fromString("elephant"))); assertEquals(8, UTF8String.fromString("hippo").levenshteinDistance(UTF8String.fromString("zzzzzzzz"))); assertEquals(1, UTF8String.fromString("hello").levenshteinDistance(UTF8String.fromString("hallo"))); assertEquals(4, UTF8String.fromString("世界千世").levenshteinDistance(UTF8String.fromString("千a世b"))); } @Test public void translate() { assertEquals( UTF8String.fromString("1a2s3ae"), UTF8String.fromString("translate").translate(ImmutableMap.of( 'r', '1', 'n', '2', 'l', '3', 't', '\0' ))); assertEquals( UTF8String.fromString("translate"), UTF8String.fromString("translate").translate(new HashMap())); assertEquals( UTF8String.fromString("asae"), UTF8String.fromString("translate").translate(ImmutableMap.of( 'r', '\0', 'n', '\0', 'l', '\0', 't', '\0' ))); assertEquals( UTF8String.fromString("aa世b"), UTF8String.fromString("花花世界").translate(ImmutableMap.of( '花', 'a', '界', 'b' ))); } @Test public void createBlankString() { assertEquals(UTF8String.fromString(" "), UTF8String.blankString(1)); assertEquals(UTF8String.fromString(" "), UTF8String.blankString(2)); assertEquals(UTF8String.fromString(" "), UTF8String.blankString(3)); assertEquals(UTF8String.fromString(""), UTF8String.blankString(0)); } @Test public void findInSet() { assertEquals(1, UTF8String.fromString("ab").findInSet(UTF8String.fromString("ab"))); assertEquals(2, UTF8String.fromString("a,b").findInSet(UTF8String.fromString("b"))); assertEquals(3, UTF8String.fromString("abc,b,ab,c,def").findInSet(UTF8String.fromString("ab"))); assertEquals(1, UTF8String.fromString("ab,abc,b,ab,c,def").findInSet(UTF8String.fromString("ab"))); assertEquals(4, UTF8String.fromString(",,,ab,abc,b,ab,c,def").findInSet(UTF8String.fromString("ab"))); assertEquals(1, UTF8String.fromString(",ab,abc,b,ab,c,def").findInSet(UTF8String.fromString(""))); assertEquals(4, UTF8String.fromString("数据砖头,abc,b,ab,c,def").findInSet(UTF8String.fromString("ab"))); assertEquals(6, UTF8String.fromString("数据砖头,abc,b,ab,c,def").findInSet(UTF8String.fromString("def"))); } @Test public void soundex() { assertEquals(UTF8String.fromString("Robert").soundex(), UTF8String.fromString("R163")); assertEquals(UTF8String.fromString("Rupert").soundex(), UTF8String.fromString("R163")); assertEquals(UTF8String.fromString("Rubin").soundex(), UTF8String.fromString("R150")); assertEquals(UTF8String.fromString("Ashcraft").soundex(), UTF8String.fromString("A261")); assertEquals(UTF8String.fromString("Ashcroft").soundex(), UTF8String.fromString("A261")); assertEquals(UTF8String.fromString("Burroughs").soundex(), UTF8String.fromString("B620")); assertEquals(UTF8String.fromString("Burrows").soundex(), UTF8String.fromString("B620")); assertEquals(UTF8String.fromString("Ekzampul").soundex(), UTF8String.fromString("E251")); assertEquals(UTF8String.fromString("Example").soundex(), UTF8String.fromString("E251")); assertEquals(UTF8String.fromString("Ellery").soundex(), UTF8String.fromString("E460")); assertEquals(UTF8String.fromString("Euler").soundex(), UTF8String.fromString("E460")); assertEquals(UTF8String.fromString("Ghosh").soundex(), UTF8String.fromString("G200")); assertEquals(UTF8String.fromString("Gauss").soundex(), UTF8String.fromString("G200")); assertEquals(UTF8String.fromString("Gutierrez").soundex(), UTF8String.fromString("G362")); assertEquals(UTF8String.fromString("Heilbronn").soundex(), UTF8String.fromString("H416")); assertEquals(UTF8String.fromString("Hilbert").soundex(), UTF8String.fromString("H416")); assertEquals(UTF8String.fromString("Jackson").soundex(), UTF8String.fromString("J250")); assertEquals(UTF8String.fromString("Kant").soundex(), UTF8String.fromString("K530")); assertEquals(UTF8String.fromString("Knuth").soundex(), UTF8String.fromString("K530")); assertEquals(UTF8String.fromString("Lee").soundex(), UTF8String.fromString("L000")); assertEquals(UTF8String.fromString("Lukasiewicz").soundex(), UTF8String.fromString("L222")); assertEquals(UTF8String.fromString("Lissajous").soundex(), UTF8String.fromString("L222")); assertEquals(UTF8String.fromString("Ladd").soundex(), UTF8String.fromString("L300")); assertEquals(UTF8String.fromString("Lloyd").soundex(), UTF8String.fromString("L300")); assertEquals(UTF8String.fromString("Moses").soundex(), UTF8String.fromString("M220")); assertEquals(UTF8String.fromString("O'Hara").soundex(), UTF8String.fromString("O600")); assertEquals(UTF8String.fromString("Pfister").soundex(), UTF8String.fromString("P236")); assertEquals(UTF8String.fromString("Rubin").soundex(), UTF8String.fromString("R150")); assertEquals(UTF8String.fromString("Robert").soundex(), UTF8String.fromString("R163")); assertEquals(UTF8String.fromString("Rupert").soundex(), UTF8String.fromString("R163")); assertEquals(UTF8String.fromString("Soundex").soundex(), UTF8String.fromString("S532")); assertEquals(UTF8String.fromString("Sownteks").soundex(), UTF8String.fromString("S532")); assertEquals(UTF8String.fromString("Tymczak").soundex(), UTF8String.fromString("T522")); assertEquals(UTF8String.fromString("VanDeusen").soundex(), UTF8String.fromString("V532")); assertEquals(UTF8String.fromString("Washington").soundex(), UTF8String.fromString("W252")); assertEquals(UTF8String.fromString("Wheaton").soundex(), UTF8String.fromString("W350")); assertEquals(UTF8String.fromString("a").soundex(), UTF8String.fromString("A000")); assertEquals(UTF8String.fromString("ab").soundex(), UTF8String.fromString("A100")); assertEquals(UTF8String.fromString("abc").soundex(), UTF8String.fromString("A120")); assertEquals(UTF8String.fromString("abcd").soundex(), UTF8String.fromString("A123")); assertEquals(UTF8String.fromString("").soundex(), UTF8String.fromString("")); assertEquals(UTF8String.fromString("123").soundex(), UTF8String.fromString("123")); assertEquals(UTF8String.fromString("世界千世").soundex(), UTF8String.fromString("世界千世")); } } ================================================ FILE: src/test/java/io/mycat/migrate/MigrateUtilsTest.java ================================================ package io.mycat.migrate; import com.google.common.collect.Lists; import io.mycat.migrate.MigrateTask; import io.mycat.migrate.MigrateUtils; import org.junit.Assert; import org.junit.Test; import java.io.ByteArrayInputStream; import java.util.*; import static io.mycat.route.function.PartitionByCRC32PreSlot.Range; /** * Created by magicdoom on 2016/9/16. */ public class MigrateUtilsTest { @Test public void balanceExpand() { String table="test"; Map> integerListMap = new TreeMap<>(); integerListMap.put(0,Lists.newArrayList(new Range(0,32))) ; integerListMap.put(1,Lists.newArrayList(new Range(33,65))) ; integerListMap.put(2,Lists.newArrayList(new Range(66,99))) ; pringList("beforse balance :",integerListMap); //dn1=0-32 dn2=33-65 dn3=66-99 int totalSlots=100; List oldDataNodes = Lists.newArrayList("dn1","dn2","dn3"); List newDataNodes = Lists.newArrayList("dn4","dn5"); Map> tasks= MigrateUtils .balanceExpand(table, integerListMap, oldDataNodes, newDataNodes,totalSlots); for (Map.Entry> stringListEntry : tasks.entrySet()) { String key=stringListEntry.getKey(); List rangeList=new ArrayList<>(); List value=stringListEntry.getValue(); for (MigrateTask task : value) { rangeList.addAll(task.getSlots()); } Assert.assertEquals(true,value.size()==2); if("dn4".equals(key)) { Assert.assertEquals(0, rangeList.get(0).start); Assert.assertEquals(12, rangeList.get(0).end); Assert.assertEquals(33, rangeList.get(1).start); Assert.assertEquals(39, rangeList.get(1).end); } else if("dn5".equals(key)) { Assert.assertEquals(40, rangeList.get(0).start); Assert.assertEquals(45, rangeList.get(0).end); Assert.assertEquals(66, rangeList.get(1).start); Assert.assertEquals(79, rangeList.get(1).end); } integerListMap.put(Integer.parseInt(key.substring(2))-1,rangeList); } pringList("after balance :",integerListMap); System.out.println("agin balance ....................."); oldDataNodes = Lists.newArrayList("dn1","dn2","dn3","dn4","dn5"); newDataNodes = Lists.newArrayList("dn6","dn7","dn8","dn9"); Map> tasks1= MigrateUtils.balanceExpand(table, integerListMap, oldDataNodes, newDataNodes,totalSlots); for (Map.Entry> stringListEntry : tasks1.entrySet()) { String key=stringListEntry.getKey(); List rangeList=new ArrayList<>(); List value=stringListEntry.getValue(); for (MigrateTask task : value) { rangeList.addAll(task.getSlots()); } if("dn6".equals(key)) { Assert.assertEquals(13, rangeList.get(0).start); Assert.assertEquals(21, rangeList.get(0).end); Assert.assertEquals(46, rangeList.get(1).start); Assert.assertEquals(48, rangeList.get(1).end); } else if("dn7".equals(key)) { Assert.assertEquals(49, rangeList.get(0).start); Assert.assertEquals(54, rangeList.get(0).end); Assert.assertEquals(80, rangeList.get(1).start); Assert.assertEquals(84, rangeList.get(1).end); } else if("dn8".equals(key)) { Assert.assertEquals(85, rangeList.get(0).start); Assert.assertEquals(88, rangeList.get(0).end); Assert.assertEquals(0, rangeList.get(1).start); Assert.assertEquals(6, rangeList.get(1).end); } else if("dn9".equals(key)) { Assert.assertEquals(7, rangeList.get(0).start); Assert.assertEquals(8, rangeList.get(0).end); Assert.assertEquals(40, rangeList.get(1).start); Assert.assertEquals(45, rangeList.get(1).end); } integerListMap.put(Integer.parseInt(key.substring(2))-1,rangeList); } pringList("agin balance :",integerListMap); oldDataNodes = Lists.newArrayList("dn1","dn2","dn3","dn4","dn5","dn6","dn7","dn8","dn9"); newDataNodes = Lists.newArrayList("dn10","dn11","dn12","dn13","dn14","dn15","dn16","dn17","dn18","dn19","dn20","dn21","dn22","dn23","dn24","dn25","dn26","dn27","dn28","dn29","dn30","dn31","dn32","dn33","dn34","dn35","dn36","dn37","dn38","dn39","dn40","dn41","dn42","dn43","dn44","dn45","dn46","dn47","dn48","dn49","dn50","dn51","dn52","dn53","dn54","dn55","dn56","dn57","dn58","dn59","dn60","dn61","dn62","dn63","dn64","dn65","dn66","dn67","dn68","dn69","dn70","dn71","dn72","dn73","dn74","dn75","dn76","dn77","dn78","dn79","dn80","dn81","dn82","dn83","dn84","dn85","dn86","dn87","dn88","dn89","dn90","dn91","dn92","dn93","dn94","dn95","dn96","dn97","dn98","dn99","dn100" ); Map> tasks2= MigrateUtils.balanceExpand(table, integerListMap, oldDataNodes, newDataNodes,totalSlots); for (Map.Entry> stringListEntry : tasks2.entrySet()) { String key=stringListEntry.getKey(); List rangeList=new ArrayList<>(); List value=stringListEntry.getValue(); for (MigrateTask task : value) { rangeList.addAll(task.getSlots()); } if("dn10".equals(key)) { Assert.assertEquals(22, rangeList.get(0).start); Assert.assertEquals(22, rangeList.get(0).end); } else if("dn100".equals(key)) { Assert.assertEquals(67, rangeList.get(0).start); Assert.assertEquals(67, rangeList.get(0).end); } else if("dn50".equals(key)) { Assert.assertEquals(69, rangeList.get(0).start); Assert.assertEquals(69, rangeList.get(0).end); } else if("dn99".equals(key)) { Assert.assertEquals(66, rangeList.get(0).start); Assert.assertEquals(66, rangeList.get(0).end); } integerListMap.put(Integer.parseInt(key.substring(2))-1,rangeList); } pringList("agin agin balance :",integerListMap); } @Test public void balanceExpand1() { String table = "test1"; //4=81920-102399 // 3=61440-81919 // 2=40960-61439 // 1=20480-40959 // 0=0-20479 Map> integerListMap = new TreeMap<>(); integerListMap.put(0, Lists.newArrayList(new Range(0, 20479))); integerListMap.put(1, Lists.newArrayList(new Range(20480, 40959))); integerListMap.put(2, Lists.newArrayList(new Range(40960, 61439))); integerListMap.put(3, Lists.newArrayList(new Range(61440, 81919))); integerListMap.put(4, Lists.newArrayList(new Range(81920, 102399))); pringList("beforse balance :", integerListMap); //dn1=0-32 dn2=33-65 dn3=66-99 int totalSlots = 102400; List oldDataNodes = Lists.newArrayList("dn1", "dn2", "dn3","dn4", "dn5"); List newDataNodes = Lists.newArrayList("dn6", "dn7", "dn8","dn9", "dn10"); Map> tasks = MigrateUtils .balanceExpand(table, integerListMap, oldDataNodes, newDataNodes, totalSlots); List allTaskList=new ArrayList<>(); for (Map.Entry> stringListEntry : tasks.entrySet()) { String key=stringListEntry.getKey(); List rangeList=new ArrayList<>(); List value=stringListEntry.getValue(); allTaskList.addAll(value); for (MigrateTask task : value) { rangeList.addAll(task.getSlots()); } integerListMap.put(Integer.parseInt(key.substring(2))-1,rangeList); } pringList("after balance :", integerListMap); List allNewDataNodes=new ArrayList<>(); allNewDataNodes.addAll(oldDataNodes); allNewDataNodes.addAll(newDataNodes); Properties prop = new Properties(); prop.put("0","0-20479"); prop.put("1","20480-40959"); prop.put("2","40960-61439"); prop.put("3","61440-81919"); prop.put("4","81920-102399"); for (MigrateTask migrateTask : allTaskList) { modifyRuleData(prop,migrateTask,allNewDataNodes); } System.out.println(); } private void modifyRuleData( Properties prop ,MigrateTask task ,List allNewDataNodes){ int fromIndex=-1; int toIndex=-1; List dataNodes= allNewDataNodes; for (int i = 0; i < dataNodes.size(); i++) { String dataNode = dataNodes.get(i); if(dataNode.equalsIgnoreCase(task.getFrom())){ fromIndex=i; } else if(dataNode.equalsIgnoreCase(task.getTo())){ toIndex=i; } } String from= prop.getProperty(String.valueOf(fromIndex)) ; String to= prop.getProperty(String.valueOf(toIndex)) ; String fromRemain=removeRangesRemain(from,task.getSlots()); String taskRanges = MigrateUtils.convertRangeListToString(task.getSlots()); String newTo=to==null? taskRanges : to+","+taskRanges; prop.setProperty(String.valueOf(fromIndex),fromRemain); prop.setProperty(String.valueOf(toIndex),newTo); } private String removeRangesRemain(String ori,List rangeList){ List ranges=MigrateUtils.convertRangeStringToList(ori); List ramain= MigrateUtils.removeAndGetRemain(ranges,rangeList); return MigrateUtils.convertRangeListToString(ramain); } private void pringList(String comm,Map> integerListMap) { System.out.println(comm); for (Map.Entry> integerListEntry : integerListMap.entrySet()) { Integer key=integerListEntry.getKey(); List value=integerListEntry.getValue(); System.out.println(key+":"+listToString(value)+":"+MigrateUtils.getCurTotalSize(value)); } } private String listToString(List rangeList) { String rtn=""; for (Range range : rangeList) { rtn=rtn+range.start+"-"+range.end+","; } return rtn; } @Test public void removeAndGetRemain(){ List oldRangeList1=Lists.newArrayList(new Range(0,51199)); List newRangeList1=Lists.newArrayList(new Range(0,20479),new Range(20480,30719)); List result1=MigrateUtils.removeAndGetRemain(oldRangeList1,newRangeList1); Assert.assertEquals(1,result1.size()); Assert.assertEquals(30720,result1.get(0).start); Assert.assertEquals(51199,result1.get(0).end); List oldRangeList2=Lists.newArrayList(new Range(51200,102399)); List newRangeList2=Lists.newArrayList(new Range(61440,81919),new Range(51200,61439)); List result2=MigrateUtils.removeAndGetRemain(oldRangeList2,newRangeList2); Assert.assertEquals(1,result2.size()); Assert.assertEquals(81920,result2.get(0).start); Assert.assertEquals(102399,result2.get(0).end); } @Test public void removeAndGetRemain1(){ List oldRangeList1=Lists.newArrayList(new Range(0,0),new Range(1,5),new Range(6,40000),new Range(40001,51199)); List newRangeList1=Lists.newArrayList(new Range(0,3),new Range(20480,30719)); List result1=MigrateUtils.removeAndGetRemain(oldRangeList1,newRangeList1); Assert.assertEquals(4,result1.size()); Assert.assertEquals(4,result1.get(0).start); Assert.assertEquals(5,result1.get(0).end); Assert.assertEquals(6,result1.get(1).start); Assert.assertEquals(20479,result1.get(1).end); Assert.assertEquals(30720,result1.get(2).start); Assert.assertEquals(40000,result1.get(2).end); Assert.assertEquals(40001,result1.get(3).start); Assert.assertEquals(51199,result1.get(3).end); } } ================================================ FILE: src/test/java/io/mycat/model/M1.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.model; import java.util.concurrent.BlockingQueue; import jsr166y.LinkedTransferQueue; /** * @author mycat */ public class M1 { private long count; private final BlockingQueue x; private final BlockingQueue y; public M1() { this.x = new LinkedTransferQueue(); this.y = new LinkedTransferQueue(); } public long getCount() { return count; } public BlockingQueue getX() { return x; } public BlockingQueue getY() { return y; } public void start() { new Thread(new A(), "A").start(); new Thread(new B(), "B").start(); new Thread(new C(), "C").start(); } private final class A implements Runnable { @Override public void run() { for (;;) { try { Thread.sleep(200L); } catch (InterruptedException e) { } for (int i = 0; i < 1000000; i++) { x.offer(new TransferObject()); } } } } private final class B implements Runnable { @Override public void run() { TransferObject t = null; for (;;) { try { t = x.take(); } catch (InterruptedException e) { continue; } t.handle(); y.offer(t); } } } private final class C implements Runnable { @Override public void run() { TransferObject t = null; for (;;) { try { t = y.take(); } catch (InterruptedException e) { continue; } t.compelete(); count++; } } } } ================================================ FILE: src/test/java/io/mycat/model/M1Main.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.model; /** * @author mycat */ public class M1Main { public static void main(String[] args) { final M1 m1 = new M1(); m1.start(); new Thread() { @Override public void run() { for (;;) { long c = m1.getCount(); try { Thread.sleep(2000L); } catch (InterruptedException e) { continue; } System.out.println("tps:" + (m1.getCount() - c) / 2); System.out.println(" x:" + m1.getX().size()); System.out.println(" y:" + m1.getY().size()); System.out.println("=============="); } } }.start(); } } ================================================ FILE: src/test/java/io/mycat/model/M2.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.model; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import io.mycat.util.ExecutorUtil; import jsr166y.LinkedTransferQueue; /** * @author mycat */ public class M2 { private long count; private final ThreadPoolExecutor x; private final BlockingQueue y; public M2() { this.x = ExecutorUtil.create("B", 1); this.y = new LinkedTransferQueue(); } public long getCount() { return count; } public ThreadPoolExecutor getX() { return x; } public BlockingQueue getY() { return y; } public void start() { new Thread(new A(), "A").start(); new Thread(new C(), "C").start(); } private final class A implements Runnable { @Override public void run() { for (;;) { try { Thread.sleep(200L); } catch (InterruptedException e) { } for (int i = 0; i < 1000000; i++) { final TransferObject t = new TransferObject(); x.execute(new Runnable() { @Override public void run() { t.handle(); y.offer(t); } }); } } } } private final class C implements Runnable { @Override public void run() { TransferObject t = null; for (;;) { try { t = y.take(); } catch (InterruptedException e) { continue; } t.compelete(); count++; } } } } ================================================ FILE: src/test/java/io/mycat/model/M2Main.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.model; /** * @author mycat */ public class M2Main { public static void main(String[] args) { final M2 m2 = new M2(); m2.start(); new Thread() { @Override public void run() { for (;;) { long c = m2.getCount(); try { Thread.sleep(2000L); } catch (InterruptedException e) { continue; } System.out.println("tps:" + (m2.getCount() - c) / 2); System.out.println(" x:" + m2.getX().getQueue().size()); System.out.println(" y:" + m2.getY().size()); System.out.println("=============="); } } }.start(); } } ================================================ FILE: src/test/java/io/mycat/model/TransferObject.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.model; /** * @author mycat */ public class TransferObject { long handleCount; long compeleteCount; public void handle() { handleCount++; } public void compelete() { compeleteCount++; } } ================================================ FILE: src/test/java/io/mycat/mpp/TestSorter.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.mpp; import org.junit.Assert; import org.junit.Test; import io.mycat.util.ByteUtil; public class TestSorter { @Test public void testDecimal() { String d1 = "-1223.000"; byte[] d1b = d1.getBytes(); Assert.assertEquals(true, -1223.0 == ByteUtil.getDouble(d1b)); d1b = "-99999.890".getBytes(); Assert.assertEquals(true, -99999.890 == ByteUtil.getDouble(d1b)); // 221346.000 byte[] data2 = new byte[] { 50, 50, 49, 51, 52, 54, 46, 48, 48, 48 }; Assert.assertEquals(true, 221346.000 == ByteUtil.getDouble(data2)); // 1234567890 byte[] data3 = new byte[] { 49, 50, 51, 52, 53, 54, 55, 56, 57, 48 }; Assert.assertEquals(true, 1234567890 == ByteUtil.getInt(data3)); // 0123456789 byte[] data4 = new byte[] { 48, 49, 50, 51, 52, 53, 54, 55, 56, 57 }; Assert.assertEquals(true, 123456789 == ByteUtil.getInt(data4)); } @Test public void testNumberCompare() { byte[] b1 = "0".getBytes(); byte[] b2 = "0".getBytes(); Assert.assertEquals(true, ByteUtil.compareNumberByte(b1, b2) == 0); b1 = "0".getBytes(); b2 = "1".getBytes(); Assert.assertEquals(true, ByteUtil.compareNumberByte(b1, b2)< 0); b1 = "10".getBytes(); b2 = "1".getBytes(); Assert.assertEquals(true, ByteUtil.compareNumberByte(b1, b2)> 0); b1 = "100.0".getBytes(); b2 = "100.0".getBytes(); Assert.assertEquals(true, ByteUtil.compareNumberByte(b1, b2)==0); b1 = "100.000".getBytes(); b2 = "100.0".getBytes(); Assert.assertEquals(true, ByteUtil.compareNumberByte(b1, b2)>0); b1 = "-100.000".getBytes(); b2 = "-100.0".getBytes(); Assert.assertEquals(true, ByteUtil.compareNumberByte(b1, b2)<0); b1 = "-100.001".getBytes(); b2 = "-100.0".getBytes(); Assert.assertEquals(true, ByteUtil.compareNumberByte(b1, b2)<0); b1 = "-100.001".getBytes(); b2 = "100.0".getBytes(); Assert.assertEquals(true, ByteUtil.compareNumberByte(b1, b2)<0); b1 = "90".getBytes(); b2 = "10000".getBytes(); Assert.assertEquals(true, ByteUtil.compareNumberByte(b1, b2)<0); b1 = "-90".getBytes(); b2 = "-10000".getBytes(); Assert.assertEquals(true, ByteUtil.compareNumberByte(b1, b2)>0); b1 = "98".getBytes(); b2 = "98000".getBytes(); Assert.assertEquals(true, ByteUtil.compareNumberByte(b1, b2)<0); b1 = "-98".getBytes(); b2= "-98000".getBytes(); Assert.assertEquals(true, ByteUtil.compareNumberByte(b1, b2)>0); b1="12002585786".getBytes(); b2="12002585785".getBytes(); Assert.assertEquals(true, ByteUtil.compareNumberByte(b1, b2)>0); } } ================================================ FILE: src/test/java/io/mycat/mysql/MySQLMessageTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.mysql; import org.junit.Assert; import org.junit.Test; import io.mycat.backend.mysql.MySQLMessage; /** * @author mycat */ public class MySQLMessageTest { @Test public void testReadBytesWithNull() { byte[] bytes = new byte[] { 1, 2, 3, 0, 5 }; MySQLMessage message = new MySQLMessage(bytes); byte[] ab = message.readBytesWithNull(); Assert.assertEquals(3, ab.length); Assert.assertEquals(4, message.position()); } @Test public void testReadBytesWithNull2() { byte[] bytes = new byte[] { 0, 1, 2, 3, 0, 5 }; MySQLMessage message = new MySQLMessage(bytes); byte[] ab = message.readBytesWithNull(); Assert.assertEquals(0, ab.length); Assert.assertEquals(1, message.position()); } @Test public void testReadBytesWithNull3() { byte[] bytes = new byte[] {}; MySQLMessage message = new MySQLMessage(bytes); byte[] ab = message.readBytesWithNull(); Assert.assertEquals(0, ab.length); Assert.assertEquals(0, message.position()); } } ================================================ FILE: src/test/java/io/mycat/mysql/ResultSetPacketParse.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.mysql; import io.mycat.util.FormatUtil; import io.mycat.util.SplitUtil; /** * @author mycat */ public class ResultSetPacketParse { public static String parse(String src) { String[] sa = SplitUtil.split(src, ',', true); StringBuilder s = new StringBuilder(); for (int i = 0; i < sa.length;) { int length = Byte.parseByte(sa[i++]) & 0xff; length |= (Byte.parseByte(sa[i++]) & 0xff) << 8; length |= (Byte.parseByte(sa[i++]) & 0xff) << 16; s.append("Length=").append(FormatUtil.format(length, 3)).append(','); s.append("Id=").append(Byte.parseByte(sa[i++])).append(':'); for (int x = 0; x < length; x++) { s.append(' ').append(sa[i++]); } s.append('\n'); } return s.toString(); } static String s = "1, 0, 0, 1, 1, 68, 0, 0, 2, 3, 100, 101, 102, 22, 99, 111, 98, 97, 114, 95, 116, 101, 115, 116, 95, 99, 111, 110, 110, 95, 98, 105, 110, 100, 95, 49, 2, 116, 49, 2, 116, 49, 10, 114, 101, 97, 100, 69, 114, 114, 67, 111, 108, 10, 114, 101, 97, 100, 69, 114, 114, 67, 111, 108, 12, 63, 0, 11, 0, 0, 0, 3, 0, 0, 0, 0, 0, 5, 0, 0, 3, -2, 0, 0, 34, 0, 4, 0, 0, 4, 3, 49, 50, 51, 46, 0, 0, 5, -1, 30, 4, 85, 110, 107, 110, 111, 119, 110, 32, 99, 111, 108, 117, 109, 110, 32, 39, 114, 101, 97, 100, 69, 114, 114, 67, 111, 108, 39, 32, 105, 110, 32, 39, 102, 105, 101, 108, 100, 32, 108, 105, 115, 116, 39"; static String s2 = "1, 0, 0, 1, 1, 68, 0, 0, 2, 3, 100, 101, 102, 22, 99, 111, 98, 97, 114, 95, 116, 101, 115, 116, 95, 99, 111, 110, 110, 95, 98, 105, 110, 100, 95, 49, 2, 116, 49, 2, 116, 49, 10, 114, 101, 97, 100, 69, 114, 114, 67, 111, 108, 10, 114, 101, 97, 100, 69, 114, 114, 67, 111, 108, 12, 63, 0, 11, 0, 0, 0, 3, 0, 0, 0, 0, 0, 5, 0, 0, 3, -2, 0, 0, 34, 0, 4, 0, 0, 4, 3, 49, 50, 51, 4, 0, 0, 5, 3, 49, 50, 51, 46, 0, 0, 6, -1, 30, 4, 85, 110, 107, 110, 111, 119, 110, 32, 99, 111, 108, 117, 109, 110, 32, 39, 114, 101, 97, 100, 69, 114, 114, 67, 111, 108, 39, 32, 105, 110, 32, 39, 102, 105, 101, 108, 100, 32, 108, 105, 115, 116, 39"; static String s3 = "1, 0, 0, 1, 1, 46, 0, 0, 1, -1, 30, 4, 85, 110, 107, 110, 111, 119, 110, 32, 99, 111, 108, 117, 109, 110, 32, 39, 114, 101, 97, 100, 69, 114, 114, 67, 111, 108, 39, 32, 105, 110, 32, 39, 102, 105, 101, 108, 100, 32, 108, 105, 115, 116, 39"; static String s4 = "1, 0, 0, 1, 1, 68, 0, 0, 2, 3, 100, 101, 102, 22, 99, 111, 98, 97, 114, 95, 116, 101, 115, 116, 95, 99, 111, 110, 110, 95, 98, 105, 110, 100, 95, 49, 2, 116, 49, 2, 116, 49, 10, 114, 101, 97, 100, 69, 114, 114, 67, 111, 108, 10, 114, 101, 97, 100, 69, 114, 114, 67, 111, 108, 12, 63, 0, 11, 0, 0, 0, 3, 0, 0, 0, 0, 0, 5, 0, 0, 3, -2, 0, 0, 34, 0, 4, 0, 0, 4, 3, 49, 50, 51, 4, 0, 0, 5, 3, 49, 50, 51, 46, 0, 0, 6, -1, 30, 4, 85, 110, 107, 110, 111, 119, 110, 32, 99, 111, 108, 117, 109, 110, 32, 39, 114, 101, 97, 100, 69, 114, 114, 67, 111, 108, 39, 32, 105, 110, 32, 39, 102, 105, 101, 108, 100, 32, 108, 105, 115, 116, 39"; static String s5 = "1, 0, 0, 1, 1, 1, 0, 0, 2, 1, 46, 0, 0, 3, -1, 30, 4, 85, 110, 107, 110, 111, 119, 110, 32, 99, 111, 108, 117, 109, 110, 32, 39, 114, 101, 97, 100, 69, 114, 114, 67, 111, 108, 39, 32, 105, 110, 32, 39, 102, 105, 101, 108, 100, 32, 108, 105, 115, 116, 39"; public static void main(String[] args) { System.out.println(ResultSetPacketParse.parse(s)); System.out.println(ResultSetPacketParse.parse(s2)); System.out.println(ResultSetPacketParse.parse(s3)); System.out.println(ResultSetPacketParse.parse(s4)); System.out.println(ResultSetPacketParse.parse(s5)); } } ================================================ FILE: src/test/java/io/mycat/parser/ManagerParserTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.parser; import org.junit.Assert; import org.junit.Test; import io.mycat.route.parser.ManagerParse; import io.mycat.route.parser.ManagerParseClear; import io.mycat.route.parser.ManagerParseReload; import io.mycat.route.parser.ManagerParseRollback; import io.mycat.route.parser.ManagerParseShow; import io.mycat.route.parser.ManagerParseStop; /** * @author mycat */ public class ManagerParserTest { @Test public void testIsSelect() { Assert.assertEquals(ManagerParse.SELECT, 0xff & ManagerParse.parse("select * from offer limit 1")); Assert.assertEquals(ManagerParse.SELECT, 0xff & ManagerParse.parse("SELECT * FROM OFFER LIMIT 1")); Assert.assertEquals(ManagerParse.SELECT, 0xff & ManagerParse.parse("SELECT * FROM OFFER limit 1")); } @Test public void testIsSet() { Assert.assertEquals(ManagerParse.SET, ManagerParse.parse("set names utf8")); Assert.assertEquals(ManagerParse.SET, ManagerParse.parse("SET NAMES UTF8")); Assert.assertEquals(ManagerParse.SET, ManagerParse.parse("set NAMES utf8")); } @Test public void testIsShow() { Assert.assertEquals(ManagerParse.SHOW, 0xff & ManagerParse.parse("show databases")); Assert.assertEquals(ManagerParse.SHOW, 0xff & ManagerParse.parse("SHOW DATABASES")); Assert.assertEquals(ManagerParse.SHOW, 0xff & ManagerParse.parse("SHOW databases")); } @Test public void testShowCommand() { Assert.assertEquals(ManagerParseShow.COMMAND, ManagerParseShow.parse("show @@command", 5)); Assert.assertEquals(ManagerParseShow.COMMAND, ManagerParseShow.parse("SHOW @@COMMAND", 5)); Assert.assertEquals(ManagerParseShow.COMMAND, ManagerParseShow.parse("show @@COMMAND", 5)); } @Test public void testShowConnection() { Assert.assertEquals(ManagerParseShow.CONNECTION, ManagerParseShow.parse("show @@connection", 5)); Assert.assertEquals(ManagerParseShow.CONNECTION, ManagerParseShow.parse("SHOW @@CONNECTION", 5)); Assert.assertEquals(ManagerParseShow.CONNECTION, ManagerParseShow.parse("show @@CONNECTION", 5)); } @Test public void testShowConnectionSQL() { Assert.assertEquals(ManagerParseShow.CONNECTION_SQL, ManagerParseShow.parse("show @@connection.sql", 5)); Assert.assertEquals(ManagerParseShow.CONNECTION_SQL, ManagerParseShow.parse("SHOW @@CONNECTION.SQL", 5)); Assert.assertEquals(ManagerParseShow.CONNECTION_SQL, ManagerParseShow.parse("show @@CONNECTION.Sql", 5)); } @Test public void testShowDatabase() { Assert.assertEquals(ManagerParseShow.DATABASE, ManagerParseShow.parse("show @@database", 5)); Assert.assertEquals(ManagerParseShow.DATABASE, ManagerParseShow.parse("SHOW @@DATABASE", 5)); Assert.assertEquals(ManagerParseShow.DATABASE, ManagerParseShow.parse("show @@DATABASE", 5)); } @Test public void testShowDataNode() { Assert.assertEquals(ManagerParseShow.DATANODE, ManagerParseShow.parse("show @@datanode", 5)); Assert.assertEquals(ManagerParseShow.DATANODE, ManagerParseShow.parse("SHOW @@DATANODE", 5)); Assert.assertEquals(ManagerParseShow.DATANODE, ManagerParseShow.parse("show @@DATANODE", 5)); Assert.assertEquals(ManagerParseShow.DATANODE, ManagerParseShow.parse("show @@DATANODE ", 5)); Assert.assertEquals(ManagerParseShow.DATANODE_WHERE, 0xff & ManagerParseShow.parse("show @@DATANODE WHERE SCHEMA=1", 5)); Assert.assertEquals(ManagerParseShow.DATANODE_WHERE, 0xff & ManagerParseShow.parse("show @@DATANODE WHERE schema =1", 5)); Assert.assertEquals(ManagerParseShow.DATANODE_WHERE, 0xff & ManagerParseShow.parse("show @@DATANODE WHERE SCHEMA= 1", 5)); } @Test public void testShowDataSource() { Assert.assertEquals(ManagerParseShow.DATASOURCE, ManagerParseShow.parse("show @@datasource", 5)); Assert.assertEquals(ManagerParseShow.DATASOURCE, ManagerParseShow.parse("SHOW @@DATASOURCE", 5)); Assert.assertEquals(ManagerParseShow.DATASOURCE, ManagerParseShow.parse(" show @@DATASOURCE ", 5)); Assert.assertEquals(ManagerParseShow.DATASOURCE, ManagerParseShow.parse(" show @@DATASOURCE ", 5)); Assert.assertEquals(ManagerParseShow.DATASOURCE_WHERE, 0xff & ManagerParseShow.parse(" show @@DATASOURCE where datanode = 1", 5)); Assert.assertEquals(ManagerParseShow.DATASOURCE_WHERE, 0xff & ManagerParseShow.parse(" show @@DATASOURCE where datanode=1", 5)); Assert.assertEquals(ManagerParseShow.DATASOURCE_WHERE, 0xff & ManagerParseShow.parse(" show @@DATASOURCE WHERE datanode = 1", 5)); Assert.assertEquals(ManagerParseShow.DATASOURCE_WHERE, 0xff & ManagerParseShow.parse(" show @@DATASOURCE where DATAnode= 1 ", 5)); } @Test public void testShowHelp() { Assert.assertEquals(ManagerParseShow.HELP, ManagerParseShow.parse("show @@help", 5)); Assert.assertEquals(ManagerParseShow.HELP, ManagerParseShow.parse("SHOW @@HELP", 5)); Assert.assertEquals(ManagerParseShow.HELP, ManagerParseShow.parse("show @@HELP", 5)); } @Test public void testShowHeartbeat() { Assert.assertEquals(ManagerParseShow.HEARTBEAT, ManagerParseShow.parse("show @@heartbeat", 5)); Assert.assertEquals(ManagerParseShow.HEARTBEAT, ManagerParseShow.parse("SHOW @@hearTBeat ", 5)); Assert.assertEquals(ManagerParseShow.HEARTBEAT, ManagerParseShow.parse(" show @@HEARTBEAT ", 6)); } @Test public void testShowParser() { Assert.assertEquals(ManagerParseShow.PARSER, ManagerParseShow.parse("show @@parser", 5)); Assert.assertEquals(ManagerParseShow.PARSER, ManagerParseShow.parse("SHOW @@PARSER", 5)); Assert.assertEquals(ManagerParseShow.PARSER, ManagerParseShow.parse("show @@PARSER", 5)); } @Test public void testShowProcessor() { Assert.assertEquals(ManagerParseShow.PROCESSOR, ManagerParseShow.parse("show @@processor", 5)); Assert.assertEquals(ManagerParseShow.PROCESSOR, ManagerParseShow.parse("SHOW @@PROCESSOR", 5)); Assert.assertEquals(ManagerParseShow.PROCESSOR, ManagerParseShow.parse("show @@PROCESSOR", 5)); } @Test public void testShowRouter() { Assert.assertEquals(ManagerParseShow.ROUTER, ManagerParseShow.parse("show @@router", 5)); Assert.assertEquals(ManagerParseShow.ROUTER, ManagerParseShow.parse("SHOW @@ROUTER", 5)); Assert.assertEquals(ManagerParseShow.ROUTER, ManagerParseShow.parse("show @@ROUTER", 5)); } @Test public void testShowServer() { Assert.assertEquals(ManagerParseShow.SERVER, ManagerParseShow.parse("show @@server", 5)); Assert.assertEquals(ManagerParseShow.SERVER, ManagerParseShow.parse("SHOW @@SERVER", 5)); Assert.assertEquals(ManagerParseShow.SERVER, ManagerParseShow.parse("show @@SERVER", 5)); } @Test public void testShowThreadPool() { Assert.assertEquals(ManagerParseShow.THREADPOOL, ManagerParseShow.parse("show @@threadPool", 5)); Assert.assertEquals(ManagerParseShow.THREADPOOL, ManagerParseShow.parse("SHOW @@THREADPOOL", 5)); Assert.assertEquals(ManagerParseShow.THREADPOOL, ManagerParseShow.parse("show @@THREADPOOL", 5)); } @Test public void testShowBackend() { Assert.assertEquals(ManagerParseShow.BACKEND, ManagerParseShow.parse("show @@backend", 5)); Assert.assertEquals(ManagerParseShow.BACKEND, ManagerParseShow.parse("SHOW @@BACkend;", 5)); Assert.assertEquals(ManagerParseShow.BACKEND, ManagerParseShow.parse("show @@BACKEND ", 5)); } @Test public void testShowTimeCurrent() { Assert.assertEquals(ManagerParseShow.TIME_CURRENT, ManagerParseShow.parse("show @@time.current", 5)); Assert.assertEquals(ManagerParseShow.TIME_CURRENT, ManagerParseShow.parse("SHOW @@TIME.CURRENT", 5)); Assert.assertEquals(ManagerParseShow.TIME_CURRENT, ManagerParseShow.parse("show @@TIME.current", 5)); } @Test public void testShowTimeStartUp() { Assert.assertEquals(ManagerParseShow.TIME_STARTUP, ManagerParseShow.parse("show @@time.startup", 5)); Assert.assertEquals(ManagerParseShow.TIME_STARTUP, ManagerParseShow.parse("SHOW @@TIME.STARTUP", 5)); Assert.assertEquals(ManagerParseShow.TIME_STARTUP, ManagerParseShow.parse("show @@TIME.startup", 5)); } @Test public void testShowVersion() { Assert.assertEquals(ManagerParseShow.VERSION, ManagerParseShow.parse("show @@version", 5)); Assert.assertEquals(ManagerParseShow.VERSION, ManagerParseShow.parse("SHOW @@VERSION", 5)); Assert.assertEquals(ManagerParseShow.VERSION, ManagerParseShow.parse("show @@VERSION", 5)); } @Test public void testShowSQL() { Assert.assertEquals(ManagerParseShow.SQL, ManagerParseShow.parse("show @@sql where id = -1079800749", 5)); Assert.assertEquals(ManagerParseShow.SQL, ManagerParseShow.parse("SHOW @@SQL WHERE ID = -1079800749", 5)); Assert.assertEquals(ManagerParseShow.SQL, ManagerParseShow.parse("show @@Sql WHERE ID = -1079800749", 5)); Assert.assertEquals(ManagerParseShow.SQL, ManagerParseShow.parse("show @@sql where id=-1079800749", 5)); Assert.assertEquals(ManagerParseShow.SQL, ManagerParseShow.parse("show @@sql where id =-1079800749 ", 5)); } @Test public void testShowSQLDetail() { Assert.assertEquals(ManagerParseShow.SQL_DETAIL, ManagerParseShow.parse("show @@sql.detail where id = -1079800749", 5)); Assert.assertEquals(ManagerParseShow.SQL_DETAIL, ManagerParseShow.parse("SHOW @@SQL.DETAIL WHERE ID = -1079800749", 5)); Assert.assertEquals(ManagerParseShow.SQL_DETAIL, ManagerParseShow.parse("show @@SQL.DETAIL WHERE ID = -1079800749", 5)); Assert.assertEquals(ManagerParseShow.SQL_DETAIL, ManagerParseShow.parse("show @@sql.detail where id=1079800749 ", 5)); Assert.assertEquals(ManagerParseShow.SQL_DETAIL, ManagerParseShow.parse("show @@sql.detail where id= -1079800749", 5)); } @Test public void testShowSQLExecute() { Assert.assertEquals(ManagerParseShow.SQL_EXECUTE, ManagerParseShow.parse("show @@sql.execute", 5)); Assert.assertEquals(ManagerParseShow.SQL_EXECUTE, ManagerParseShow.parse("SHOW @@SQL.EXECUTE", 5)); Assert.assertEquals(ManagerParseShow.SQL_EXECUTE, ManagerParseShow.parse("show @@SQL.EXECUTE", 5)); } @Test public void testShowSQLSlow() { Assert.assertEquals(ManagerParseShow.SQL_SLOW, ManagerParseShow.parse("show @@sql.slow", 5)); Assert.assertEquals(ManagerParseShow.SQL_SLOW, ManagerParseShow.parse("SHOW @@SQL.SLOW", 5)); Assert.assertEquals(ManagerParseShow.SQL_SLOW, ManagerParseShow.parse("SHOW @@sql.slow", 5)); } @Test public void testShowVariables() { Assert.assertEquals(ManagerParseShow.VARIABLES, ManagerParseShow.parse("show variables", 5)); Assert.assertEquals(ManagerParseShow.VARIABLES, ManagerParseShow.parse("SHOW VARIABLES", 5)); Assert.assertEquals(ManagerParseShow.VARIABLES, ManagerParseShow.parse("show VARIABLES", 5)); } @Test public void testShowCollation() { Assert.assertEquals(ManagerParseShow.COLLATION, ManagerParseShow.parse("show collation", 5)); Assert.assertEquals(ManagerParseShow.COLLATION, ManagerParseShow.parse("SHOW COLLATION", 5)); Assert.assertEquals(ManagerParseShow.COLLATION, ManagerParseShow.parse("show COLLATION", 5)); } @Test public void testSwitchPool() { Assert.assertEquals(ManagerParse.SWITCH, 0xff & ManagerParse.parse("switch @@pool offer2$0-2")); Assert.assertEquals(ManagerParse.SWITCH, 0xff & ManagerParse.parse("SWITCH @@POOL offer2$0-2")); Assert.assertEquals(ManagerParse.SWITCH, 0xff & ManagerParse.parse("switch @@pool offer2$0-2 :2")); } @Test public void testComment() { Assert.assertEquals(ManagerParse.SWITCH, 0xff & ManagerParse.parse("/* abc */switch @@pool offer2$0-2")); Assert.assertEquals(ManagerParse.SHOW, 0xff & ManagerParse.parse(" /** 111**/Show @@help")); Assert.assertEquals(ManagerParse.SELECT, 0xff & ManagerParse.parse(" /***/ select * from t ")); } @Test public void testShowWhitComment() { Assert.assertEquals(ManagerParseShow.VARIABLES, ManagerParseShow.parse(" /** 111**/show variables", " /** 111**/show".length())); Assert.assertEquals(ManagerParseShow.VARIABLES, ManagerParseShow.parse(" /**111**/ SHOW VARIABLES", " /** 111**/show".length())); Assert.assertEquals(ManagerParseShow.VARIABLES, ManagerParseShow.parse(" /**111**/ SHOW variables", " /** 111**/show".length())); } @Test public void testStop() { Assert.assertEquals(ManagerParse.STOP, 0xff & ManagerParse.parse("stop @@")); Assert.assertEquals(ManagerParse.STOP, 0xff & ManagerParse.parse(" STOP ")); } @Test public void testStopHeartBeat() { Assert.assertEquals(ManagerParseStop.HEARTBEAT, ManagerParseStop.parse("stop @@heartbeat ds:1000", 4)); Assert.assertEquals(ManagerParseStop.HEARTBEAT, ManagerParseStop.parse(" STOP @@HEARTBEAT ds:1000", 5)); Assert.assertEquals(ManagerParseStop.HEARTBEAT, ManagerParseStop.parse(" STOP @@heartbeat ds:1000", 5)); } @Test public void testReload() { Assert.assertEquals(ManagerParse.RELOAD, 0xff & ManagerParse.parse("reload @@")); Assert.assertEquals(ManagerParse.RELOAD, 0xff & ManagerParse.parse(" RELOAD ")); } @Test public void testReloadConfig() { Assert.assertEquals(ManagerParseReload.CONFIG, ManagerParseReload.parse("reload @@config", 7)); Assert.assertEquals(ManagerParseReload.CONFIG, ManagerParseReload.parse(" RELOAD @@CONFIG ", 7)); Assert.assertEquals(ManagerParseReload.CONFIG, ManagerParseReload.parse(" RELOAD @@config ", 7)); } @Test public void testReloadRoute() { Assert.assertEquals(ManagerParseReload.ROUTE, ManagerParseReload.parse("reload @@route", 7)); Assert.assertEquals(ManagerParseReload.ROUTE, ManagerParseReload.parse(" RELOAD @@ROUTE ", 7)); Assert.assertEquals(ManagerParseReload.ROUTE, ManagerParseReload.parse(" RELOAD @@route ", 7)); } @Test public void testReloadUser() { Assert.assertEquals(ManagerParseReload.USER, ManagerParseReload.parse("reload @@user", 7)); Assert.assertEquals(ManagerParseReload.USER, ManagerParseReload.parse(" RELOAD @@USER ", 7)); Assert.assertEquals(ManagerParseReload.USER, ManagerParseReload.parse(" RELOAD @@user ", 7)); } @Test public void testRollback() { Assert.assertEquals(ManagerParse.ROLLBACK, 0xff & ManagerParse.parse("rollback @@")); Assert.assertEquals(ManagerParse.ROLLBACK, 0xff & ManagerParse.parse(" ROLLBACK ")); } @Test public void testOnOff() { Assert.assertEquals(ManagerParse.ONLINE, ManagerParse.parse("online ")); Assert.assertEquals(ManagerParse.ONLINE, ManagerParse.parse(" Online")); Assert.assertEquals(ManagerParse.OTHER, ManagerParse.parse(" Online2")); Assert.assertEquals(ManagerParse.OTHER, ManagerParse.parse("Online2 ")); Assert.assertEquals(ManagerParse.OFFLINE, ManagerParse.parse(" Offline")); Assert.assertEquals(ManagerParse.OFFLINE, ManagerParse.parse("offLine\t")); Assert.assertEquals(ManagerParse.OTHER, ManagerParse.parse("onLin")); Assert.assertEquals(ManagerParse.OTHER, ManagerParse.parse(" onlin")); } @Test public void testRollbackConfig() { Assert.assertEquals(ManagerParseRollback.CONFIG, ManagerParseRollback.parse("rollback @@config", 8)); Assert.assertEquals(ManagerParseRollback.CONFIG, ManagerParseRollback.parse(" ROLLBACK @@CONFIG ", 9)); Assert.assertEquals(ManagerParseRollback.CONFIG, ManagerParseRollback.parse(" ROLLBACK @@config ", 9)); } @Test public void testRollbackUser() { Assert.assertEquals(ManagerParseRollback.USER, ManagerParseRollback.parse("rollback @@user", 9)); Assert.assertEquals(ManagerParseRollback.USER, ManagerParseRollback.parse(" ROLLBACK @@USER ", 9)); Assert.assertEquals(ManagerParseRollback.USER, ManagerParseRollback.parse(" ROLLBACK @@user ", 9)); } @Test public void testRollbackRoute() { Assert.assertEquals(ManagerParseRollback.ROUTE, ManagerParseRollback.parse("rollback @@route", 9)); Assert.assertEquals(ManagerParseRollback.ROUTE, ManagerParseRollback.parse(" ROLLBACK @@ROUTE ", 9)); Assert.assertEquals(ManagerParseRollback.ROUTE, ManagerParseRollback.parse(" ROLLBACK @@route ", 9)); } @Test public void testGetWhere() { Assert.assertEquals("123", ManagerParseShow.getWhereParameter("where id = 123")); Assert.assertEquals("datanode", ManagerParseShow.getWhereParameter("where datanode = datanode")); Assert.assertEquals("schema", ManagerParseShow.getWhereParameter("where schema =schema ")); } @Test public void testShowSlowSchema() { Assert.assertEquals(ManagerParseShow.SLOW_SCHEMA, 0xff & ManagerParseShow.parse("show @@slow where schema=a", 5)); Assert.assertEquals(ManagerParseShow.SLOW_SCHEMA, 0xff & ManagerParseShow.parse(" SHOW @@SLOW WHERE SCHEMA=B", 6)); Assert.assertEquals(ManagerParseShow.SLOW_SCHEMA, 0xff & ManagerParseShow.parse(" show @@slow WHERE SCHEMA = a ", 5)); } @Test public void testShowSlowDataNode() { Assert.assertEquals(ManagerParseShow.SLOW_DATANODE, 0xff & ManagerParseShow.parse("show @@slow where datanode= a", 5)); Assert.assertEquals(ManagerParseShow.SLOW_DATANODE, 0xff & ManagerParseShow.parse("SHOW @@SLOW WHERE DATANODE= A", 5)); Assert.assertEquals(ManagerParseShow.SLOW_DATANODE, 0xff & ManagerParseShow.parse(" show @@SLOW where DATANODE= b ", 5)); } @Test public void testclearSlowSchema() { Assert.assertEquals(ManagerParseClear.SLOW_SCHEMA, 0xff & ManagerParseClear.parse("clear @@slow where schema=s", 5)); Assert.assertEquals(ManagerParseClear.SLOW_SCHEMA, 0xff & ManagerParseClear.parse("CLEAR @@SLOW WHERE SCHEMA= S", 5)); Assert.assertEquals(ManagerParseClear.SLOW_SCHEMA, 0xff & ManagerParseClear.parse("CLEAR @@slow where SCHEMA= s", 5)); } @Test public void testclearSlowDataNode() { Assert.assertEquals(ManagerParseClear.SLOW_DATANODE, 0xff & ManagerParseClear.parse("clear @@slow where datanode=d", 5)); Assert.assertEquals(ManagerParseClear.SLOW_DATANODE, 0xff & ManagerParseClear.parse("CLEAR @@SLOW WHERE DATANODE= D", 5)); Assert.assertEquals(ManagerParseClear.SLOW_DATANODE, 0xff & ManagerParseClear.parse("clear @@SLOW where DATANODE= d", 5)); } @Test public void testHeartBearDetail() { Assert.assertEquals(ManagerParseShow.HEARTBEAT_DETAIL, 0xff & ManagerParseShow.parse("show @@heartbeat.detail where name=master",5)); } @Test public void testSynStatus() { Assert.assertEquals(ManagerParseShow.DATASOURCE_SYNC, 0xff & ManagerParseShow.parse("show @@datasource.synstatus",5)); } @Test public void testSynDetail() { Assert.assertEquals(ManagerParseShow.DATASOURCE_SYNC_DETAIL, 0xff & ManagerParseShow.parse("show @@datasource.syndetail where name=slave",5)); } } ================================================ FILE: src/test/java/io/mycat/parser/ManagerParserTestPerf.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.parser; import io.mycat.route.parser.ManagerParse; /** * @author mycat */ public class ManagerParserTestPerf { public void testPerformance() { for (int i = 0; i < 250000; i++) { ManagerParse.parse("show databases"); ManagerParse.parse("set autocommit=1"); ManagerParse.parse(" show @@datasource "); ManagerParse.parse("select id,name,value from t"); } } public void testPerformanceWhere() { for (int i = 0; i < 500000; i++) { ManagerParse.parse(" show @@datasource where datanode = 1"); ManagerParse.parse(" show @@datanode where schema = 1"); } } } ================================================ FILE: src/test/java/io/mycat/parser/Performance.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.parser; /** * @author mycat */ public interface Performance { String SQL_BENCHMARK_SELECT = " seLEcT id, member_id , image_path \t , image_size , STATUS, gmt_modified from wp_image wheRe \t\t\n id = ? AND member_id\t=\t-123.456"; // String SQL_BENCHMARK_SELECT = // "select ID, GMT_CREATE, GMT_MODIFIED, INBOX_FOLDER_ID, MESSAGE_ID, FEEDBACK_TYPE, TARGET_ID, TRADE_ID, SUBJECT, SENDER_ID, SENDER_TYPE, S_DISPLAY_NAME, SENDER_STATUS, RECEIVER_ID, RECEIVER_TYPE, R_DISPLAY_NAME, RECEIVER_STATUS, SPAM_STATUS, REPLY_STATUS, ATTACHMENT_STATUS, SENDER_COUNTRY, RECEIVER_COUNTRY,APP_FROM,APP_TO,APP_SOURCE,SENDER_VACOUNT,RECEIVER_VACOUNT, DISTRIBUTE_STATUS,ORG_RECEIVER_ID,CUSTOMER_ID,OPERATOR_ID,OPERATOR_NAME,FOLLOW_STATUS,DELETE_STATUS,FOLLOW_TIME,BATCH_COUNT from MESSAGE_REC_RECORD where RECEIVER_VACOUNT =? and ID = ?"; String SQL_BENCHMARK_EXPR_SELECT = "( seLect id, member_id , image_path \t , image_size , STATUS, gmt_modified from wp_image where \t\t\n id = ? and member_id\t=\t?)"; } ================================================ FILE: src/test/java/io/mycat/parser/ServerParseTest.java ================================================ package io.mycat.parser; import junit.framework.Assert; import org.junit.Test; import io.mycat.server.parser.ServerParse; public class ServerParseTest { /** * public static final int OTHER = -1; public static final int BEGIN = 1; public static final int COMMIT = 2; public static final int DELETE = 3; public static final int INSERT = 4; public static final int REPLACE = 5; public static final int ROLLBACK = 6; public static final int SELECT = 7; public static final int SET = 8; public static final int SHOW = 9; public static final int START = 10; public static final int UPDATE = 11; public static final int KILL = 12; public static final int SAVEPOINT = 13; public static final int USE = 14; public static final int EXPLAIN = 15; public static final int KILL_QUERY = 16; public static final int HELP = 17; public static final int MYSQL_CMD_COMMENT = 18; public static final int MYSQL_COMMENT = 19; public static final int CALL = 20; public static final int DESCRIBE = 21; */ @Test public void testDesc() { String sql = "desc a"; int result = ServerParse.parse(sql); int sqlType = result & 0xff; Assert.assertEquals(ServerParse.DESCRIBE, sqlType); } @Test public void testDescribe() { String sql = "describe a"; int result = ServerParse.parse(sql); int sqlType = result & 0xff; Assert.assertEquals(ServerParse.DESCRIBE, sqlType); } @Test public void testDelete() { String sql = "delete from a where id = 1"; int result = ServerParse.parse(sql); int sqlType = result & 0xff; Assert.assertEquals(ServerParse.DELETE, sqlType); } @Test public void testInsert() { String sql = "insert into a(name) values ('zhangsan')"; int result = ServerParse.parse(sql); int sqlType = result & 0xff; Assert.assertEquals(ServerParse.INSERT, sqlType); } @Test public void testReplace() { String sql = "replace into t(id, update_time) select 1, now(); "; int result = ServerParse.parse(sql); int sqlType = result & 0xff; Assert.assertEquals(ServerParse.REPLACE, sqlType); } @Test public void testSet() { String sql = "SET @var_name = 'value'; "; int result = ServerParse.parse(sql); int sqlType = result & 0xff; Assert.assertEquals(ServerParse.SET, sqlType); } @Test public void testShow() { String sql = "show full tables"; int result = ServerParse.parse(sql); int sqlType = result & 0xff; Assert.assertEquals(ServerParse.SHOW, sqlType); } @Test public void testStart() { String sql = "start "; int result = ServerParse.parse(sql); int sqlType = result & 0xff; Assert.assertEquals(ServerParse.START, sqlType); } @Test public void testUpdate() { String sql = "update a set name='wdw' where id = 1"; int result = ServerParse.parse(sql); int sqlType = result & 0xff; Assert.assertEquals(ServerParse.UPDATE, sqlType); } @Test public void testKill() { String sql = "kill 1"; int result = ServerParse.parse(sql); int sqlType = result & 0xff; Assert.assertEquals(ServerParse.KILL, sqlType); } @Test public void testSavePoint() { String sql = "SAVEPOINT "; int result = ServerParse.parse(sql); int sqlType = result & 0xff; Assert.assertEquals(ServerParse.SAVEPOINT, sqlType); } @Test public void testUse() { String sql = "use db1 "; int result = ServerParse.parse(sql); int sqlType = result & 0xff; Assert.assertEquals(ServerParse.USE, sqlType); } @Test public void testExplain() { String sql = "explain select * from a "; int result = ServerParse.parse(sql); int sqlType = result & 0xff; Assert.assertEquals(ServerParse.EXPLAIN, sqlType); } @Test public void testKillQuery() { String sql = "kill query 1102 "; int result = ServerParse.parse(sql); int sqlType = result & 0xff; Assert.assertEquals(ServerParse.KILL_QUERY, sqlType); } @Test public void testHelp() { String sql = "help contents "; int result = ServerParse.parse(sql); int sqlType = result & 0xff; Assert.assertEquals(ServerParse.HELP, sqlType); } @Test public void testMysqlCmdComment() { } @Test public void testMysqlComment() { } @Test public void testCall() { String sql = "CALL demo_in_parameter(@p_in); "; int result = ServerParse.parse(sql); int sqlType = result & 0xff; Assert.assertEquals(ServerParse.CALL, sqlType); } @Test public void testRollback() { String sql = "rollback; "; int result = ServerParse.parse(sql); int sqlType = result & 0xff; Assert.assertEquals(ServerParse.ROLLBACK, sqlType); } @Test public void testSelect() { String sql = "select * from a"; int result = ServerParse.parse(sql); int sqlType = result & 0xff; Assert.assertEquals(ServerParse.SELECT, sqlType); } @Test public void testBegin() { String sql = "begin"; int result = ServerParse.parse(sql); int sqlType = result & 0xff; Assert.assertEquals(ServerParse.BEGIN, sqlType); } @Test public void testCommit() { String sql = "COMMIT 'nihao'"; int result = ServerParse.parse(sql); int sqlType = result & 0xff; Assert.assertEquals(ServerParse.COMMIT, sqlType); } } ================================================ FILE: src/test/java/io/mycat/parser/ServerParserTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.parser; import org.junit.Assert; import org.junit.Test; import io.mycat.server.parser.ServerParse; import io.mycat.server.parser.ServerParseSelect; import io.mycat.server.parser.ServerParseSet; import io.mycat.server.parser.ServerParseShow; import io.mycat.server.parser.ServerParseStart; /** * @author mycat */ public class ServerParserTest { @Test public void testIsBegin() { Assert.assertEquals(ServerParse.BEGIN, ServerParse.parse("begin")); Assert.assertEquals(ServerParse.BEGIN, ServerParse.parse("BEGIN")); Assert.assertEquals(ServerParse.BEGIN, ServerParse.parse("BegIn")); } @Test public void testIsCommit() { Assert.assertEquals(ServerParse.COMMIT, ServerParse.parse("commit")); Assert.assertEquals(ServerParse.COMMIT, ServerParse.parse("COMMIT")); Assert.assertEquals(ServerParse.COMMIT, ServerParse.parse("cOmmiT ")); } @Test public void testComment() { Assert.assertEquals(ServerParse.MYSQL_CMD_COMMENT, ServerParse.parse("/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */")); Assert.assertEquals(ServerParse.MYSQL_CMD_COMMENT, ServerParse.parse("/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */")); Assert.assertEquals(ServerParse.MYSQL_CMD_COMMENT, ServerParse.parse("/*!40101 SET @saved_cs_client = @@character_set_client */")); Assert.assertEquals(ServerParse.MYSQL_COMMENT, ServerParse.parse("/*SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */")); Assert.assertEquals(ServerParse.MYSQL_COMMENT, ServerParse.parse("/*SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */")); Assert.assertEquals(ServerParse.MYSQL_COMMENT, ServerParse.parse("/*SET @saved_cs_client = @@character_set_client */")); } @Test public void testMycatComment() { Assert.assertEquals(ServerParse.SELECT, 0xff & ServerParse.parse("/*#mycat:schema=DN1*/SELECT ...")); Assert.assertEquals(ServerParse.UPDATE, 0xff & ServerParse.parse("/*#mycat: schema = DN1 */ UPDATE ...")); Assert.assertEquals(ServerParse.DELETE, 0xff & ServerParse.parse("/*#mycat: sql = SELECT id FROM user */ DELETE ...")); } @Test public void testOldMycatComment() { Assert.assertEquals(ServerParse.SELECT, 0xff & ServerParse.parse("/*!mycat:schema=DN1*/SELECT ...")); Assert.assertEquals(ServerParse.UPDATE, 0xff & ServerParse.parse("/*!mycat: schema = DN1 */ UPDATE ...")); Assert.assertEquals(ServerParse.DELETE, 0xff & ServerParse.parse("/*!mycat: sql = SELECT id FROM user */ DELETE ...")); } @Test public void testIsDelete() { Assert.assertEquals(ServerParse.DELETE, ServerParse.parse("delete ...")); Assert.assertEquals(ServerParse.DELETE, ServerParse.parse("DELETE ...")); Assert.assertEquals(ServerParse.DELETE, ServerParse.parse("DeletE ...")); } @Test public void testIsInsert() { Assert.assertEquals(ServerParse.INSERT, ServerParse.parse("insert ...")); Assert.assertEquals(ServerParse.INSERT, ServerParse.parse("INSERT ...")); Assert.assertEquals(ServerParse.INSERT, ServerParse.parse("InserT ...")); } @Test public void testIsReplace() { Assert.assertEquals(ServerParse.REPLACE, ServerParse.parse("replace ...")); Assert.assertEquals(ServerParse.REPLACE, ServerParse.parse("REPLACE ...")); Assert.assertEquals(ServerParse.REPLACE, ServerParse.parse("rEPLACe ...")); } @Test public void testIsRollback() { Assert.assertEquals(ServerParse.ROLLBACK, ServerParse.parse("rollback")); Assert.assertEquals(ServerParse.ROLLBACK, ServerParse.parse("ROLLBACK")); Assert.assertEquals(ServerParse.ROLLBACK, ServerParse.parse("rolLBACK ")); } @Test public void testIsSelect() { Assert.assertEquals(ServerParse.SELECT, 0xff & ServerParse.parse("select ...")); Assert.assertEquals(ServerParse.SELECT, 0xff & ServerParse.parse("SELECT ...")); Assert.assertEquals(ServerParse.SELECT, 0xff & ServerParse.parse("sELECt ...")); } @Test public void testIsSet() { Assert.assertEquals(ServerParse.SET, 0xff & ServerParse.parse("set ...")); Assert.assertEquals(ServerParse.SET, 0xff & ServerParse.parse("SET ...")); Assert.assertEquals(ServerParse.SET, 0xff & ServerParse.parse("sEt ...")); } @Test public void testIsShow() { Assert.assertEquals(ServerParse.SHOW, 0xff & ServerParse.parse("show ...")); Assert.assertEquals(ServerParse.SHOW, 0xff & ServerParse.parse("SHOW ...")); Assert.assertEquals(ServerParse.SHOW, 0xff & ServerParse.parse("sHOw ...")); } @Test public void testIsStart() { Assert.assertEquals(ServerParse.START, 0xff & ServerParse.parse("start ...")); Assert.assertEquals(ServerParse.START, 0xff & ServerParse.parse("START ...")); Assert.assertEquals(ServerParse.START, 0xff & ServerParse.parse("stART ...")); } @Test public void testIsUpdate() { Assert.assertEquals(ServerParse.UPDATE, ServerParse.parse("update ...")); Assert.assertEquals(ServerParse.UPDATE, ServerParse.parse("UPDATE ...")); Assert.assertEquals(ServerParse.UPDATE, ServerParse.parse("UPDate ...")); } @Test public void testIsShowDatabases() { Assert.assertEquals(ServerParseShow.DATABASES, ServerParseShow.parse("show databases", 4)); Assert.assertEquals(ServerParseShow.DATABASES, ServerParseShow.parse("SHOW DATABASES", 4)); Assert.assertEquals(ServerParseShow.DATABASES, ServerParseShow.parse("SHOW databases ", 4)); } @Test public void testIsShowDataSources() { Assert.assertEquals(ServerParseShow.DATASOURCES, ServerParseShow.parse("show datasources", 4)); Assert.assertEquals(ServerParseShow.DATASOURCES, ServerParseShow.parse("SHOW DATASOURCES", 4)); Assert.assertEquals(ServerParseShow.DATASOURCES, ServerParseShow.parse(" SHOW DATASOURCES ", 6)); } @Test public void testShowMycatStatus() { Assert.assertEquals(ServerParseShow.MYCAT_STATUS, ServerParseShow.parse("show mycat_status", 4)); Assert.assertEquals(ServerParseShow.MYCAT_STATUS, ServerParseShow.parse("show mycat_status ", 4)); Assert.assertEquals(ServerParseShow.MYCAT_STATUS, ServerParseShow.parse(" SHOW MYCAT_STATUS", " SHOW".length())); Assert.assertEquals(ServerParseShow.OTHER, ServerParseShow.parse(" show mycat_statu", " SHOW".length())); Assert.assertEquals(ServerParseShow.OTHER, ServerParseShow.parse(" show mycat_status2", " SHOW".length())); Assert.assertEquals(ServerParseShow.OTHER, ServerParseShow.parse("Show mycat_status2 ", "SHOW".length())); } @Test public void testShowMycatCluster() { Assert.assertEquals(ServerParseShow.MYCAT_CLUSTER, ServerParseShow.parse("show mycat_cluster", 4)); Assert.assertEquals(ServerParseShow.MYCAT_CLUSTER, ServerParseShow.parse("Show mycat_CLUSTER ", 4)); Assert.assertEquals(ServerParseShow.MYCAT_CLUSTER, ServerParseShow.parse(" show MYCAT_cluster", 5)); Assert.assertEquals(ServerParseShow.OTHER, ServerParseShow.parse(" show mycat_clust", 5)); Assert.assertEquals(ServerParseShow.OTHER, ServerParseShow.parse(" show mycat_cluster2", 5)); Assert.assertEquals(ServerParseShow.OTHER, ServerParseShow.parse("Show mycat_cluster9 ", 4)); } @Test public void testIsShowOther() { Assert.assertEquals(ServerParseShow.OTHER, ServerParseShow.parse("show ...", 4)); Assert.assertEquals(ServerParseShow.OTHER, ServerParseShow.parse("SHOW ...", 4)); Assert.assertEquals(ServerParseShow.OTHER, ServerParseShow.parse("SHOW ... ", 4)); } @Test public void testIsSetAutocommitOn() { Assert.assertEquals(ServerParseSet.AUTOCOMMIT_ON, ServerParseSet.parse("set autocommit=1", 3)); Assert.assertEquals(ServerParseSet.AUTOCOMMIT_ON, ServerParseSet.parse("set autoCOMMIT = 1", 3)); Assert.assertEquals(ServerParseSet.AUTOCOMMIT_ON, ServerParseSet.parse("SET AUTOCOMMIT=on", 3)); Assert.assertEquals(ServerParseSet.AUTOCOMMIT_ON, ServerParseSet.parse("set autoCOMMIT = ON", 3)); } @Test public void testIsSetAutocommitOn2() { Assert.assertEquals(ServerParseSet.AUTOCOMMIT_ON, ServerParseSet.parse("set @@session.autoCOMMIT = ON", 3)); } @Test public void testIsSetAutocommitOff() { Assert.assertEquals(ServerParseSet.AUTOCOMMIT_OFF, ServerParseSet.parse("set autocommit=0", 3)); Assert.assertEquals(ServerParseSet.AUTOCOMMIT_OFF, ServerParseSet.parse("SET AUTOCOMMIT= 0", 3)); Assert.assertEquals(ServerParseSet.AUTOCOMMIT_OFF, ServerParseSet.parse("set autoCOMMIT =OFF", 3)); Assert.assertEquals(ServerParseSet.AUTOCOMMIT_OFF, ServerParseSet.parse("set autoCOMMIT = off", 3)); Assert.assertEquals(ServerParseSet.AUTOCOMMIT_OFF, ServerParseSet.parse("set @@session.autocommit =OFF", 3)); } @Test public void testIsSetAutocommitOff2() { Assert.assertEquals(ServerParseSet.AUTOCOMMIT_OFF, ServerParseSet.parse("set @@session.autocommit =OFF", 3)); } @Test public void testIsSetNames() { Assert.assertEquals(ServerParseSet.NAMES, 0xff & ServerParseSet.parse("set names utf8", 3)); Assert.assertEquals(ServerParseSet.NAMES, 0xff & ServerParseSet.parse("SET NAMES UTF8", 3)); Assert.assertEquals(ServerParseSet.NAMES, 0xff & ServerParseSet.parse("set NAMES utf8", 3)); } @Test public void testIsCharacterSetResults() { Assert.assertEquals(ServerParseSet.CHARACTER_SET_RESULTS, 0xff & ServerParseSet.parse("SET character_set_results = NULL", 3)); Assert.assertEquals(ServerParseSet.CHARACTER_SET_RESULTS, 0xff & ServerParseSet.parse("SET CHARACTER_SET_RESULTS= NULL", 3)); Assert.assertEquals(ServerParseSet.CHARACTER_SET_RESULTS, 0xff & ServerParseSet.parse("Set chARActer_SET_RESults = NULL", 3)); Assert.assertEquals(ServerParseSet.CHARACTER_SET_CONNECTION, 0xff & ServerParseSet.parse("Set chARActer_SET_Connection = NULL", 3)); Assert.assertEquals(ServerParseSet.CHARACTER_SET_CLIENT, 0xff & ServerParseSet.parse("Set chARActer_SET_client = NULL", 3)); } @Test public void testIsSetOther() { Assert.assertEquals(ServerParseSet.OTHER, ServerParseSet.parse("set ...", 3)); Assert.assertEquals(ServerParseSet.OTHER, ServerParseSet.parse("SET ...", 3)); Assert.assertEquals(ServerParseSet.OTHER, ServerParseSet.parse("sEt ...", 3)); } @Test public void testIsKill() { Assert.assertEquals(ServerParse.KILL, 0xff & ServerParse.parse(" kill ...")); Assert.assertEquals(ServerParse.KILL, 0xff & ServerParse.parse("kill 111111 ...")); Assert.assertEquals(ServerParse.KILL, 0xff & ServerParse.parse("KILL 1335505632")); } @Test public void testIsKillQuery() { Assert.assertEquals(ServerParse.KILL_QUERY, 0xff & ServerParse.parse(" kill query ...")); Assert.assertEquals(ServerParse.KILL_QUERY, 0xff & ServerParse.parse("kill query 111111 ...")); Assert.assertEquals(ServerParse.KILL_QUERY, 0xff & ServerParse.parse("KILL QUERY 1335505632")); } @Test public void testIsSavepoint() { Assert.assertEquals(ServerParse.SAVEPOINT, ServerParse.parse(" savepoint ...")); Assert.assertEquals(ServerParse.SAVEPOINT, ServerParse.parse("SAVEPOINT ")); Assert.assertEquals(ServerParse.SAVEPOINT, ServerParse.parse(" SAVEpoint a")); } @Test public void testIsUse() { Assert.assertEquals(ServerParse.USE, 0xff & ServerParse.parse(" use ...")); Assert.assertEquals(ServerParse.USE, 0xff & ServerParse.parse("USE ")); Assert.assertEquals(ServerParse.USE, 0xff & ServerParse.parse(" Use a")); } @Test public void testIsStartTransaction() { Assert.assertEquals(ServerParseStart.TRANSACTION, ServerParseStart.parse(" start transaction ...", 6)); Assert.assertEquals(ServerParseStart.TRANSACTION, ServerParseStart.parse("START TRANSACTION", 5)); Assert.assertEquals(ServerParseStart.TRANSACTION, ServerParseStart.parse(" staRT TRANSaction ", 6)); } @Test public void testIsSelectVersionComment() { Assert.assertEquals(ServerParseSelect.VERSION_COMMENT, ServerParseSelect.parse(" select @@version_comment ", 7)); Assert.assertEquals(ServerParseSelect.VERSION_COMMENT, ServerParseSelect.parse("SELECT @@VERSION_COMMENT", 6)); Assert.assertEquals(ServerParseSelect.VERSION_COMMENT, ServerParseSelect.parse(" selECT @@VERSION_comment ", 7)); } @Test public void testIsSelectVersion() { Assert.assertEquals(ServerParseSelect.VERSION, ServerParseSelect.parse(" select version () ", 7)); Assert.assertEquals(ServerParseSelect.VERSION, ServerParseSelect.parse("SELECT VERSION( )", 6)); Assert.assertEquals(ServerParseSelect.VERSION, ServerParseSelect.parse(" selECT VERSION() ", 7)); } @Test public void testIsSelectDatabase() { Assert.assertEquals(ServerParseSelect.DATABASE, ServerParseSelect.parse(" select database() ", 7)); Assert.assertEquals(ServerParseSelect.DATABASE, ServerParseSelect.parse("SELECT DATABASE()", 6)); Assert.assertEquals(ServerParseSelect.DATABASE, ServerParseSelect.parse(" selECT DATABASE() ", 7)); } @Test public void testIsSelectUser() { Assert.assertEquals(ServerParseSelect.USER, ServerParseSelect.parse(" select user() ", 7)); Assert.assertEquals(ServerParseSelect.USER, ServerParseSelect.parse("SELECT USER()", 6)); Assert.assertEquals(ServerParseSelect.USER, ServerParseSelect.parse(" selECT USER() ", 7)); } @Test public void testTxReadUncommitted() { Assert.assertEquals(ServerParseSet.TX_READ_UNCOMMITTED, ServerParseSet.parse(" SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED ", " SET".length())); Assert.assertEquals(ServerParseSet.TX_READ_UNCOMMITTED, ServerParseSet.parse(" set session transaction isolation level read uncommitted ", " SET".length())); Assert.assertEquals(ServerParseSet.TX_READ_UNCOMMITTED, ServerParseSet.parse(" set session transaCTION ISOLATION LEvel read uncommitteD ", " SET".length())); } @Test public void testTxReadCommitted() { Assert.assertEquals(ServerParseSet.TX_READ_COMMITTED, ServerParseSet.parse(" SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED ", " SET".length())); Assert.assertEquals(ServerParseSet.TX_READ_COMMITTED, ServerParseSet.parse(" set session transaction isolation level read committed ", " SET".length())); Assert.assertEquals(ServerParseSet.TX_READ_COMMITTED, ServerParseSet.parse(" set session transaCTION ISOLATION LEVel read committed ", " SET".length())); } @Test public void testTxReadOnly() { Assert.assertEquals(ServerParseSet.TX_READONLY, ServerParseSet.parse(" SET SESSION TRANSACTION READ ONLY ", " SET".length())); Assert.assertEquals(ServerParseSet.TX_READONLY, ServerParseSet.parse(" set session transaction read only ", " SET".length())); Assert.assertEquals(ServerParseSet.TX_READONLY, ServerParseSet.parse(" set session transaCTION Read Only ", " SET".length())); } @Test public void testSqlSelectLimit() { Assert.assertEquals(ServerParseSet.SQL_SELECT_LIMIT, ServerParseSet.parse(" SET SQL_SELECT_LIMIT=1 ", " SET".length())); Assert.assertEquals(ServerParseSet.SQL_SELECT_LIMIT, ServerParseSet.parse(" set sql_select_limit=1 ", " SET".length())); Assert.assertEquals(ServerParseSet.SQL_SELECT_LIMIT, ServerParseSet.parse(" Set Sql_Select_Limit=1 ", " SET".length())); } @Test public void testTxReadWrite() { Assert.assertEquals(ServerParseSet.TX_READWRITE, ServerParseSet.parse(" SET SESSION TRANSACTION READ WRITE ", " SET".length())); Assert.assertEquals(ServerParseSet.TX_READWRITE, ServerParseSet.parse(" set session transaction read write ", " SET".length())); Assert.assertEquals(ServerParseSet.TX_READWRITE, ServerParseSet.parse(" set session transaCTION Read Write ", " SET".length())); } @Test public void testTxRepeatedRead() { Assert.assertEquals(ServerParseSet.TX_REPEATED_READ, ServerParseSet.parse(" SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ ", " SET".length())); Assert.assertEquals(ServerParseSet.TX_REPEATED_READ, ServerParseSet.parse(" set session transaction isolation level repeatable read ", " SET".length())); Assert.assertEquals(ServerParseSet.TX_REPEATED_READ, ServerParseSet.parse(" set session transaction isOLATION LEVEL REPEatable read ", " SET".length())); } @Test public void testTxSerializable() { Assert.assertEquals(ServerParseSet.TX_SERIALIZABLE, ServerParseSet.parse(" SET SESSION TRANSACTION ISOLATION LEVEL SERIALIZABLE ", " SET".length())); Assert.assertEquals(ServerParseSet.TX_SERIALIZABLE, ServerParseSet.parse(" set session transaction isolation level serializable ", " SET".length())); Assert.assertEquals(ServerParseSet.TX_SERIALIZABLE, ServerParseSet.parse(" set session transaction isOLATION LEVEL SERIAlizable ", " SET".length())); } @Test public void testIdentity() { String stmt = "select @@identity"; int indexAfterLastInsertIdFunc = ServerParseSelect.indexAfterIdentity(stmt, stmt.indexOf('i')); Assert.assertEquals(stmt.length(), indexAfterLastInsertIdFunc); Assert.assertEquals(ServerParseSelect.IDENTITY, ServerParseSelect.parse(stmt, 6)); stmt = "select @@identity as id"; Assert.assertEquals(ServerParseSelect.IDENTITY, ServerParseSelect.parse(stmt, 6)); stmt = "select @@identitY id"; Assert.assertEquals(ServerParseSelect.IDENTITY, ServerParseSelect.parse(stmt, 6)); stmt = "select /*foo*/@@identitY id"; Assert.assertEquals(ServerParseSelect.IDENTITY, ServerParseSelect.parse(stmt, 6)); stmt = "select/*foo*/ @@identitY id"; Assert.assertEquals(ServerParseSelect.IDENTITY, ServerParseSelect.parse(stmt, 6)); stmt = "select/*foo*/ @@identitY As id"; Assert.assertEquals(ServerParseSelect.IDENTITY, ServerParseSelect.parse(stmt, 6)); stmt = "select @@identity ,"; Assert.assertEquals(ServerParseSelect.OTHER, ServerParseSelect.parse(stmt, 6)); stmt = "select @@identity as, "; Assert.assertEquals(ServerParseSelect.OTHER, ServerParseSelect.parse(stmt, 6)); stmt = "select @@identity as id , "; Assert.assertEquals(ServerParseSelect.OTHER, ServerParseSelect.parse(stmt, 6)); stmt = "select @@identity ass id "; Assert.assertEquals(ServerParseSelect.OTHER, ServerParseSelect.parse(stmt, 6)); } @Test public void testLastInsertId() { String stmt = " last_insert_iD()"; int indexAfterLastInsertIdFunc = ServerParseSelect.indexAfterLastInsertIdFunc(stmt, stmt.indexOf('l')); Assert.assertEquals(stmt.length(), indexAfterLastInsertIdFunc); stmt = " last_insert_iD ()"; indexAfterLastInsertIdFunc = ServerParseSelect.indexAfterLastInsertIdFunc(stmt, stmt.indexOf('l')); Assert.assertEquals(stmt.length(), indexAfterLastInsertIdFunc); stmt = " last_insert_iD ( /**/ )"; indexAfterLastInsertIdFunc = ServerParseSelect.indexAfterLastInsertIdFunc(stmt, stmt.indexOf('l')); Assert.assertEquals(stmt.length(), indexAfterLastInsertIdFunc); stmt = " last_insert_iD ( ) "; indexAfterLastInsertIdFunc = ServerParseSelect.indexAfterLastInsertIdFunc(stmt, stmt.indexOf('l')); Assert.assertEquals(stmt.lastIndexOf(')') + 1, indexAfterLastInsertIdFunc); stmt = " last_insert_id( )"; indexAfterLastInsertIdFunc = ServerParseSelect.indexAfterLastInsertIdFunc(stmt, stmt.indexOf('l')); Assert.assertEquals(stmt.lastIndexOf(')') + 1, indexAfterLastInsertIdFunc); stmt = "last_iNsert_id( ) "; indexAfterLastInsertIdFunc = ServerParseSelect.indexAfterLastInsertIdFunc(stmt, stmt.indexOf('l')); Assert.assertEquals(stmt.lastIndexOf(')') + 1, indexAfterLastInsertIdFunc); stmt = " last_insert_iD"; indexAfterLastInsertIdFunc = ServerParseSelect.indexAfterLastInsertIdFunc(stmt, stmt.indexOf('l')); Assert.assertEquals(-1, indexAfterLastInsertIdFunc); stmt = " last_insert_i "; indexAfterLastInsertIdFunc = ServerParseSelect.indexAfterLastInsertIdFunc(stmt, stmt.indexOf('l')); Assert.assertEquals(-1, indexAfterLastInsertIdFunc); stmt = " last_insert_i d "; indexAfterLastInsertIdFunc = ServerParseSelect.indexAfterLastInsertIdFunc(stmt, stmt.indexOf('l')); Assert.assertEquals(-1, indexAfterLastInsertIdFunc); stmt = " last_insert_id ( "; indexAfterLastInsertIdFunc = ServerParseSelect.indexAfterLastInsertIdFunc(stmt, stmt.indexOf('l')); Assert.assertEquals(-1, indexAfterLastInsertIdFunc); stmt = " last_insert_id( d) "; indexAfterLastInsertIdFunc = ServerParseSelect.indexAfterLastInsertIdFunc(stmt, stmt.indexOf('l')); Assert.assertEquals(-1, indexAfterLastInsertIdFunc); stmt = " last_insert_id( ) d "; indexAfterLastInsertIdFunc = ServerParseSelect.indexAfterLastInsertIdFunc(stmt, stmt.indexOf('l')); Assert.assertEquals(stmt.lastIndexOf(')') + 1, indexAfterLastInsertIdFunc); stmt = " last_insert_id(d)"; indexAfterLastInsertIdFunc = ServerParseSelect.indexAfterLastInsertIdFunc(stmt, stmt.indexOf('l')); Assert.assertEquals(-1, indexAfterLastInsertIdFunc); stmt = " last_insert_id(#\r\nd) "; indexAfterLastInsertIdFunc = ServerParseSelect.indexAfterLastInsertIdFunc(stmt, stmt.indexOf('l')); Assert.assertEquals(-1, indexAfterLastInsertIdFunc); stmt = " last_insert_id(#\n\r) "; indexAfterLastInsertIdFunc = ServerParseSelect.indexAfterLastInsertIdFunc(stmt, stmt.indexOf('l')); Assert.assertEquals(stmt.lastIndexOf(')') + 1, indexAfterLastInsertIdFunc); stmt = " last_insert_id (#\n\r)"; indexAfterLastInsertIdFunc = ServerParseSelect.indexAfterLastInsertIdFunc(stmt, stmt.indexOf('l')); Assert.assertEquals(stmt.lastIndexOf(')') + 1, indexAfterLastInsertIdFunc); stmt = " last_insert_id(#\n\r)"; indexAfterLastInsertIdFunc = ServerParseSelect.indexAfterLastInsertIdFunc(stmt, stmt.indexOf('l')); Assert.assertEquals(stmt.lastIndexOf(')') + 1, indexAfterLastInsertIdFunc); stmt = "select last_insert_id(#\n\r)"; Assert.assertEquals(ServerParseSelect.LAST_INSERT_ID, ServerParseSelect.parse(stmt, 6)); stmt = "select last_insert_id(#\n\r) as id"; Assert.assertEquals(ServerParseSelect.LAST_INSERT_ID, ServerParseSelect.parse(stmt, 6)); stmt = "select last_insert_id(#\n\r) as `id`"; Assert.assertEquals(ServerParseSelect.LAST_INSERT_ID, ServerParseSelect.parse(stmt, 6)); stmt = "select last_insert_id(#\n\r) as 'id'"; Assert.assertEquals(ServerParseSelect.LAST_INSERT_ID, ServerParseSelect.parse(stmt, 6)); stmt = "select last_insert_id(#\n\r) id"; Assert.assertEquals(ServerParseSelect.LAST_INSERT_ID, ServerParseSelect.parse(stmt, 6)); stmt = "select last_insert_id(#\n\r) `id`"; Assert.assertEquals(ServerParseSelect.LAST_INSERT_ID, ServerParseSelect.parse(stmt, 6)); stmt = "select last_insert_id(#\n\r) 'id'"; Assert.assertEquals(ServerParseSelect.LAST_INSERT_ID, ServerParseSelect.parse(stmt, 6)); stmt = "select last_insert_id(#\n\r) a"; Assert.assertEquals(ServerParseSelect.LAST_INSERT_ID, ServerParseSelect.parse(stmt, 6)); // NOTE: this should be invalid, ignore this bug stmt = "select last_insert_id(#\n\r) as"; Assert.assertEquals(ServerParseSelect.LAST_INSERT_ID, ServerParseSelect.parse(stmt, 6)); stmt = "select last_insert_id(#\n\r) asd"; Assert.assertEquals(ServerParseSelect.LAST_INSERT_ID, ServerParseSelect.parse(stmt, 6)); // NOTE: this should be invalid, ignore this bug stmt = "select last_insert_id(#\n\r) as 777"; Assert.assertEquals(ServerParseSelect.LAST_INSERT_ID, ServerParseSelect.parse(stmt, 6)); // NOTE: this should be invalid, ignore this bug stmt = "select last_insert_id(#\n\r) 777"; Assert.assertEquals(ServerParseSelect.LAST_INSERT_ID, ServerParseSelect.parse(stmt, 6)); stmt = "select last_insert_id(#\n\r)as `77``7`"; Assert.assertEquals(ServerParseSelect.LAST_INSERT_ID, ServerParseSelect.parse(stmt, 6)); stmt = "select last_insert_id(#\n\r)ass"; Assert.assertEquals(ServerParseSelect.LAST_INSERT_ID, ServerParseSelect.parse(stmt, 6)); stmt = "select last_insert_id(#\n\r)as 'a'"; Assert.assertEquals(ServerParseSelect.LAST_INSERT_ID, ServerParseSelect.parse(stmt, 6)); stmt = "select last_insert_id(#\n\r)as 'a\\''"; Assert.assertEquals(ServerParseSelect.LAST_INSERT_ID, ServerParseSelect.parse(stmt, 6)); stmt = "select last_insert_id(#\n\r)as 'a'''"; Assert.assertEquals(ServerParseSelect.LAST_INSERT_ID, ServerParseSelect.parse(stmt, 6)); stmt = "select last_insert_id(#\n\r)as 'a\"'"; Assert.assertEquals(ServerParseSelect.LAST_INSERT_ID, ServerParseSelect.parse(stmt, 6)); stmt = " select last_insert_id(#\n\r) As 'a\"'"; Assert.assertEquals(ServerParseSelect.LAST_INSERT_ID, ServerParseSelect.parse(stmt, 9)); stmt = "select last_insert_id(#\n\r)as 'a\"\\'"; Assert.assertEquals(ServerParseSelect.OTHER, ServerParseSelect.parse(stmt, 6)); stmt = "select last_insert_id(#\n\r)as `77``7` ,"; Assert.assertEquals(ServerParseSelect.OTHER, ServerParseSelect.parse(stmt, 6)); stmt = "select last_insert_id(#\n\r)as `77`7`"; Assert.assertEquals(ServerParseSelect.OTHER, ServerParseSelect.parse(stmt, 6)); stmt = "select last_insert_id(#\n\r) as,"; Assert.assertEquals(ServerParseSelect.OTHER, ServerParseSelect.parse(stmt, 6)); stmt = "select last_insert_id(#\n\r) ass a"; Assert.assertEquals(ServerParseSelect.OTHER, ServerParseSelect.parse(stmt, 6)); stmt = "select last_insert_id(#\n\r) as 'a"; Assert.assertEquals(ServerParseSelect.OTHER, ServerParseSelect.parse(stmt, 6)); } @Test public void testLockTable() { Assert.assertEquals(ServerParse.LOCK, ServerParse.parse("lock tables ttt write;")); Assert.assertEquals(ServerParse.LOCK, ServerParse.parse(" lock tables ttt read;")); Assert.assertEquals(ServerParse.LOCK, ServerParse.parse("lock tables")); } @Test public void testUnlockTable() { Assert.assertEquals(ServerParse.UNLOCK, ServerParse.parse("unlock tables")); Assert.assertEquals(ServerParse.UNLOCK, ServerParse.parse(" unlock tables")); } @Test public void testSetXAOn() { Assert.assertEquals(ServerParseSet.XA_FLAG_ON, ServerParseSet.parse("set xa=on", 3)); Assert.assertEquals(ServerParseSet.XA_FLAG_ON, ServerParseSet.parse("set xa = on", 3)); Assert.assertEquals(ServerParseSet.XA_FLAG_ON, ServerParseSet.parse("set xa \t\n\r = \t\n\r on", 3)); } @Test public void testSetXAOff() { Assert.assertEquals(ServerParseSet.XA_FLAG_OFF, ServerParseSet.parse("set xa=off", 3)); Assert.assertEquals(ServerParseSet.XA_FLAG_OFF, ServerParseSet.parse("set xa = off", 3)); Assert.assertEquals(ServerParseSet.XA_FLAG_OFF, ServerParseSet.parse("set xa \t\n\r = \t\n\r off", 3)); } } ================================================ FILE: src/test/java/io/mycat/parser/ServerParserTestPerf.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.parser; import io.mycat.server.parser.ServerParseSet; /** * @author mycat */ public final class ServerParserTestPerf { private static void parseSetPerf() { // ServerParse.parse("show databases"); // ServerParseSet.parse("set autocommit=1"); // ServerParseSet.parse("set names=1"); ServerParseSet.parse("SET character_set_results = NULL", 4); // ServerParse.parse("select id,name,value from t"); // ServerParse.parse("select * from offer where member_id='abc'"); } public static void main(String[] args) { parseSetPerf(); int count = 10000000; System.currentTimeMillis(); long t1 = System.currentTimeMillis(); for (int i = 0; i < count; i++) { parseSetPerf(); } long t2 = System.currentTimeMillis(); // print time System.out.println("take:" + ((t2 - t1) * 1000 * 1000) / count + " ns."); } } ================================================ FILE: src/test/java/io/mycat/parser/TestEscapeProcess.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights * reserved. DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. This * code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. This code is distributed in the hope that it will * be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General * Public License version 2 for more details (a copy is included in the LICENSE * file that accompanied this code). You should have received a copy of the GNU * General Public License version 2 along with this work; if not, write to the * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA * 02110-1301 USA. Any questions about this component can be directed to it's * project Web address https://code.google.com/p/opencloudb/. */ package io.mycat.parser; import static org.junit.Assert.assertEquals; import io.mycat.MycatServer; import io.mycat.server.interceptor.impl.DefaultSqlInterceptor; import org.junit.Test; public class TestEscapeProcess { String sql = "insert into t_uud_user_account(USER_ID,USER_NAME,PASSWORD,CREATE_TIME,STATUS,NICK_NAME,USER_ICON_URL,USER_ICON_URL2,USER_ICON_URL3,ACCOUNT_TYPE) " + "values (2488899998,'u\\'aa\\'\\'a''aa','af8f9dffa5d420fbc249141645b962ee','2013-12-01 00:00:00',0,NULL,NULL,NULL,NULL,1)"; String sqlret = "insert into t_uud_user_account(USER_ID,USER_NAME,PASSWORD,CREATE_TIME,STATUS,NICK_NAME,USER_ICON_URL,USER_ICON_URL2,USER_ICON_URL3,ACCOUNT_TYPE) " + "values (2488899998,'u''aa''''a''aa','af8f9dffa5d420fbc249141645b962ee','2013-12-01 00:00:00',0,NULL,NULL,NULL,NULL,1)"; String starWithEscapeSql = "\\insert into t_uud_user_account(USER_ID,USER_NAME,PASSWORD,CREATE_TIME,STATUS,NICK_NAME,USER_ICON_URL,USER_ICON_URL2,USER_ICON_URL3,ACCOUNT_TYPE) " + "values (2488899998,'u\\'aa\\'\\'a''aa','af8f9dffa5d420fbc249141645b962ee','2013-12-01 00:00:00',0,NULL,NULL,NULL,NULL,1)\\"; String starWithEscapeSqlret = "\\insert into t_uud_user_account(USER_ID,USER_NAME,PASSWORD,CREATE_TIME,STATUS,NICK_NAME,USER_ICON_URL,USER_ICON_URL2,USER_ICON_URL3,ACCOUNT_TYPE) " + "values (2488899998,'u''aa''''a''aa','af8f9dffa5d420fbc249141645b962ee','2013-12-01 00:00:00',0,NULL,NULL,NULL,NULL,1)\\"; @Test public void testEscapeProcess() { MycatServer.getInstance().getConfig().getSystem().setDefaultSqlParser("fdbparser"); String sqlProcessed = DefaultSqlInterceptor.processEscape(sql); assertEquals(sqlProcessed, sqlret); String sqlProcessed1 = DefaultSqlInterceptor .processEscape(starWithEscapeSql); assertEquals(sqlProcessed1, starWithEscapeSqlret); } } ================================================ FILE: src/test/java/io/mycat/parser/druid/DruidSelectParserTest.java ================================================ package io.mycat.parser.druid; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.TreeMap; import org.junit.Assert; import org.junit.Test; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import io.mycat.route.parser.druid.impl.DruidSelectParser; /** * Created by Hash Zhang on 2016/4/29. * Modified by Hash Zhang on 2016/5/25 add testGroupByWithViewAlias. */ public class DruidSelectParserTest { DruidSelectParser druidSelectParser = new DruidSelectParser(); /** * 此方法检测DruidSelectParser的buildGroupByCols方法是否修改了函数列 * 因为select的函数列并不做alias处理， * 所以在groupby也对函数列不做修改 * * @throws NoSuchMethodException * @throws InvocationTargetException * @throws IllegalAccessException */ @Test public void testGroupByWithAlias() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { String functionColumn = "DATE_FORMAT(h.times,'%b %d %Y %h:%i %p')"; Object result = invokeGroupBy(functionColumn); Assert.assertEquals(functionColumn, ((String[]) result)[0]); } /** * 此方法检测DruidSelectParser对于子查询别名的全局解析 * * @throws NoSuchMethodException * @throws InvocationTargetException * @throws IllegalAccessException */ @Test public void testGroupByWithViewAlias() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { String functionColumn = "select id from (select h.id from hotnews h union select h.title from hotnews h ) as t1 group by t1.id;"; Object result = invokeGroupBy(functionColumn); Assert.assertEquals(functionColumn, ((String[]) result)[0]); } public Object invokeGroupBy(String functionColumn) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { Map aliaColumns = new TreeMap<>(); SQLIdentifierExpr sqlExpr = new SQLIdentifierExpr(functionColumn);// mock(SQLIdentifierExpr.class); SQLIdentifierExpr expr = new SQLIdentifierExpr(functionColumn); // mock(SQLIdentifierExpr.class); List groupByItems = new ArrayList<>(); groupByItems.add(sqlExpr); // when((sqlExpr).getName()).thenReturn(functionColumn); Class c = DruidSelectParser.class; Method method = c.getDeclaredMethod("buildGroupByCols", new Class[]{List.class, Map.class}); method.setAccessible(true); return method.invoke(druidSelectParser, groupByItems, aliaColumns); } } ================================================ FILE: src/test/java/io/mycat/parser/druid/DruidSequenceHandlerTest.java ================================================ package io.mycat.parser.druid; import static junit.framework.Assert.assertEquals; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.junit.Test; import io.mycat.config.model.SystemConfig; import io.mycat.route.parser.druid.DruidSequenceHandler; /** * 获取MYCAT SEQ 表名。 */ public class DruidSequenceHandlerTest { @Test public void test() { DruidSequenceHandler handler = new DruidSequenceHandler(SystemConfig.SEQUENCEHANDLER_LOCALFILE,SystemConfig.SEQUENCEHANDLER_PATTERN); String sql = "select next value for mycatseq_xxxx".toUpperCase(); String tableName = handler.getTableName(sql); assertEquals(tableName, "XXXX"); sql = " insert into test(id,sid)values(next value for MYCATSEQ_TEST,1)".toUpperCase(); tableName = handler.getTableName(sql); assertEquals(tableName, "TEST"); sql = " insert into test(id,sid)values(next value for MYCATSEQ_TEST ,1)".toUpperCase(); tableName = handler.getTableName(sql); assertEquals(tableName, "TEST"); sql = " insert into test(id)values(next value for MYCATSEQ_TEST )".toUpperCase(); tableName = handler.getTableName(sql); assertEquals(tableName, "TEST"); } @Test public void test2() { DruidSequenceHandler handler = new DruidSequenceHandler(SystemConfig.SEQUENCEHANDLER_LOCALFILE,SystemConfig.SEQUENCEHANDLER_PATTERN); String sql = "/* APPLICATIONNAME=DBEAVER 3.3.2 - MAIN CONNECTION */ SELECT NEXT VALUE FOR MYCATSEQ_XXXX".toUpperCase(); String tableName = handler.getTableName(sql); assertEquals(tableName, "XXXX"); } public static void main(String[] args) { String patten="(?:(\\s*next\\s+value\\s+for\\s*MYCATSEQ_(\\w+))(,|\\)|\\s)*)+"; Pattern pattern = Pattern.compile(patten,Pattern.CASE_INSENSITIVE); String sql="insert into test(id,sid)values( next value for MYCATSEQ_TEST ,1)"; Matcher matcher = pattern.matcher(sql); System.out.println(matcher.find()); System.out.println(matcher.group(1)); System.out.println(matcher.group(2)); } } ================================================ FILE: src/test/java/io/mycat/parser/druid/DruidUpdateParserTest.java ================================================ package io.mycat.parser.druid; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; import com.alibaba.druid.sql.ast.statement.SQLUpdateStatement; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlUpdateStatement; import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.TableConfig; import io.mycat.route.RouteResultset; import io.mycat.route.parser.druid.impl.DruidUpdateParser; import org.junit.Assert; import org.junit.Test; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.sql.SQLNonTransientException; import java.util.ArrayList; import java.util.List; import java.util.Map; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; /** * @author Hash Zhang * @version 1.0.0 * @date 2016/7/7 */ public class DruidUpdateParserTest { /** * 测试单表更新分片字段 * @throws NoSuchMethodException */ @Test public void testUpdateShardColumn() throws NoSuchMethodException{ throwExceptionParse("update hotnews set id = 1 where name = 234;", true); throwExceptionParse("update hotnews set id = 1 where id = 3;", true); throwExceptionParse("update hotnews set id = 1, name = '123' where id = 1 and name = '234'", false); throwExceptionParse("update hotnews set id = 1, name = '123' where id = 1 or name = '234'", true); throwExceptionParse("update hotnews set id = 'A', name = '123' where id = 'A' and name = '234'", false); throwExceptionParse("update hotnews set id = 'A', name = '123' where id = 'A' or name = '234'", true); throwExceptionParse("update hotnews set id = 1.5, name = '123' where id = 1.5 and name = '234'", false); throwExceptionParse("update hotnews set id = 1.5, name = '123' where id = 1.5 or name = '234'", true); throwExceptionParse("update hotnews set id = 1, name = '123' where name = '234' and (id = 1 or age > 3)", true); throwExceptionParse("update hotnews set id = 1, name = '123' where id = 1 and (name = '234' or age > 3)", false); // 子查询，特殊的运算符between等情况 throwExceptionParse("update hotnews set id = 1, name = '123' where id = 1 and name in (select name from test)", false); throwExceptionParse("update hotnews set id = 1, name = '123' where name = '123' and id in (select id from test)", true); throwExceptionParse("update hotnews set id = 1, name = '123' where id between 1 and 3", true); throwExceptionParse("update hotnews set id = 1, name = '123' where id between 1 and 3 and name = '234'", true); throwExceptionParse("update hotnews set id = 1, name = '123' where id between 1 and 3 or name = '234'", true); throwExceptionParse("update hotnews set id = 1, name = '123' where id = 1 and name between '124' and '234'", false); } /** * 测试单表别名更新分片字段 * @throws NoSuchMethodException */ @Test public void testAliasUpdateShardColumn() throws NoSuchMethodException{ throwExceptionParse("update hotnews h set h.id = 1 where h.name = 234;", true); throwExceptionParse("update hotnews h set h.id = 1 where h.id = 3;", true); throwExceptionParse("update hotnews h set h.id = 1, h.name = '123' where h.id = 1 and h.name = '234'", false); throwExceptionParse("update hotnews h set h.id = 1, h.name = '123' where h.id = 1 or h.name = '234'", true); throwExceptionParse("update hotnews h set h.id = 'A', h.name = '123' where h.id = 'A' and h.name = '234'", false); throwExceptionParse("update hotnews h set h.id = 'A', h.name = '123' where h.id = 'A' or h.name = '234'", true); throwExceptionParse("update hotnews h set h.id = 1.5, h.name = '123' where h.id = 1.5 and h.name = '234'", false); throwExceptionParse("update hotnews h set h.id = 1.5, h.name = '123' where h.id = 1.5 or h.name = '234'", true); throwExceptionParse("update hotnews h set id = 1, h.name = '123' where h.id = 1 and h.name = '234'", false); throwExceptionParse("update hotnews h set h.id = 1, h.name = '123' where id = 1 or h.name = '234'", true); throwExceptionParse("update hotnews h set h.id = 1, h.name = '123' where h.name = '234' and (h.id = 1 or h.age > 3)", true); throwExceptionParse("update hotnews h set h.id = 1, h.name = '123' where h.id = 1 and (h.name = '234' or h.age > 3)", false); } public void throwExceptionParse(String sql, boolean throwException) throws NoSuchMethodException { MySqlStatementParser parser = new MySqlStatementParser(sql); List statementList = parser.parseStatementList(); SQLStatement sqlStatement = statementList.get(0); MySqlUpdateStatement update = (MySqlUpdateStatement) sqlStatement; SchemaConfig schemaConfig = mock(SchemaConfig.class); Map tables = mock(Map.class); TableConfig tableConfig = mock(TableConfig.class); String tableName = "hotnews"; when((schemaConfig).getTables()).thenReturn(tables); when(tables.get(tableName)).thenReturn(tableConfig); when(tableConfig.getParentTC()).thenReturn(null); RouteResultset routeResultset = new RouteResultset(sql, 11); Class c = DruidUpdateParser.class; Method method = c.getDeclaredMethod("confirmShardColumnNotUpdated", new Class[]{SQLUpdateStatement.class, SchemaConfig.class, String.class, String.class, String.class, RouteResultset.class}); method.setAccessible(true); try { method.invoke(c.newInstance(), update, schemaConfig, tableName, "ID", "", routeResultset); if (throwException) { System.out.println("未抛异常，解析通过则不对！"); Assert.assertTrue(false); } else { System.out.println("未抛异常，解析通过，此情况分片字段可能在update语句中但是实际不会被更新"); Assert.assertTrue(true); } } catch (Exception e) { if (throwException) { System.out.println(e.getCause().getClass()); Assert.assertTrue(e.getCause() instanceof SQLNonTransientException); System.out.println("抛异常原因为SQLNonTransientException则正确"); } else { System.out.println("抛异常，需要检查"); Assert.assertTrue(false); } } } /* * 添加一个static方法用于打印一个SQL的where子句，比如这样的一条SQL: * update mytab t set t.ptn_col = 'A', col1 = 3 where ptn_col = 'A' and (col1 = 4 or col2 > 5); * where子句的语法树如下 * AND * / \ * = OR * / \ / \ * ptn_col 'A' = > * / \ / \ * col1 4 col2 5 * 其输出如下，(按层输出，并且每层最后输出下一层的节点数目) * BooleanAnd Num of nodes in next level: 2 * Equality BooleanOr Num of nodes in next level: 4 * ptn_col 'A' Equality Equality Num of nodes in next level: 4 * col1 4 col2 5 Num of nodes in next level: 0 * * 因为大部分的update的where子句都比较简单，按层次打印应该足够清晰，未来可以完全按照逻辑打印类似上面的整棵树结构 */ public static void printWhereClauseAST(SQLExpr sqlExpr) { // where子句的AST sqlExpr可以通过 MySqlUpdateStatement.getWhere(); 获得 if (sqlExpr == null) return; ArrayList exprNode = new ArrayList<>(); int i = 0, curLevel = 1, nextLevel = 0; SQLExpr iterExpr; exprNode.add(sqlExpr); while (true) { iterExpr = exprNode.get(i++); if (iterExpr == null) break; if (iterExpr instanceof SQLBinaryOpExpr) { System.out.print(((SQLBinaryOpExpr) iterExpr).getOperator()); } else { System.out.print(iterExpr.toString()); } System.out.print("\t"); curLevel--; if (iterExpr instanceof SQLBinaryOpExpr) { if (((SQLBinaryOpExpr) iterExpr).getLeft() != null) { exprNode.add(((SQLBinaryOpExpr) iterExpr).getLeft()); nextLevel++; } if (((SQLBinaryOpExpr) iterExpr).getRight() != null) { exprNode.add(((SQLBinaryOpExpr) iterExpr).getRight()); nextLevel++; } } if (curLevel == 0) { System.out.println("\t\tNum of nodes in next level: " + nextLevel); curLevel = nextLevel; nextLevel = 0; } if (exprNode.size() == i) break; } } } ================================================ FILE: src/test/java/io/mycat/parser/druid/MycatSchemaStatVisitorTest.java ================================================ package io.mycat.parser.druid; import java.util.ArrayList; import java.util.List; import org.junit.Test; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; import com.alibaba.druid.sql.parser.SQLStatementParser; import com.alibaba.druid.stat.TableStat.Condition; import io.mycat.route.parser.druid.MycatSchemaStatVisitor; import junit.framework.Assert; /** * TODO: 增加描述 * * @author user * @date 2015-6-2 下午5:50:25 * @version 0.1.0 * @copyright wonhigh.cn */ public class MycatSchemaStatVisitorTest { /** * 从注解中解析 mycat schema */ @Test public void test4() { String sql = "/*!mycat:schema = helper1 */update adm_task a set a.remark = 'test' where id=1"; Assert.assertEquals(getSchema(sql), "helper1."); sql = "/*!mycat:schema=helper1*/update adm_task a set a.remark = 'test' where id=1"; Assert.assertEquals(getSchema(sql), "helper1."); sql = "/*!mycat:schema= helper1*/update adm_task a set a.remark = 'test' where id=1"; Assert.assertEquals(getSchema(sql), "helper1."); System.out.println(getSchema(sql)); } /** * 3层嵌套or语句 */ @Test public void test1() { String sql = "select id from travelrecord " + " where id = 1 and ( fee=3 or days=5 or (traveldate = '2015-05-04 00:00:07.375' " + " and (user_id=2 or fee=days or fee = 0))) and id=2" ; List> list = getConditionList(sql); Assert.assertEquals(list.size(), 5); Assert.assertEquals(list.get(0).size(), 2); Assert.assertEquals(list.get(1).size(), 2); Assert.assertEquals(list.get(2).size(), 3); Assert.assertEquals(list.get(3).size(), 4); Assert.assertEquals(list.get(4).size(), 3); Assert.assertEquals(list.get(0).get(0).toString(), "travelrecord.days = 5"); Assert.assertEquals(list.get(0).get(1).toString(), "travelrecord.id = (1, 2)"); Assert.assertEquals(list.get(1).get(0).toString(), "travelrecord.fee = 3"); Assert.assertEquals(list.get(1).get(1).toString(), "travelrecord.id = (1, 2)"); Assert.assertEquals(list.get(2).get(0).toString(), "travelrecord.fee = 0"); Assert.assertEquals(list.get(2).get(1).toString(), "travelrecord.traveldate = 2015-05-04 00:00:07.375"); Assert.assertEquals(list.get(2).get(2).toString(), "travelrecord.id = (1, 2)"); Assert.assertEquals(list.get(3).get(0).toString(), "travelrecord.fee ="); Assert.assertEquals(list.get(3).get(1).toString(), "travelrecord.days ="); Assert.assertEquals(list.get(3).get(2).toString(), "travelrecord.traveldate = 2015-05-04 00:00:07.375"); Assert.assertEquals(list.get(3).get(3).toString(), "travelrecord.id = (1, 2)"); Assert.assertEquals(list.get(4).get(0).toString(), "travelrecord.user_id = 2"); Assert.assertEquals(list.get(4).get(1).toString(), "travelrecord.traveldate = 2015-05-04 00:00:07.375"); Assert.assertEquals(list.get(4).get(2).toString(), "travelrecord.id = (1, 2)"); System.out.println(list.size()); } /** * 1层嵌套or语句 */ @Test public void test2() { String sql = "select id from travelrecord " + " where id = 1 and ( fee=3 or days=5 or name = 'zhangsan')" ; List> list = getConditionList(sql); Assert.assertEquals(list.size(), 3); Assert.assertEquals(list.get(0).size(), 2); Assert.assertEquals(list.get(1).size(), 2); Assert.assertEquals(list.get(2).size(), 2); Assert.assertEquals(list.get(0).get(0).toString(), "travelrecord.name = zhangsan"); Assert.assertEquals(list.get(0).get(1).toString(), "travelrecord.id = 1"); Assert.assertEquals(list.get(1).get(0).toString(), "travelrecord.days = 5"); Assert.assertEquals(list.get(1).get(1).toString(), "travelrecord.id = 1"); Assert.assertEquals(list.get(2).get(0).toString(), "travelrecord.fee = 3"); Assert.assertEquals(list.get(2).get(1).toString(), "travelrecord.id = 1"); } /** * 1层嵌套or语句 */ @Test public void test3() { String sql = "select id from travelrecord " + " where id = 1 and fee=3 or days=5 or name = 'zhangsan'" ; List> list = getConditionList(sql); Assert.assertEquals(list.size(), 3); Assert.assertEquals(list.get(0).size(), 1); Assert.assertEquals(list.get(1).size(), 1); Assert.assertEquals(list.get(2).size(), 2); Assert.assertEquals(list.get(0).get(0).toString(), "travelrecord.name = zhangsan"); Assert.assertEquals(list.get(1).get(0).toString(), "travelrecord.days = 5"); Assert.assertEquals(list.get(2).get(0).toString(), "travelrecord.id = 1"); Assert.assertEquals(list.get(2).get(1).toString(), "travelrecord.fee = 3"); } String sql = "select count(*) count from (select *\r\n" + " from (select b.sales_count,\r\n" + " b.special_type,\r\n" + " a.prod_offer_id offer_id,\r\n" + " a.alias_name as offer_name,\r\n" + " (select c.attr_value_name\r\n" + " from attr_value c\r\n" + " where c.attr_id = '994001448'\r\n" + " and c.attr_value = b.special_type) as attr_value_name,\r\n" + " a.offer_type offer_kind,\r\n" + " a.offer_comments,\r\n" + " a.is_ge\r\n" + " from prod_offer a, special_offer b\r\n" + " where a.prod_offer_id = b.prod_offer_id\r\n" + " and (a.offer_type = '11' or a.offer_type = '10')\r\n" + " and (b.region_id = '731' or b.region_id = '10000000')\r\n" + " and a.status_cd = '10'\r\n" + " and b.special_type = '0'\r\n" + " union all\r\n" + " select b.sales_count,\r\n" + " b.special_type,\r\n" + " a.prod_offer_id offer_id,\r\n" + " a.alias_name as offer_name,\r\n" + " (select c.attr_value_name\r\n" + " from attr_value c\r\n" + " where c.attr_id = '994001448'\r\n" + " and c.attr_value = b.special_type) as attr_value_name,\r\n" + " a.offer_type offer_kind,\r\n" + " a.offer_comments,\r\n" + " a.is_ge\r\n" + " from prod_offer a, special_offer b\r\n" + " where a.prod_offer_id = b.prod_offer_id\r\n" + " and (a.offer_type = '11' or a.offer_type = '10')\r\n" + " and (b.region_id = '731' or b.region_id = '10000000')\r\n" + " and a.status_cd = '10'\r\n" + " and b.special_type = '1'\r\n" + " and exists (select 1\r\n" + " from prod_offer_channel l\r\n" + " where a.prod_offer_id = l.prod_offer_id\r\n" + " and l.channel_id = '11')\r\n" + " and not exists\r\n" + " (select 1\r\n" + " from product_offer_cat ml\r\n" + " where ml.prod_offer_id = a.prod_offer_id\r\n" + " and ml.offer_cat_type = '89')\r\n" + " and (exists (select 1\r\n" + " from sales_restrication\r\n" + " where object_id = a.prod_offer_id\r\n" + " and domain_id = '1965868'\r\n" + " and restrication_flag = '0'\r\n" + " and domain_type = '19F'\r\n" + " and state = '00A') or exists\r\n" + " (select 1\r\n" + " from sales_restrication\r\n" + " where object_id = a.prod_offer_id\r\n" + " and domain_id = '843073100000000'\r\n" + " and restrication_flag = '0'\r\n" + " and domain_type = '19E'\r\n" + " and state = '00A') or exists\r\n" + " (select 1\r\n" + " from sales_restrication\r\n" + " where object_id = a.prod_offer_id\r\n" + " and domain_id = '1965868'\r\n" + " and restrication_flag = '0'\r\n" + " and domain_type = '19X'\r\n" + " and state = '00A'\r\n" + " and (max_value = 1 or min_value = 1)\r\n" + " and extended_field = '731') or exists\r\n" + " (select 1\r\n" + " from sales_restrication\r\n" + " where object_id = a.prod_offer_id\r\n" + " and domain_id = concat('-', '11')\r\n" + " and restrication_flag = '0'\r\n" + " and domain_type = '19F'\r\n" + " and state = '00A') or not exists\r\n" + " (select 1\r\n" + " from sales_restrication\r\n" + " where object_id = a.prod_offer_id\r\n" + " and restrication_flag = '0'\r\n" + " and (domain_type in ('19F', '19E') or\r\n" + " (domain_type = '19X' and\r\n" + " extended_field = '731' and\r\n" + " (max_value = 1 or min_value = 1)))\r\n" + " and state = '00A'))\r\n" + " and not exists (select 1\r\n" + " from sales_restrication\r\n" + " where object_id = a.prod_offer_id\r\n" + " and domain_id = '1965868'\r\n" + " and restrication_flag = '1'\r\n" + " and domain_type = '19F'\r\n" + " and state = '00A')\r\n" + " and not exists (select 1\r\n" + " from sales_restrication\r\n" + " where object_id = a.prod_offer_id\r\n" + " and domain_id = '843073100000000'\r\n" + " and restrication_flag = '1'\r\n" + " and domain_type = '19E'\r\n" + " and state = '00A')\r\n" + " and not exists\r\n" + " (select 1\r\n" + " from sales_restrication\r\n" + " where object_id = a.prod_offer_id\r\n" + " and domain_id = '1965868'\r\n" + " and restrication_flag = '1'\r\n" + " and domain_type = '19X'\r\n" + " and state = '00A'\r\n" + " and (min_value = 1 or max_value = 1)\r\n" + " and extended_field = '731')\r\n" + " and not exists (select 1\r\n" + " from sales_restrication\r\n" + " where object_id = a.prod_offer_id\r\n" + " and domain_id = concat('-', '11')\r\n" + " and restrication_flag = '1'\r\n" + " and domain_type = '19F'\r\n" + " and state = '00A')\r\n" + " and exists\r\n" + " (select 1\r\n" + " from prod_offer_region v1\r\n" + " where v1.prod_offer_id = a.prod_offer_id\r\n" + " and (v1.common_region_id = '731' or\r\n" + " v1.common_region_id = '10000000' or\r\n" + " v1.common_region_id = '73101'))) t\r\n" + " order by t.sales_count desc)"; /** * 8层以上 嵌套or语句 */ @Test public void test5() { List> list = getConditionList(sql); Assert.assertTrue(list.size() < 100); } private String getSchema(String sql) { SQLStatementParser parser =null; parser = new MySqlStatementParser(sql); MycatSchemaStatVisitor visitor = null; SQLStatement statement = null; //解析出现问题统一抛SQL语法错误 try { statement = parser.parseStatement(); visitor = new MycatSchemaStatVisitor(); } catch (Exception e) { e.printStackTrace(); } statement.accept(visitor); return visitor.getCurrentTable(); } private List> getConditionList(String sql) { SQLStatementParser parser =null; parser = new MySqlStatementParser(sql); MycatSchemaStatVisitor visitor = null; SQLStatement statement = null; //解析出现问题统一抛SQL语法错误 try { statement = parser.parseStatement(); visitor = new MycatSchemaStatVisitor(); } catch (Exception e) { e.printStackTrace(); } statement.accept(visitor); List> mergedConditionList = new ArrayList>(); if(visitor.hasOrCondition()) {//包含or语句 //TODO //根据or拆分 mergedConditionList = visitor.splitConditions(); } else {//不包含OR语句 mergedConditionList.add(visitor.getConditions()); } return mergedConditionList; } } ================================================ FILE: src/test/java/io/mycat/parser/primitive/TestFunctionParser.java ================================================ package io.mycat.parser.primitive; import io.mycat.route.parser.primitive.FunctionParser; import io.mycat.route.parser.primitive.Model.Function; import junit.framework.Assert; import org.junit.Test; import java.sql.SQLNonTransientException; /** * @author Hash Zhang * @version 1.0.0 * @date 2016/7/26 */ public class TestFunctionParser { @Test public void testMultiFunctions() throws SQLNonTransientException { Assert.assertEquals("[arg1, a.t]",testFunctionParse("function1(arg1,a.t)")); Assert.assertEquals("[arg1, a.t]",testFunctionParse("function1(arg1,a.t,\"ast(,)\")")); Assert.assertEquals("[arg1, a.t, c.t, x]",testFunctionParse("function1(arg1,a.t,\"ast(,)\",\",\",function2(c.t,function3(x)))")); Assert.assertEquals("[arg1, a.t, c.t, x]",testFunctionParse("function1(arg1,a.t,\"ast(,)\",\",\",function2(c.t,\"(,)\",function3(function4(x))))")); } public String testFunctionParse(String function) throws SQLNonTransientException { Function function1 = FunctionParser.parseFunction(function); return FunctionParser.getFields(function1).toString(); } } ================================================ FILE: src/test/java/io/mycat/parser/util/PairUtilTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.parser.util; import junit.framework.TestCase; import org.junit.Assert; import org.junit.Test; import io.mycat.route.parser.util.Pair; import io.mycat.route.parser.util.PairUtil; /** * @author mycat */ public class PairUtilTest extends TestCase { @Test public void testSequenceSlicing() { Assert.assertEquals(new Pair(0, 2), PairUtil.sequenceSlicing("2")); Assert.assertEquals(new Pair(1, 2), PairUtil.sequenceSlicing("1: 2")); Assert.assertEquals(new Pair(1, 0), PairUtil.sequenceSlicing(" 1 :")); Assert.assertEquals(new Pair(-1, 0), PairUtil.sequenceSlicing("-1: ")); Assert.assertEquals(new Pair(-1, 0), PairUtil.sequenceSlicing(" -1:0")); Assert.assertEquals(new Pair(0, 0), PairUtil.sequenceSlicing(" :")); } @Test public void splitIndexTest() { String src1 = "offer_group[10]"; Pair pair1 = PairUtil.splitIndex(src1, '[', ']'); Assert.assertEquals("offer_group", pair1.getKey()); Assert.assertEquals(Integer.valueOf(10), pair1.getValue()); String src2 = "offer_group"; Pair pair2 = PairUtil.splitIndex(src2, '[', ']'); Assert.assertEquals("offer_group", pair2.getKey()); Assert.assertEquals(Integer.valueOf(-1), pair2.getValue()); } } ================================================ FILE: src/test/java/io/mycat/performance/AbstractMultiTreadBatchTester.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.performance; import java.io.FileInputStream; import java.sql.SQLException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.Iterator; import java.util.LinkedList; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicLong; public abstract class AbstractMultiTreadBatchTester { SimpleConPool conPool; protected AtomicLong finshiedCount = new AtomicLong(); protected AtomicLong failedCount = new AtomicLong(); protected int threadCount = 0;// 线程数 protected String url; protected String user; protected String password; protected ExecutorService executor; long start; protected String[] rangeItems; protected boolean outputMiddleInf = true; protected String sqlFile; public boolean parseArgs(String[] args) { if (args.length < 5) { System.out .println("input param,format: [jdbcurl] [user] [password] [threadpoolsize] [recordrange or customer sql file] "); System.out .println("jdbc:mysql://localhost:8066/TESTDB test test 10 \"0-300M,300M1-600M,600M1-900M\" "); System.out .println("jdbc:mysql://localhost:8066/TESTDB test test 10 file=mytempate.sql "); return false; } url = args[0]; user = args[1]; password = args[2]; threadCount = Integer.parseInt(args[3]); if (args[4].contains("file=")) { sqlFile = args[4].substring(args[4].indexOf('=') + 1); } else { rangeItems = args[4].split(","); } return true; } public void run(String[] args) throws Exception { if (!this.parseArgs(args)) { return; } startTest(); runAndReport(); } public void addFinshed(int count) { finshiedCount.addAndGet(count); } public ArrayList createJobs(SimpleConPool conPool, long minId, long maxId) throws Exception { long recordCount = maxId - minId + 1; int batchSize = 1000; long totalBatch = recordCount / batchSize; ArrayList jobs = new ArrayList(); for (int i = 0; i < totalBatch; i++) { long startId = minId + i * batchSize; long endId = (startId + batchSize - 1); if (endId >= maxId) { endId = maxId; } else if (i == totalBatch - 1) { endId = maxId; } long myCount = endId - startId + 1; Runnable job = createJob(getConPool(), myCount, 100, startId, finshiedCount, failedCount); // System.out.println("job record id is " + startId + "-" + endId); jobs.add(job); } return jobs; } public abstract Runnable createJob(SimpleConPool conPool2, long myCount, int batchSize, long startId, AtomicLong finshiedCount2, AtomicLong failedCount2); @SuppressWarnings("unchecked") public ArrayList[] createAllJobs() throws Exception { if (sqlFile != null) {// from sql template file java.util.Properties pros = RandomDataValueUtil .loadFromPropertyFile(sqlFile); long total = Long.valueOf(pros.getProperty("total")); String sqlTemplate = pros.getProperty("sql"); String batchSizeStr=pros.getProperty("batch"); String autocommitStr=pros.getProperty("autocommit"); boolean autocommit= autocommitStr!=null && Boolean.valueOf(autocommitStr); int batchSize=batchSizeStr==null?100:Integer.parseInt(batchSizeStr); System.out.println("total record "+total+ " batch size:"+batchSize+" autocomit "+autocommit); return createSQLTemplateJobs(total, sqlTemplate,batchSize,autocommit); } else { ArrayList[] allJobs = new ArrayList[rangeItems.length]; for (int i = 0; i < rangeItems.length; i++) { String[] items = rangeItems[i].split("-"); long min = parseLong(items[0]); long max = parseLong(items[1]); allJobs[i] = createJobs(conPool, min, max); } return allJobs; } } private ArrayList[] createSQLTemplateJobs(long total, String sqlTemplate,int batchSize,boolean autocommit) throws Exception { LinkedList sqlTemplateItems = RandomDataValueUtil .parselRandVarTemplateString(sqlTemplate); @SuppressWarnings("unchecked") ArrayList[] allJobs = new ArrayList[1]; long totalBatch = total / threadCount; allJobs[0] = new ArrayList((int) threadCount); for (int i = 0; i < threadCount; i++) { allJobs[0].add(new UserTableInsertJob(getConPool(), totalBatch, batchSize, finshiedCount, failedCount, sqlTemplateItems,autocommit)); } if (totalBatch * threadCount < total) { allJobs[0].add(new UserTableInsertJob(getConPool(), total - totalBatch * threadCount, batchSize, finshiedCount, failedCount, sqlTemplateItems,autocommit)); } return allJobs; } public void addFailed(int count) { failedCount.addAndGet(count); } public SimpleConPool getConPool() throws SQLException, ClassNotFoundException { if (conPool == null) { Class.forName("com.mysql.jdbc.Driver"); conPool = new SimpleConPool(url, user, password, threadCount); } return conPool; } @SuppressWarnings("unchecked") public void startTest() throws Exception { executor = Executors.newFixedThreadPool(threadCount); System.out.println("create jobs ..."); ArrayList[] allJobs = createAllJobs(); Iterator[] itors = new Iterator[allJobs.length]; for (int i = 0; i < allJobs.length; i++) { itors[i] = allJobs[i].iterator(); } System.out.println("create jobs finished ,begin run test..."); int total = 0; start = System.currentTimeMillis(); boolean finished = false; while (!finished) { finished = true; for (int i = 0; i < itors.length; i++) { if (itors[i].hasNext()) { total++; executor.execute(itors[i].next()); if (finished) { finished = !itors[i].hasNext(); } } } } // executor.execute(job); System.out.println("success create job count: " + total + " teset threads: " + threadCount); } public void runAndReport() throws InterruptedException { executor.shutdown(); SimpleDateFormat df = new SimpleDateFormat("dd HH:mm:ss"); while (!executor.isTerminated()) { if (outputMiddleInf) { long sucess = finshiedCount.get() - failedCount.get(); System.out.println(df.format(new Date()) + " finished records :" + finshiedCount.get() + " failed:" + failedCount.get() + " speed:" + sucess * 1000.0 / (System.currentTimeMillis() - start)); } Thread.sleep(1000); } long usedTime = (System.currentTimeMillis() - start) / 1000; System.out.println("finishend:" + finshiedCount.get() + " failed:" + failedCount.get()); long sucess = finshiedCount.get() - failedCount.get(); System.out.println("used time total:" + usedTime + "seconds"); System.out.println("tps:" + sucess / (usedTime + 0.1)); } /** * can parse values like 200M ,200K,200M1(2000001) * * @param val * @return */ private static long parseLong(String val) { val = val.toUpperCase(); int indx = val.indexOf("M"); int plus = 10000; if (indx < 0) { indx = val.indexOf("K"); plus = 1000; } if (indx > 0) { String longVal = val.substring(0, indx); long theVale = Long.parseLong(longVal) * plus; String remain = val.substring(indx + 1); if (remain.length() > 0) { theVale += Integer.parseInt(remain); } return theVale; } else { return Long.parseLong(val); } } } ================================================ FILE: src/test/java/io/mycat/performance/GoodsInsertJob.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.performance; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; public class GoodsInsertJob implements Runnable { private final long endId; private long finsihed; private final int batchSize; private final AtomicLong finshiedCount; private final AtomicLong failedCount; Calendar date = Calendar.getInstance(); DateFormat datafomat = new SimpleDateFormat("yyyy-MM-dd"); private final SimpleConPool conPool; public GoodsInsertJob(SimpleConPool conPool, long totalRecords, int batchSize, long startId, AtomicLong finshiedCount, AtomicLong failedCount) { super(); this.conPool = conPool; this.endId = startId + totalRecords - 1; this.batchSize = batchSize; this.finsihed = startId; this.finshiedCount = finshiedCount; this.failedCount = failedCount; } private int insert(Connection con,List> list) throws SQLException { PreparedStatement ps; String sql = "insert into goods (id,name ,good_type,good_img_url,good_created ,good_desc, price ) values(?,? ,?,?,? ,?, ?)"; ps = con.prepareStatement(sql); for (Map map : list) { ps.setLong(1, Long.parseLong(map.get("id"))); ps.setString(2, (String) map.get("name")); ps.setShort(3, Short.parseShort(map.get("good_type"))); ps.setString(4, (String) map.get("good_img_url")); ps.setString(5, (String) map.get("good_created")); ps.setString(6, (String) map.get("good_desc")); ps.setDouble(7, Double.parseDouble(map.get("price"))); ps.addBatch(); } ps.executeBatch(); return list.size(); } private List> getNextBatch() { if (finsihed >= endId) { return Collections.emptyList(); } long end = (finsihed + batchSize) < this.endId ? (finsihed + batchSize) : endId; // the last batch if (end + batchSize > this.endId) { end = this.endId; } List> list = new ArrayList>( ); for (long i = finsihed; i < end; i++) { Map m = new HashMap(); m.put("id", i + ""); m.put("name", "googs " + i); m.put("good_type", i % 100 + ""); m.put("good_img_url", "http://openclouddb.org/" + i); m.put("good_created", getRandomDay(i)); m.put("good_desc", "best goods " + i); m.put("price", (i + 0.0) % 1000 + ""); list.add(m); } finsihed += list.size(); return list; } private String getRandomDay(long i) { int month = Long.valueOf(i % 11 + 1).intValue(); int day = Long.valueOf(i % 27 + 1).intValue(); date.set(Calendar.MONTH, month); date.set(Calendar.DAY_OF_MONTH, day); return datafomat.format(date.getTime()); } @Override public void run() { Connection con = null; try { List> batch = getNextBatch(); while (!batch.isEmpty()) { try { if (con == null || con.isClosed()) { con = conPool.getConnection(); con.setAutoCommit(true); } insert(con, batch); finshiedCount.addAndGet(batch.size()); } catch (Exception e) { failedCount.addAndGet(batch.size()); e.printStackTrace(); } batch = getNextBatch(); } } finally { if (con != null) { this.conPool.returnCon(con); } } } } ================================================ FILE: src/test/java/io/mycat/performance/RandomDataValueUtil.java ================================================ package io.mycat.performance; import java.io.FileInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Random; /** * genarate random test data * such as * values ('$date{yyyyMMddHHmmsss-[2014-2015]y}/psn$date{yyyy}s/$int(0-9999)/16767:20725','$char(2,0-99) OPP_$enum(BJ,SH,GZ,SZ)_$int(0-9)',$int(10,11),$int(400,420,500,600,800),$int(0-1000),$int(0-100),Sint(0-10),$int(0-99),'201408040028317067b41c0db-4a93-4360-9eb4-e159d1dbef45',$phone,2,2014071715,2315998,1397,152317998,1395,'0000'); * @author wuzhih * */ /** * * @author wuzhih * */ public class RandomDataValueUtil { /** * eval template contains random vars and replace them with real value alues * ( * '${date(yyyyMMddHHmmsss-[2014-2015]y)}/psn${date(yyyy)}s/${int(0-9999)}/1 * 6 7 6 7 : 2 0 7 2 5 ' , ' $ { s t r i n g ( 2 , 0 - 9 9 )} * OPP_${enum(BJ,SH,GZ,SZ)}_${int(0-9)} * ',${int(10,11)},$int(400,420,500,600,800),$int(0-1000),$int(0-100),$int(0-10),$int(0-99),'201408040028317067b41c0db-4a93-4360-9eb4-e159d1 * d b e f 4 5 ' , $ p h o n e , 2 , 2 0 1 4 0 7 1 7 1 5 , 2 3 1 5 9 9 8 , 1 * 3 9 7 , 1 5 2 3 1 7 9 9 8 , 1 3 9 5 , ' 0 0 0 0 ' ) * * @param templateStr * @return * @throws IllegalAccessException * @throws InstantiationException */ public static LinkedList parselRandVarTemplateString( String templateStr) throws Exception { char[] chars = templateStr.toCharArray(); LinkedList stringItems = new LinkedList(); int curPos = 0; int prevPattenEndPos = 0; while (curPos < chars.length) { char c = chars[curPos]; if (c == '$' && curPos + 1 < chars.length && chars[curPos + 1] == '{') { int start = curPos; curPos += 2; int end = -1; if (curPos < chars.length) { for (int i = curPos; i < chars.length; i++) { if (chars[i] == '}') { end = i; // found pattern if (prevPattenEndPos < start) {// some constant // string chars StringItem item = new StringItem(); item.initString(templateStr.substring( prevPattenEndPos, start)); stringItems.add(item); } // add variable pattern item stringItems.add(StringItemFactory .parseVarPattern(templateStr.substring( curPos, end))); prevPattenEndPos = end + 1; curPos = end + 1; break; } } if (end == -1) { // not found pattern end throw new RuntimeException( "can't find var patten end pos ,start at " + start); } } else { curPos++; } } else { curPos++; } } // add last if (prevPattenEndPos < templateStr.length()) { StringItem item = new StringItem(); item.initString(templateStr.substring(prevPattenEndPos, templateStr.length())); stringItems.add(item); } return stringItems; } public static Properties loadFromPropertyFile(String sqlFile) throws IOException { java.util.Properties pros = new Properties(); FileInputStream fin = null; fin = new FileInputStream(sqlFile); pros.load(fin); fin.close(); return pros; } public static String evalRandValueString(LinkedList items) { StringBuilder sb = new StringBuilder(); for (StringItem item : items) { sb.append(item.getValue()); } return sb.toString(); } public static void main(String[] args) throws Exception { String sqlTemplate = "insert into opp_call (logthread, instanceid,callresult,partner, app_id,api_id,apiversion,format,token , phone,calltype, calldate,callminutes,callcost,ecipcalltime,ecipcallcost ,respcode) values ('${date(yyyyMMddHHmmssSSS-[2014-2015]y)}/psn2002s/${int(0-9999)}/${int(1111-9999)}:20725','${char([0-9]2:2)} OPP_${enum(BJ,SH,WU,GZ)}_1',10,${int(10-999)},${int(10-99)},100,3,15,'${date(yyyyMMddHHmmssSSS-[2014-2015]y}${char([a-f,0-9]8:8)}-${char([a-f,0-9]4:4)}-${char([0-9]4:4)}-9eb4-${char([a-f,0-9]12:12)}',${phone(139-189)},2,${date(yyyyMMddHH-[2014-2015]y},2315998,1397,${date(HHmmssSSS)},${int(100-1000)},'${enum(0000,0001,0002)}');"; System.out.println("SQL template:\r\n" + sqlTemplate); LinkedList allItems = parselRandVarTemplateString(sqlTemplate); // for (StringItem item : allItems) { // System.out.println(item); // } System.out.println("Random SQLs "); int total = 5; for (int i = 0; i < total; i++) { System.out.println(evalRandValueString(allItems)); } } } class StringItemFactory { private static final Map> strItemsMap = new HashMap>(); static { strItemsMap.put("date", DateVarItem.class); strItemsMap.put("int", IntVarItem.class); strItemsMap.put("char", CharVarItem.class); strItemsMap.put("enum", EnumVarItem.class); strItemsMap.put("phone", PhoneVarItem.class); } public static StringItem parseVarPattern(String content) throws InstantiationException, IllegalAccessException { String name = content.substring(0, content.indexOf('(')); Class cls = strItemsMap.get(name); if (cls == null) { throw new RuntimeException("not find var type of " + name); } StringItem obj = cls.newInstance(); obj.initString(content); return obj; } } class PhoneVarItem extends StringItem { // long[] rang = { 13900000000L, 19900000000L }; public void initString(String content) { int start = content.indexOf('('); int end = content.indexOf(')'); String range = content.substring(start + 1, end); String[] items = range.split("-"); rang[0] = Long.valueOf(patchLenth(items[0], 11 - items[0].length())); rang[1] = Long.valueOf(patchLenth(items[1], 11 - items[1].length())); } public static String patchLenth(String origin, int patchlen) { StringBuffer sb = new StringBuffer(); sb.append(origin); for (int i = 0; i < patchlen; i++) { sb.append('0'); } return sb.toString(); } public String getValue() { long span = rang[1] - rang[0] + 1; return Math.abs(rand.nextInt()) % span + rang[0] + ""; } @Override public String toString() { return "PhoneVarItem [rang=" + Arrays.toString(rang) + "]"; } } class EnumVarItem extends StringItem { // {enum(BJ,SH,GZ,SZ) String[] enums = {}; public void initString(String content) { int start = content.indexOf('('); int end = content.indexOf(')'); String range = content.substring(start + 1, end); String[] items = range.split(","); enums = items; } public String getValue() { return enums[Math.abs(rand.nextInt()) % enums.length]; } @Override public String toString() { return "EnumarItem [enums=" + Arrays.toString(enums) + "]"; } } class CharVarItem extends StringItem { List rang = new ArrayList(); int minLen = 1; int maxLen = 256; public void initString(String content) { // char([a-z]1:3) int start = content.indexOf('['); int end = content.indexOf(']'); String[] items = content.substring(start + 1, end).split(","); for (String itemStr : items) { if (itemStr.indexOf('-') > 0) { String[] pair = itemStr.split("-"); char[] curRange = new char[2]; curRange[0] = pair[0].charAt(0); curRange[1] = pair[1].charAt(0); rang.add(curRange); } } int splitPos = content.indexOf(':'); if (splitPos > 0) { int splitStart = (end == -1) ? content.indexOf('(') : end; int splitEnd = content.indexOf(')'); String[] pair = content.substring(splitStart + 1, splitEnd).split( ":"); minLen = Integer.valueOf(pair[0]); maxLen = Integer.valueOf(pair[1]); } } public String getValue() { int lenth = Math.abs(rand.nextInt()) % (maxLen - minLen + 1) + minLen; char[] chars = new char[lenth]; for (int i = 0; i < chars.length; i++) { int randInt = Math.abs(rand.nextInt()); int choise = randInt % rang.size(); char[] choisedRange = rang.get(choise); char randChar = (char) (randInt % (choisedRange[1] - choisedRange[0] + 1) + choisedRange[0]); chars[i] = randChar; } return new String(chars); } @Override public String toString() { return "CharVarItem [rang=" + rang + ", minLen=" + minLen + ", maxLen=" + maxLen + "]"; } } class StringItem { protected static final Random rand = new Random(); public String content; public StringItem() { } public String getValue() { return content; } public void initString(String content) { this.content = content; } @Override public String toString() { return "StringItem [content=" + content + "]"; } } class IntVarItem extends StringItem { long[] rang = { 0, Integer.MAX_VALUE }; long[] enums = {}; boolean isEnumInt = false; public void initString(String content) { int start = content.indexOf('('); int end = content.indexOf(')'); if (content.indexOf('-') > 0) { String range = content.substring(start + 1, end); String[] items = range.split("-"); rang[0] = Integer.valueOf(items[0]); rang[1] = Integer.valueOf(items[1]); } else { isEnumInt = true; String range = content.substring(start + 1, end); String[] items = range.split(","); enums = new long[items.length]; for (int i = 0; i < enums.length; i++) { enums[i] = Long.valueOf(items[i]); } } } public String getValue() { if (isEnumInt) { return enums[Math.abs(rand.nextInt()) % enums.length] + ""; } else { long span = rang[1] - rang[0] + 1; return Math.abs(rand.nextInt()) % span + rang[0] + ""; } } @Override public String toString() { return "IntVarItem [rang=" + Arrays.toString(rang) + ", enums=" + Arrays.toString(enums) + ", isEnumInt=" + isEnumInt + "]"; } } class DateVarItem extends StringItem { String format; int[] yearRang = { 1970, 2999 }; int[] monRang = { 1, 12 }; int[] dayRang = { 1, 31 }; int[] hourRang = { 0, 23 }; int[] minuteRang = { 0, 59 }; int[] secondRang = { 0, 59 }; int[] sssRang = { 0, 999 }; public DateVarItem() { } public void initString(String content) { int fmtEndPos = content.indexOf('-'); if (fmtEndPos == -1) { fmtEndPos = content.indexOf(')'); } format = content.substring(5, fmtEndPos); int yearP = content.indexOf("]y", fmtEndPos); if (yearP > 0) { yearRang = getRangeofPattern(content, yearP); } int monthP = content.indexOf("]M", fmtEndPos); if (monthP > 0) { monRang = getRangeofPattern(content, monthP); } int dayP = content.indexOf("]d", fmtEndPos); if (dayP > 0) { dayRang = getRangeofPattern(content, dayP); } int hourP = content.indexOf("]H", fmtEndPos); if (hourP > 0) { hourRang = getRangeofPattern(content, hourP); } int minuteP = content.indexOf("]m", fmtEndPos); if (minuteP > 0) { minuteRang = getRangeofPattern(content, minuteP); } int secondP = content.indexOf("]s", fmtEndPos); if (secondP > 0) { secondRang = getRangeofPattern(content, secondP); } int millisS = content.indexOf("]S", fmtEndPos); if (millisS > 0) { sssRang = getRangeofPattern(content, millisS); } } private static final int[] getRangeofPattern(String theString, int endPos) { String subString = theString.substring(0, endPos); int start = subString.lastIndexOf('['); String range = subString.substring(start + 1, endPos); String[] items = range.split("-"); int[] values = new int[2]; values[0] = Integer.valueOf(items[0]); values[1] = Integer.valueOf(items[1]); return values; } public String getValue() { int yearSpan = yearRang[1] - yearRang[0] + 1; int year = Math.abs(rand.nextInt()) % yearSpan + yearRang[0]; int monthSpan = monRang[1] - monRang[0] + 1; int month = Math.abs(rand.nextInt()) % monthSpan + monRang[0]; int daySpan = dayRang[1] - dayRang[0] + 1; int day = Math.abs(rand.nextInt()) % daySpan + dayRang[0]; int hourSpan = hourRang[1] - hourRang[0] + 1; int hour = Math.abs(rand.nextInt()) % hourSpan + hourRang[0]; int minuteSpan = minuteRang[1] - minuteRang[0] + 1; int minute = Math.abs(rand.nextInt()) % minuteSpan + minuteRang[0]; int secondSpan = secondRang[1] - secondRang[0] + 1; int second = Math.abs(rand.nextInt()) % secondSpan + secondRang[0]; int sssSpan = sssRang[1] - sssRang[0] + 1; int sss = Math.abs(rand.nextInt()) % sssSpan + sssRang[0]; java.util.Calendar cl = Calendar.getInstance(); cl.set(Calendar.YEAR, year); cl.set(Calendar.MONTH, month); cl.set(Calendar.DATE, day); cl.set(Calendar.HOUR_OF_DAY, hour); cl.set(Calendar.MINUTE, minute); cl.set(Calendar.SECOND, second); cl.set(Calendar.MILLISECOND, sss); return new java.text.SimpleDateFormat(format).format(cl.getTime()); } @Override public String toString() { return "DateVarItem [format=" + format + ", yearRang=" + Arrays.toString(yearRang) + ", monRang=" + Arrays.toString(monRang) + ", dayRang=" + Arrays.toString(dayRang) + "]"; } } ================================================ FILE: src/test/java/io/mycat/performance/SimpleConPool.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.performance; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.util.concurrent.CopyOnWriteArrayList; public class SimpleConPool { private final String url; private final String user; private final String password; private CopyOnWriteArrayList cons = new CopyOnWriteArrayList(); public SimpleConPool(String url, String user, String password, int maxCon) throws SQLException { super(); this.url = url; this.user = user; this.password = password; for (int i = 0; i < maxCon; i++) { cons.add(getCon()); } System.out.println("success ful created connections ,total :" + maxCon); } public void close() { for (Connection con : this.cons) { try { if (con != null && !con.isClosed()) { con.close(); } } catch (Exception e) { e.printStackTrace(); } } cons.clear(); } private Connection getCon() throws SQLException { Connection theCon = DriverManager.getConnection(url, user, password); return theCon; } public void returnCon(Connection con) { try { if (con.isClosed()) { System.out.println("closed connection ,aband"); } else { this.cons.add(con); } } catch (SQLException e) { e.printStackTrace(); } } public Connection getConnection() throws SQLException { Connection con = null; if (cons.isEmpty()) { System.out.println("warn no connection in pool,create new one"); con = getCon(); return con; } else { con = cons.remove(0); } if (con.isClosed()) { System.out.println("warn connection closed ,create new one"); con = getCon(); } return con; } } ================================================ FILE: src/test/java/io/mycat/performance/TestGlobalTableInsertPerf.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.performance; import java.util.concurrent.atomic.AtomicLong; /** * @author shenzhw * */ public class TestGlobalTableInsertPerf extends AbstractMultiTreadBatchTester{ public boolean parseArgs(String[] args) { if (args.length < 5) { System.out .println("input param,format: [jdbcurl] [user] [password] [threadpoolsize] recordcount "); return false; } url = args[0]; user = args[1]; password = args[2]; threadCount = Integer.parseInt(args[3]); rangeItems = new String[]{"0-"+Integer.parseInt(args[4])}; return true; } public static void main(String[] args) throws Exception { new TestGlobalTableInsertPerf().run(args); } @Override public Runnable createJob(SimpleConPool conPool2, long myCount, int batch, long startId, AtomicLong finshiedCount2, AtomicLong failedCount2) { return new GoodsInsertJob(conPool2, myCount, batch, startId, finshiedCount, failedCount); } } ================================================ FILE: src/test/java/io/mycat/performance/TestInsertGlobalSeqPerf.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.performance; import java.util.concurrent.atomic.AtomicLong; /** * @author wuzh * */ public class TestInsertGlobalSeqPerf extends AbstractMultiTreadBatchTester { public static void main(String[] args) throws Exception { new TestInsertGlobalSeqPerf().run(args); } @Override public Runnable createJob(SimpleConPool conPool2, long myCount, int batch, long startId, AtomicLong finshiedCount2, AtomicLong failedCount2) { return new TravelRecordGlobalSeqInsertJob(conPool2, myCount, batch, startId, finshiedCount, failedCount); } } ================================================ FILE: src/test/java/io/mycat/performance/TestInsertPerf.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.performance; import java.util.concurrent.atomic.AtomicLong; /** * @author wuzh * */ public class TestInsertPerf extends AbstractMultiTreadBatchTester { public static void main(String[] args) throws Exception { new TestInsertPerf().run(args); } @Override public Runnable createJob(SimpleConPool conPool2, long myCount, int batch, long startId, AtomicLong finshiedCount2, AtomicLong failedCount2) { return new TravelRecordInsertJob(conPool2, myCount, batch, startId, finshiedCount, failedCount); } } ================================================ FILE: src/test/java/io/mycat/performance/TestMaxConnection.java ================================================ /* Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.performance; import java.sql.SQLException; public class TestMaxConnection { public static void main(String[] args) { if (args.length < 4) { System.out .println("input param,format: [jdbcurl] [user] [password] [poolsize] "); return; } String url = args[0]; String user = args[1]; String password = args[2]; Integer poolsize = Integer.parseInt(args[3]); SimpleConPool pool = null; long start=System.currentTimeMillis(); try { pool = new SimpleConPool(url, user, password, poolsize); } catch (SQLException e) { e.printStackTrace(); } System.out.println("success create threadpool ,used time "+(System.currentTimeMillis()-start)); int i = 0; try { for (i = 0; i < poolsize; i++) { pool.getConnection().createStatement() .execute("select * from company limit 1"); } } catch (SQLException e) { System.out.println("exectute sql err " + i + " err:" + e.toString()); } finally { pool.close(); } } } ================================================ FILE: src/test/java/io/mycat/performance/TestMergeSelectPerf.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.performance; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.util.ArrayList; import java.util.concurrent.atomic.AtomicInteger; /** * test multi node merge (min,max ,sum ,order by ,limit ) performance * * @author wuzhi * */ public class TestMergeSelectPerf { private static AtomicInteger finshiedCount = new AtomicInteger(); private static AtomicInteger failedCount = new AtomicInteger(); public static void addFinshed(int count) { finshiedCount.addAndGet(count); } public static void addFailed(int count) { failedCount.addAndGet(count); } private static Connection getCon(String url, String user, String passwd) throws SQLException { Connection theCon = DriverManager.getConnection(url, user, passwd); return theCon; } public static void main(String[] args) throws Exception { Class.forName("com.mysql.jdbc.Driver"); if (args.length < 5) { System.out .println("input param,format: [jdbcurl] [user] [password] [threadpoolsize] [executetimes] "); return; } int threadCount = 0;// 线程数 String url = args[0]; String user = args[1]; String password = args[2]; threadCount = Integer.parseInt(args[3]); int executetimes = Integer.parseInt(args[4]); System.out.println("concerent threads:" + threadCount); System.out.println("execute sql times:" + executetimes); ArrayList threads = new ArrayList(threadCount); ArrayList jobs = new ArrayList( threadCount); for (int i = 0; i < threadCount; i++) { try { Connection con = getCon(url, user, password); System.out.println("create thread " + i); TravelRecordMergeJob job = new TravelRecordMergeJob(con, executetimes, finshiedCount, failedCount); Thread thread = new Thread(job); threads.add(thread); jobs.add(job); } catch (Exception e) { System.out.println("failed create thread " + i + " err " + e.toString()); } } System.out.println("all thread started,waiting finsh..."); System.out.println("success create thread count: " + threads.size()); for (Thread thread : threads) { thread.start(); } long start=System.currentTimeMillis(); System.out.println("all thread started,waiting finsh..."); boolean notFinished = true; while (notFinished) { notFinished = false; for (Thread thread : threads) { if (thread.isAlive()) { notFinished = true; break; } } report(jobs); Thread.sleep(1000); } report(jobs); System.out.println("total time :" +(System.currentTimeMillis()-start)/1000); } public static void report(ArrayList jobs) { int tps = 0; for (TravelRecordMergeJob job : jobs) { tps += job.getTPS(); } System.out.println("finishend:" + finshiedCount.get() + " failed:" + failedCount.get()); System.out.println("tps:" +tps); } } ================================================ FILE: src/test/java/io/mycat/performance/TestMergeSorter.java ================================================ package io.mycat.performance; //package org.opencloudb.performance; // //import java.nio.ByteBuffer; //import java.util.Collection; //import java.util.HashSet; //import java.util.Random; //import java.util.Set; // //import org.opencloudb.mpp.ColMeta; //import org.opencloudb.mpp.OrderCol; //import org.opencloudb.mpp.tmp.FastRowDataSorter; //import org.opencloudb.mpp.tmp.MutilNodeMergeItf; //import org.opencloudb.mysql.BufferUtil; //import org.opencloudb.net.mysql.RowDataPacket; // //public class TestMergeSorter { // // // @Test // public static void main(String[] args) { // ColMeta colMeta = new ColMeta(0, ColMeta.COL_TYPE_INT); // OrderCol col = new OrderCol(colMeta, OrderCol.COL_ORDER_TYPE_DESC); // OrderCol[] orderCols = { col }; // MutilNodeMergeItf sorter = new FastRowDataSorter(orderCols); // byte idLen = 4; // byte packId = 0; // int maxCount = 10000; // int bound = maxCount * 2; // Random rd = new Random(); // Set set = new HashSet(); // while (set.size() < maxCount) { // set.add(rd.nextInt(bound)); // } // for (Integer integer : set) { // String name = "name".concat(String.valueOf(integer)); // int length = name.length(); // RowDataPacket row = new RowDataPacket(2); // ByteBuffer buffer = ByteBuffer.allocate(3 + 1 + 1 + 4 + 1 + length); // BufferUtil.writeUB3(buffer, buffer.capacity());// PACKLEN // buffer.put(packId++);// packID // buffer.put(idLen);// LEN // BufferUtil.writeInt(buffer, integer); // buffer.put((byte) length); // buffer.put(name.getBytes()); // row.read(buffer.array()); // sorter.addRow(row); // } // set.clear(); // System.gc(); // System.out.println("add finished"); // for (int i = 0; i < 100; i++) { // long st = System.currentTimeMillis(); // Collection res = sorter.getResult(); // long end = System.currentTimeMillis();// 37.246//15.196 // System.out.println((end - st) / 1000.0); // } // // for (RowDataPacket row : res) { // // byte[] x = row.fieldValues.get(0); // // byte[] name = row.fieldValues.get(1); // // ByteBuffer wrap = ByteBuffer.wrap(x); // // wrap.order(ByteOrder.LITTLE_ENDIAN); // // System.out.println(wrap.getInt()+","+new String(name)); // // // // } // sorter.close(); // } //} ================================================ FILE: src/test/java/io/mycat/performance/TestRandomDataUtil.java ================================================ package io.mycat.performance; import java.util.Arrays; import java.util.LinkedList; import junit.framework.Assert; import org.junit.Test; public class TestRandomDataUtil { @Test public void testParselRandVarTemplateString() throws Exception { LinkedList result=RandomDataValueUtil.parselRandVarTemplateString("${date(yyyy-MM-dd HH [2014-2015]y-[1-6]M-[1-31]d-[7-21]H-[0-59]m-[5-33]s-[111-990]S)}"); Assert.assertEquals(true, result.size()==1); DateVarItem item=(DateVarItem) result.get(0); Assert.assertEquals(Arrays.toString(item.dayRang),Arrays.toString(new int[]{1,31})); Assert.assertEquals(Arrays.toString(item.yearRang),Arrays.toString(new int[]{2014,2015})); Assert.assertEquals(Arrays.toString(item.monRang),Arrays.toString(new int[]{1,6})); Assert.assertEquals(Arrays.toString(item.hourRang),Arrays.toString(new int[]{7,21})); Assert.assertEquals(Arrays.toString(item.secondRang),Arrays.toString(new int[]{5,33})); Assert.assertEquals(Arrays.toString(item.sssRang),Arrays.toString(new int[]{111,990})); } } ================================================ FILE: src/test/java/io/mycat/performance/TestSelectPerf.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.performance; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.text.DecimalFormat; import java.util.LinkedList; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicInteger; /** * * @author shenzhw * */ public class TestSelectPerf { private static AtomicInteger finshiedCount = new AtomicInteger(); private static AtomicInteger failedCount = new AtomicInteger(); private static LinkedList sqlTemplateItems; private static long minId; private static long maxId; private static int executeTimes; public static void addFinshed(int count) { finshiedCount.addAndGet(count); } public static void addFailed(int count) { failedCount.addAndGet(count); } private static Connection getCon(String url, String user, String passwd) throws SQLException { Connection theCon = DriverManager.getConnection(url, user, passwd); return theCon; } private static SelectJob createQueryJob(Connection con) { SelectJob job = null; if (sqlTemplateItems != null) { job = new UserTableSelectJob(con, sqlTemplateItems, executeTimes, finshiedCount, failedCount); } else { job = new TravelRecordSelectJob(con, minId, maxId, executeTimes, finshiedCount, failedCount); } return job; } private static void doTest(String url, String user, String password, int threadCount, long minId, long maxId, int executetimes, boolean outmidle) { final CopyOnWriteArrayList threads = new CopyOnWriteArrayList(); final CopyOnWriteArrayList jobs = new CopyOnWriteArrayList(); for (int i = 0; i < threadCount; i++) { try { Connection con = getCon(url, user, password); System.out.println("create thread " + i); SelectJob job = createQueryJob(con); Thread thread = new Thread((Runnable) job); threads.add(thread); jobs.add(job); } catch (Exception e) { System.out.println("failed create thread " + i + " err " + e.toString()); } } System.out.println("success create thread count: " + threads.size()); for (Thread thread : threads) { thread.start(); } System.out.println("all thread started,waiting finsh..."); long start = System.currentTimeMillis(); boolean notFinished = true; int remainThread = 0; while (notFinished) { notFinished = false; remainThread = 0; for (Thread thread : threads) { if (thread.isAlive()) { notFinished = true; remainThread++; } } if (remainThread < threads.size() / 2) { System.out .println("warning many test threads finished ,qps may NOT Accurate ,alive threads:" + remainThread); } if (outmidle) { report(jobs); } try { Thread.sleep(1000); } catch (InterruptedException e) { e.printStackTrace(); } } report(jobs); System.out.println("finished all,total time :" + (System.currentTimeMillis() - start) / 1000); } public static void main(String[] args) throws Exception { Class.forName("com.mysql.jdbc.Driver"); if (args.length < 5) { System.out .println("input param,format: [jdbcurl] [user] [password] [threadpoolsize] [executetimes] [minId-maxId|sqlfile] [repeat]"); System.out .println("jdbc:mysql://localhost:8066/TESTDB test test 10 10000 1-1000000 1 "); System.out .println("jdbc:mysql://localhost:8066/TESTDB test test 10 10000 file=mytempate.sql 1"); return; } int threadCount = 0;// 线程数 String url = args[0]; String user = args[1]; String password = args[2]; threadCount = Integer.parseInt(args[3]); int repeate = 1; executeTimes = Integer.parseInt(args[4]); System.out.println("execute sql times:" + executeTimes); String param5 = args[5]; if (param5.contains("file=")) { String sqlFile = args[5].substring(args[5].indexOf('=') + 1); java.util.Properties pros = RandomDataValueUtil .loadFromPropertyFile(sqlFile); String sqlTemplate = pros.getProperty("sql"); sqlTemplateItems = RandomDataValueUtil .parselRandVarTemplateString(sqlTemplate); } else { minId = Integer.parseInt((args[5].split("-"))[0]); maxId = Integer.parseInt((args[5].split("-"))[1]); System.out.println("concerent threads:" + threadCount); System.out.println("maxId:" + maxId); } if (args.length > 6) { repeate = Integer.parseInt(args[6]); System.out.println("repeat test times:" + repeate); } for (int i = 0; i < repeate; i++) { try { doTest(url, user, password, threadCount, minId, maxId, executeTimes, repeate < 2); } catch (Exception e) { e.printStackTrace(); } } } public static void report(CopyOnWriteArrayList jobs) { double tps = 0; long maxTTL = 0; long minTTL = Integer.MAX_VALUE; long ttlCount = 0; long ttlSum = 0; DecimalFormat df = new DecimalFormat("0.00"); for (SelectJob job : jobs) { double jobTps = job.getTPS(); if (jobTps > 0) { tps += job.getTPS(); if (job.getMaxTTL() > maxTTL) { maxTTL = job.getMaxTTL(); } if (job.getMinTTL() < minTTL) { minTTL = job.getMinTTL(); } ttlCount += job.getValidTTLCount(); ttlSum += job.getValidTTLSum(); } } double avgSum =(ttlCount > 0) ? (ttlSum+0.0) / ttlCount : 0; System.out.println("finishend:" + finshiedCount.get() + " failed:" + failedCount.get() + " qps:" + df.format(tps) + ",query time min:" + minTTL + "ms,max:" + maxTTL + "ms,avg:" + df.format(avgSum) ); } } interface SelectJob { double getTPS(); long getValidTTLSum(); long getValidTTLCount(); long getMinTTL(); long getMaxTTL(); } ================================================ FILE: src/test/java/io/mycat/performance/TestUpdatePerf.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.performance; import java.util.concurrent.atomic.AtomicLong; /** * @author wuzh * */ public class TestUpdatePerf extends AbstractMultiTreadBatchTester { private int repeats = 1; public TestUpdatePerf(int repearts) { this.repeats = repearts; if (repeats > 1) { this.outputMiddleInf = false; } } public static void main(String[] args) throws Exception { int repeats = 1; if (args.length > 5) { repeats = Integer.parseInt(args[5]); } for (int i = 0; i < repeats; i++) { new TestUpdatePerf(repeats).run(args); } } @Override public Runnable createJob(SimpleConPool conPool2, long myCount, int batch, long startId, AtomicLong finshiedCount2, AtomicLong failedCount2) { return new TravelRecordUpdateJob(conPool2, myCount, batch, startId, finshiedCount, failedCount); } } ================================================ FILE: src/test/java/io/mycat/performance/TravelRecordGlobalSeqInsertJob.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.performance; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; public class TravelRecordGlobalSeqInsertJob implements Runnable { private final long endId; private long finsihed; private final int batchSize; private final AtomicLong finshiedCount; private final AtomicLong failedCount; Calendar date = Calendar.getInstance(); DateFormat datafomat = new SimpleDateFormat("yyyy-MM-dd"); private final SimpleConPool conPool; public TravelRecordGlobalSeqInsertJob(SimpleConPool conPool, long totalRecords, int batchSize, long startId, AtomicLong finshiedCount, AtomicLong failedCount) { super(); this.conPool = conPool; this.endId = startId + totalRecords - 1; this.batchSize = batchSize; this.finsihed = startId; this.finshiedCount = finshiedCount; this.failedCount = failedCount; } private int insert(Connection con, List> list) throws SQLException { PreparedStatement ps; String sql = "insert into travelrecord (user_id,traveldate,fee,days) values(?,?,?,?,?)"; ps = con.prepareStatement(sql); for (Map map : list) { //ps.setLong(1, Long.parseLong(map.get("id"))); ps.setString(1, (String) map.get("user_id")); ps.setString(2, (String) map.get("traveldate")); ps.setString(3, (String) map.get("fee")); ps.setString(4, (String) map.get("days")); ps.addBatch(); } ps.executeBatch(); con.commit(); ps.clearBatch(); ps.close(); return list.size(); } private List> getNextBatch() { if (finsihed >= endId) { return Collections.emptyList(); } long end = (finsihed + batchSize) < this.endId ? (finsihed + batchSize) : endId; // the last batch if (end + batchSize > this.endId) { end = this.endId; } List> list = new ArrayList>( ); for (long i = finsihed; i <= end; i++) { Map m = new HashMap(); m.put("id", i + ""); m.put("user_id", "user " + i); m.put("traveldate", getRandomDay(i)); m.put("fee", i % 10000 + ""); m.put("days", i % 10000 + ""); list.add(m); } // System.out.println("finsihed :" + finsihed + "-" + end); finsihed += list.size(); return list; } private String getRandomDay(long i) { int month = Long.valueOf(i % 11 + 1).intValue(); int day = Long.valueOf(i % 27 + 1).intValue(); date.set(Calendar.MONTH, month); date.set(Calendar.DAY_OF_MONTH, day); return datafomat.format(date.getTime()); } @Override public void run() { Connection con = null; try { List> batch = getNextBatch(); while (!batch.isEmpty()) { try { if (con == null || con.isClosed()) { con = conPool.getConnection(); con.setAutoCommit(false); } insert(con, batch); finshiedCount.addAndGet(batch.size()); } catch (Exception e) { e.printStackTrace(); try { con.rollback(); } catch (SQLException e1) { e1.printStackTrace(); e1.printStackTrace(); } failedCount.addAndGet(batch.size()); } batch = getNextBatch(); } } finally { if (con != null) { this.conPool.returnCon(con); } } } } ================================================ FILE: src/test/java/io/mycat/performance/TravelRecordInsertJob.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.performance; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; public class TravelRecordInsertJob implements Runnable { private final long endId; private long finsihed; private final long batchSize; private final AtomicLong finshiedCount; private final AtomicLong failedCount; private final SimpleConPool conPool; public TravelRecordInsertJob(SimpleConPool conPool, long totalRecords, long batchSize, long startId, AtomicLong finshiedCount, AtomicLong failedCount) { super(); this.conPool = conPool; this.endId = startId + totalRecords - 1; this.batchSize = batchSize; this.finsihed = startId; this.finshiedCount = finshiedCount; this.failedCount = failedCount; } private long insert(Connection con, List> list) throws SQLException { PreparedStatement ps; String sql = "insert into travelrecord (id,user_id,traveldate,fee,days) values(?,?,?,?,?)"; ps = con.prepareStatement(sql); for (Map map : list) { ps.setLong(1, Long.parseLong(map.get("id"))); ps.setString(2, (String) map.get("user_id")); ps.setString(3, (String) map.get("traveldate")); ps.setString(4, (String) map.get("fee")); ps.setString(5, (String) map.get("days")); ps.addBatch(); } ps.executeBatch(); con.commit(); ps.clearBatch(); ps.close(); return list.size(); } private List> getNextBatch() { if (finsihed >= endId) { return Collections.emptyList(); } long end = (finsihed + batchSize) < this.endId ? (finsihed + batchSize) : endId; // the last batch if (end + batchSize > this.endId) { end = this.endId; } List> list = new ArrayList>( Integer.valueOf((end - finsihed) + "")); for (long i = finsihed; i <= end; i++) { Map m = new HashMap(); m.put("id", i + ""); m.put("user_id", "user " + i); m.put("traveldate", getRandomDay(i)); m.put("fee", i % 10000 + ""); m.put("days", i % 10000 + ""); list.add(m); } // System.out.println("finsihed :" + finsihed + "-" + end); finsihed += list.size(); return list; } private String getRandomDay(long i) { int year = Long.valueOf(i % 10 + 2000).intValue(); int month = Long.valueOf(i % 11 + 1).intValue(); int day = Long.valueOf(i % 27 + 1).intValue(); return year + "-" + month + "-" + day; } @Override public void run() { Connection con = null; try { List> batch = getNextBatch(); while (!batch.isEmpty()) { try { if (con == null || con.isClosed()) { con = conPool.getConnection(); con.setAutoCommit(false); } insert(con, batch); finshiedCount.addAndGet(batch.size()); } catch (Exception e) { e.printStackTrace(); try { con.rollback(); } catch (SQLException e1) { e1.printStackTrace(); e1.printStackTrace(); } failedCount.addAndGet(batch.size()); } batch = getNextBatch(); } } finally { if (con != null) { this.conPool.returnCon(con); } } } } ================================================ FILE: src/test/java/io/mycat/performance/TravelRecordMergeJob.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.performance; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; public class TravelRecordMergeJob implements Runnable { private final Connection con; private final int executeTimes; Calendar date = Calendar.getInstance(); DateFormat datafomat = new SimpleDateFormat("yyyy-MM-dd"); Random random = new Random(); private final AtomicInteger finshiedCount; private final AtomicInteger failedCount; private volatile long usedTime; private volatile int success; public TravelRecordMergeJob(Connection con, int executeTimes, AtomicInteger finshiedCount, AtomicInteger failedCount) { super(); this.con = con; this.executeTimes = executeTimes; this.finshiedCount = finshiedCount; this.failedCount = failedCount; } private void select() { ResultSet rs = null; try { String sql = "select sum(fee) total_fee, days,count(id),max(fee),min(fee) from travelrecord where days = " + (Math.abs(random.nextInt()) % 10000) + " or days =" + (Math.abs(random.nextInt()) % 10000)+ " group by days order by days desc"; rs = con.createStatement().executeQuery(sql); finshiedCount.addAndGet(1); success++; } catch (Exception e) { failedCount.addAndGet(1); e.printStackTrace(); } finally { if (rs != null) { try { rs.close(); } catch (SQLException e) { e.printStackTrace(); } } } } @Override public void run() { long start = System.currentTimeMillis(); for (int i = 0; i < executeTimes; i++) { this.select(); usedTime = System.currentTimeMillis() - start; } try { con.close(); } catch (SQLException e) { e.printStackTrace(); } } public long getUsedTime() { return this.usedTime; } public int getTPS() { if(usedTime>0) { return (int) (this.success*1000/this.usedTime); }else { return 0; } } } ================================================ FILE: src/test/java/io/mycat/performance/TravelRecordSelectJob.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.performance; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; public class TravelRecordSelectJob implements Runnable ,SelectJob{ private final Connection con; private final long minId; private final long maxId; private final int executeTimes; Random random = new Random(); private final AtomicInteger finshiedCount; private final AtomicInteger failedCount; private volatile long usedTime; private volatile long success; private volatile long maxTTL = 0; private volatile long minTTL = Integer.MAX_VALUE; private volatile long validTTLSum = 0; private volatile long validTTLCount = 0; public TravelRecordSelectJob(Connection con, long minId, long maxId, int executeTimes, AtomicInteger finshiedCount, AtomicInteger failedCount) { super(); this.con = con; this.minId = minId; this.maxId = maxId; this.executeTimes = executeTimes; this.finshiedCount = finshiedCount; this.failedCount = failedCount; } private long select() { ResultSet rs = null; long used = -1; try { String sql = "select * from travelrecord where id=" + ((Math.abs(random.nextLong()) % (maxId - minId)) + minId); long startTime = System.currentTimeMillis(); rs = con.createStatement().executeQuery(sql); if (rs.next()) { } used = System.currentTimeMillis() - startTime; finshiedCount.addAndGet(1); success++; } catch (Exception e) { failedCount.addAndGet(1); e.printStackTrace(); } finally { if (rs != null) { try { rs.close(); } catch (SQLException e) { e.printStackTrace(); } } } return used; } @Override public void run() { long curmaxTTL = this.maxTTL; long curminTTL = this.minTTL; long curvalidTTLSum = this.validTTLSum; long curvalidTTLCount = this.validTTLCount; long start = System.currentTimeMillis(); for (int i = 0; i < executeTimes; i++) { long ttlTime = this.select(); if (ttlTime != -1) { if (ttlTime > curmaxTTL) { curmaxTTL = ttlTime; } else if (ttlTime < curminTTL) { curminTTL = ttlTime; } curvalidTTLSum += ttlTime; curvalidTTLCount += 1; } usedTime = System.currentTimeMillis() - start; } maxTTL = curmaxTTL; minTTL = curminTTL; validTTLSum = curvalidTTLSum; validTTLCount = curvalidTTLCount; try { con.close(); } catch (SQLException e) { e.printStackTrace(); } } public long getUsedTime() { return this.usedTime; } public double getTPS() { if (usedTime > 0) { return (this.success * 1000+0.0) / this.usedTime; } else { return 0; } } public long getMaxTTL() { return maxTTL; } public long getMinTTL() { return minTTL; } public long getValidTTLSum() { return validTTLSum; } public long getValidTTLCount() { return validTTLCount; } public static void main(String[] args) { Random r = new Random(); for (int i = 0; i < 10; i++) { int f = r.nextInt(90000 - 80000) + 80000; System.out.println(f); } } } ================================================ FILE: src/test/java/io/mycat/performance/TravelRecordUpdateJob.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.performance; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; public class TravelRecordUpdateJob implements Runnable { private final long endId; private long finsihed; private final int batchSize; private final AtomicLong finshiedCount; private final AtomicLong failedCount; Calendar date = Calendar.getInstance(); final SimpleConPool conPool; DateFormat datafomat = new SimpleDateFormat("yyyy-MM-dd"); public TravelRecordUpdateJob(SimpleConPool conPool, long totalRecords, int batchSize, long startId, AtomicLong finshiedCount, AtomicLong failedCount) { super(); this.conPool = conPool; this.endId = startId + totalRecords - 1; this.batchSize = batchSize; this.finsihed = startId; this.finshiedCount = finshiedCount; this.failedCount = failedCount; } private int update(Connection con, List> list) throws SQLException { PreparedStatement ps; String sql = "update travelrecord set user_id =? ,traveldate=?,fee=?,days=? where id=?"; ps = con.prepareStatement(sql); for (Map map : list) { ps.setString(1, (String) map.get("user_id")); ps.setString(2, (String) map.get("traveldate")); ps.setString(3, (String) map.get("fee")); ps.setString(4, (String) map.get("days")); ps.setLong(5, Long.parseLong(map.get("id"))); ps.addBatch(); } ps.executeBatch(); return list.size(); } private List> getNextBatch() { if (finsihed >= endId) { return Collections.emptyList(); } long end = (finsihed + batchSize) < this.endId ? (finsihed + batchSize) : endId; // the last batch if (end + batchSize > this.endId) { end = this.endId; } List> list = new ArrayList>( ); for (long i = finsihed; i <= end; i++) { Map m = new HashMap(); m.put("id", i + ""); m.put("user_id", "user " + i); m.put("traveldate", getRandomDay(i)); m.put("fee", i % 10000 + ""); m.put("days", i % 7 + ""); list.add(m); } finsihed += list.size(); return list; } private String getRandomDay(long i) { int month = Long.valueOf(i % 11 + 1).intValue(); int day = Long.valueOf(i % 27 + 1).intValue(); date.set(Calendar.MONTH, month); date.set(Calendar.DAY_OF_MONTH, day); return datafomat.format(date.getTime()); } @Override public void run() { Connection con = null; try { List> batch = getNextBatch(); while (!batch.isEmpty()) { try { if (con == null || con.isClosed()) { con = conPool.getConnection(); con.setAutoCommit(true); } update(con, batch); finshiedCount.addAndGet(batch.size()); } catch (Exception e) { failedCount.addAndGet(batch.size()); e.printStackTrace(); } batch = getNextBatch(); } } finally { if (con != null) { this.conPool.returnCon(con); } } } } ================================================ FILE: src/test/java/io/mycat/performance/UserTableInsertJob.java ================================================ package io.mycat.performance; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.concurrent.atomic.AtomicLong; public class UserTableInsertJob implements Runnable { private long finsihed; private final long batchSize; private final AtomicLong finshiedCount; private final AtomicLong failedCount; private final SimpleConPool conPool; private final long totalRecords; private LinkedList sqlTemplateItems; private final boolean autocommit; public UserTableInsertJob(SimpleConPool conPool, long totalRecords, long batchSize, AtomicLong finshiedCount, AtomicLong failedCount, LinkedList sqlTemplateItems, boolean autoCommit) { super(); this.conPool = conPool; this.totalRecords = totalRecords; this.batchSize = batchSize; this.finshiedCount = finshiedCount; this.failedCount = failedCount; this.sqlTemplateItems = sqlTemplateItems; this.autocommit = autoCommit; } private long insert(Connection con, List list) throws SQLException { Statement stms = con.createStatement(); for (String sql : list) { stms.addBatch(sql); } stms.executeBatch(); if (this.autocommit == false) { con.commit(); } // stms.clearBatch(); stms.close(); return list.size(); } private List getNextBatch() { if (finsihed >= totalRecords) { return Collections.emptyList(); } int curBatchSize = (int) ((finsihed + batchSize) < totalRecords ? batchSize : (totalRecords - finsihed)); ArrayList list = new ArrayList(curBatchSize); for (long i = 0; i < curBatchSize; i++) { list.add(RandomDataValueUtil.evalRandValueString(sqlTemplateItems)); } // System.out.println("finsihed :" + finsihed + "-" + end); finsihed += list.size(); return list; } @Override public void run() { Connection con = null; try { List batch = getNextBatch(); while (!batch.isEmpty()) { try { if (con == null || con.isClosed()) { con = conPool.getConnection(); if (con.getAutoCommit() != autocommit) { con.setAutoCommit(autocommit); } } insert(con, batch); finshiedCount.addAndGet(batch.size()); } catch (Exception e) { System.out.println("caught err in thread :" + Thread.currentThread().getId() + " " + e); try { con.rollback(); } catch (SQLException e1) { System.out.println("caught err in thread :" + Thread.currentThread().getId() + " rollback err " + e1); } failedCount.addAndGet(batch.size()); } batch = getNextBatch(); } } finally { if (con != null) { this.conPool.returnCon(con); } } } } ================================================ FILE: src/test/java/io/mycat/performance/UserTableSelectJob.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.performance; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.util.LinkedList; import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; public class UserTableSelectJob implements Runnable, SelectJob { private final Connection con; private final int executeTimes; Random random = new Random(); private final AtomicInteger finshiedCount; private final AtomicInteger failedCount; private volatile long usedTime; private volatile long success; private volatile long maxTTL = 0; private volatile long minTTL = Integer.MAX_VALUE; private volatile long validTTLSum = 0; private volatile long validTTLCount = 0; private LinkedList sqlTemplateItems; public UserTableSelectJob(Connection con, LinkedList sqlTemplateItems, int executeTimes, AtomicInteger finshiedCount, AtomicInteger failedCount) { super(); this.con = con; this.sqlTemplateItems = sqlTemplateItems; this.executeTimes = executeTimes; this.finshiedCount = finshiedCount; this.failedCount = failedCount; } private long select() { ResultSet rs = null; long used = -1; try { String sql = RandomDataValueUtil .evalRandValueString(sqlTemplateItems); long startTime = System.currentTimeMillis(); rs = con.createStatement().executeQuery(sql); if (rs.next()) { } used = System.currentTimeMillis() - startTime; finshiedCount.addAndGet(1); success++; } catch (Exception e) { failedCount.addAndGet(1); e.printStackTrace(); } finally { if (rs != null) { try { rs.close(); } catch (SQLException e) { e.printStackTrace(); } } } return used; } @Override public void run() { long curmaxTTL = this.maxTTL; long curminTTL = this.minTTL; long curvalidTTLSum = this.validTTLSum; long curvalidTTLCount = this.validTTLCount; long start = System.currentTimeMillis(); for (int i = 0; i < executeTimes; i++) { long ttlTime = this.select(); if (ttlTime != -1) { if (ttlTime > curmaxTTL) { curmaxTTL = ttlTime; } else if (ttlTime < curminTTL) { curminTTL = ttlTime; } curvalidTTLSum += ttlTime; curvalidTTLCount += 1; } usedTime = System.currentTimeMillis() - start; if (i % 100 == 0) { maxTTL = curmaxTTL; minTTL = curminTTL; validTTLSum = curvalidTTLSum; validTTLCount = curvalidTTLCount; } } maxTTL = curmaxTTL; minTTL = curminTTL; validTTLSum = curvalidTTLSum; validTTLCount = curvalidTTLCount; try { con.close(); } catch (SQLException e) { e.printStackTrace(); } } public long getUsedTime() { return this.usedTime; } public double getTPS() { if (usedTime > 0) { return (this.success * 1000+0.0) / this.usedTime; } else { return 0; } } public long getMaxTTL() { return maxTTL; } public long getMinTTL() { return minTTL; } public long getValidTTLSum() { return validTTLSum; } public long getValidTTLCount() { return validTTLCount; } public static void main(String[] args) { Random r = new Random(); for (int i = 0; i < 10; i++) { int f = r.nextInt(90000 - 80000) + 80000; System.out.println(f); } } } ================================================ FILE: src/test/java/io/mycat/postgres/PostgresTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.postgres; import org.junit.Test; /** * @author mycat */ public class PostgresTest { @Test public void testNothing() { } } ================================================ FILE: src/test/java/io/mycat/queue/FixedQueue.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.queue; import java.util.concurrent.locks.ReentrantLock; /** * 固定容量的阻塞队列 * * @author mycat */ public final class FixedQueue { private final E[] items; private int putIndex; private int takeIndex; private int count; private final ReentrantLock lock; @SuppressWarnings("unchecked") public FixedQueue(int capacity) { if (capacity <= 0) { throw new IllegalArgumentException(); } this.items = (E[]) new Object[capacity]; this.lock = new ReentrantLock(); } public int size() { final ReentrantLock lock = this.lock; lock.lock(); try { return count; } finally { lock.unlock(); } } public boolean offer(E e) { if (e == null) { throw new NullPointerException(); } final ReentrantLock lock = this.lock; lock.lock(); try { if (count >= items.length) { return false; } else { insert(e); return true; } } finally { lock.unlock(); } } public E poll() { final ReentrantLock lock = this.lock; lock.lock(); try { if (count == 0) { return null; } return extract(); } finally { lock.unlock(); } } public void clear() { final E[] items = this.items; final ReentrantLock lock = this.lock; lock.lock(); try { int i = takeIndex; int j = count; while (j-- > 0) { items[i] = null; i = inc(i); } count = 0; putIndex = 0; takeIndex = 0; } finally { lock.unlock(); } } private void insert(E x) { items[putIndex] = x; putIndex = inc(putIndex); ++count; } private E extract() { E[] items = this.items; int i = takeIndex; E x = items[i]; items[i] = null; takeIndex = inc(i); --count; return x; } private int inc(int i) { return (++i == items.length) ? 0 : i; } } ================================================ FILE: src/test/java/io/mycat/queue/Queue.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.queue; /** * @author mycat */ public final class Queue { private final static int MIN_SHRINK_SIZE = 1024; private T[] items; private int count = 0; private int start = 0, end = 0; private int suggestedSize, size = 0; public Queue(int suggestedSize) { this.size = this.suggestedSize = suggestedSize; items = newArray(this.size); } public Queue() { this(4); } public synchronized void clear() { count = start = end = 0; size = suggestedSize; items = newArray(size); } public synchronized boolean hasElements() { return (count != 0); } public synchronized int size() { return count; } public synchronized void prepend(T item) { if (count == size) { makeMoreRoom(); } if (start == 0) { start = size - 1; } else { start--; } this.items[start] = item; count++; if (count == 1) { notify(); } } public synchronized void append(T item) { append0(item, count == 0); } public synchronized void appendSilent(T item) { append0(item, false); } public synchronized void appendLoud(T item) { append0(item, true); } public synchronized T getNonBlocking() { if (count == 0) { return null; } // pull the object off, and clear our reference to it T retval = items[start]; items[start] = null; start = (start + 1) % size; count--; return retval; } public synchronized void waitForItem() { while (count == 0) { try { wait(); } catch (InterruptedException e) { } } } public synchronized T get(long maxwait) { if (count == 0) { try { wait(maxwait); } catch (InterruptedException e) { } if (count == 0) { return null; } } return get(); } public synchronized T get() { while (count == 0) { try { wait(); } catch (InterruptedException e) { } } // pull the object off, and clear our reference to it T retval = items[start]; items[start] = null; start = (start + 1) % size; count--; // if we are only filling 1/8th of the space, shrink by half if ((size > MIN_SHRINK_SIZE) && (size > suggestedSize) && (count < (size >> 3))) { shrink(); } return retval; } private void append0(T item, boolean notify) { if (count == size) { makeMoreRoom(); } this.items[end] = item; end = (end + 1) % size; count++; if (notify) { notify(); } } private void makeMoreRoom() { T[] items = newArray(size * 2); System.arraycopy(this.items, start, items, 0, size - start); System.arraycopy(this.items, 0, items, size - start, end); start = 0; end = size; size *= 2; this.items = items; } // shrink by half private void shrink() { T[] items = newArray(size / 2); if (start > end) { // the data wraps around System.arraycopy(this.items, start, items, 0, size - start); System.arraycopy(this.items, 0, items, size - start, end + 1); } else { // the data does not wrap around System.arraycopy(this.items, start, items, 0, end - start + 1); } size = size / 2; start = 0; end = count; this.items = items; } @SuppressWarnings("unchecked") private T[] newArray(int size) { return (T[]) new Object[size]; } @Override public String toString() { StringBuilder buf = new StringBuilder(); buf.append("[count=").append(count); buf.append(", size=").append(size); buf.append(", start=").append(start); buf.append(", end=").append(end); buf.append(", elements={"); for (int i = 0; i < count; i++) { int pos = (i + start) % size; if (i > 0) { buf.append(", "); } buf.append(items[pos]); } buf.append("}]"); return buf.toString(); } } ================================================ FILE: src/test/java/io/mycat/queue/QueuePerfMain.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.queue; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import jsr166y.LinkedTransferQueue; /** * Queue 性能测试 * * @author mycat */ public class QueuePerfMain { private static byte[] testData = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 0 }; private static BlockingQueue arrayQueue = new ArrayBlockingQueue(5000000); private static FixedQueue fixedQueue = new FixedQueue(5000000); private static Queue testQueue = new Queue(); private static BlockingQueue linkedQueue = new LinkedBlockingQueue(); private static LinkedTransferQueue transferQueue = new LinkedTransferQueue(); public static void tArrayQueue() { new Thread() { @Override public void run() { while (true) { arrayQueue.offer(testData); } } }.start(); new Thread() { @Override public void run() { int count = 0; long num = 0; while (true) { try { Thread.sleep(100L); } catch (InterruptedException e) { } count++; num += arrayQueue.size(); arrayQueue.clear(); if (count == 50) { System.out.println(num / 50); count = 0; num = 0; } } } }.start(); } public static void tFixedQueue() { new Thread() { @Override public void run() { while (true) { fixedQueue.offer(testData); } } }.start(); new Thread() { @Override public void run() { int count = 0; long num = 0; while (true) { try { Thread.sleep(100L); } catch (InterruptedException e) { } count++; num += fixedQueue.size(); fixedQueue.clear(); if (count == 50) { System.out.println(num / 50); count = 0; num = 0; } } } }.start(); } public static void tQueue() { new Thread() { @Override public void run() { while (true) { testQueue.append(testData); } } }.start(); new Thread() { @Override public void run() { int count = 0; long num = 0; while (true) { try { Thread.sleep(100L); } catch (InterruptedException e) { } count++; num += testQueue.size(); testQueue.clear(); if (count == 50) { System.out.println(num / 50); count = 0; num = 0; } } } }.start(); } public static void tLinkedQueue() { new Thread() { @Override public void run() { while (true) { linkedQueue.offer(testData); } } }.start(); new Thread() { @Override public void run() { int count = 0; long num = 0; while (true) { try { Thread.sleep(100L); } catch (InterruptedException e) { } count++; num += linkedQueue.size(); linkedQueue.clear(); if (count == 50) { System.out.println(num / 50); count = 0; num = 0; } } } }.start(); } public static void tTransferQueue() { new Thread() { @Override public void run() { while (true) { transferQueue.offer(testData); } } }.start(); new Thread() { @Override public void run() { int count = 0; long num = 0; while (true) { try { Thread.sleep(100L); } catch (InterruptedException e) { } count++; num += transferQueue.size(); transferQueue.clear(); if (count == 50) { System.out.println(num / 50); count = 0; num = 0; } } } }.start(); } public static void main(String[] args) { // testArrayQueue(); // testFixedQueue(); // testQueue(); // testLinkedQueue(); // testTransferQueue(); } } ================================================ FILE: src/test/java/io/mycat/queue/QueueSimpleMain.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.queue; import jsr166y.LinkedTransferQueue; /** * @author mycat */ public class QueueSimpleMain { static long putCount = 0; static long takeCount = 0; public static void main(String[] args) { // final SynchronousQueue queue = new // SynchronousQueue(); // final ArrayBlockingQueue queue = new // ArrayBlockingQueue(10000000); final LinkedTransferQueue queue = new LinkedTransferQueue(); // final LinkedBlockingQueue queue = new // LinkedBlockingQueue(); new Thread() { @Override public void run() { for (;;) { long put = putCount; long take = takeCount; try { Thread.sleep(5000L); } catch (InterruptedException e) { e.printStackTrace(); } System.out.println("put:" + (putCount - put) / 5 + " take:" + (takeCount - take) / 5); } } }.start(); new Thread() { @Override public void run() { for (;;) { if (queue.offer("A")) { putCount++; } } } }.start(); new Thread() { @Override public void run() { for (;;) { // try { if (queue.poll() != null) { takeCount++; } // } catch (InterruptedException e) { // e.printStackTrace(); // } // try { // Thread.sleep(10L); // } catch (InterruptedException e) { // // e.printStackTrace(); // } } } }.start(); } } ================================================ FILE: src/test/java/io/mycat/route/.gitignore ================================================ /DruidMysqlSqlSubqueriesParserTest.java ================================================ FILE: src/test/java/io/mycat/route/DDLRouteTest.java ================================================ package io.mycat.route; import java.util.ArrayList; import java.util.List; import java.util.Map; import org.junit.Test; import io.mycat.MycatServer; import io.mycat.SimpleCachePool; import io.mycat.cache.CacheService; import io.mycat.cache.LayerCachePool; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.config.model.TableConfig; import io.mycat.route.RouteResultset; import io.mycat.route.RouteService; import io.mycat.route.RouteStrategy; import io.mycat.route.factory.RouteStrategyFactory; import io.mycat.route.util.RouterUtil; import io.mycat.server.parser.ServerParse; import junit.framework.Assert; public class DDLRouteTest { protected Map schemaMap; protected LayerCachePool cachePool = new SimpleCachePool(); protected RouteStrategy routeStrategy ; public DDLRouteTest() { String schemaFile = "/route/schema.xml"; String ruleFile = "/route/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schemaMap = schemaLoader.getSchemas(); MycatServer.getInstance().getConfig().getSchemas().putAll(schemaMap); MycatServer.getInstance().getConfig().getSystem().setUseGlobleTableCheck(0); //DDL Route Test 不开启全局表一致性检查 RouteStrategyFactory.init(); routeStrategy = RouteStrategyFactory.getRouteStrategy("druidparser"); } @Test public void testSpecialCharDDL() throws Exception { SchemaConfig schema = schemaMap.get("TESTDB"); CacheService cacheService = new CacheService(); RouteService routerService = new RouteService(cacheService); // alter table test String sql = " ALTER TABLE COMPANY\r\nADD COLUMN TEST VARCHAR(255) NULL AFTER CREATE_DATE,\r\n CHARACTER SET = UTF8"; sql = RouterUtil.getFixedSql(sql); List dataNodes = new ArrayList<>(); String tablename = RouterUtil.getTableName(sql, RouterUtil.getAlterTablePos(sql, 0)); Map tables = schema.getTables(); TableConfig tc; if (tables != null && (tc = tables.get(tablename)) != null) { dataNodes = tc.getDataNodes(); } int nodeSize = dataNodes.size(); int rs = ServerParse.parse(sql); int sqlType = rs & 0xff; RouteResultset rrs = routerService.route(new SystemConfig(), schema, sqlType, sql, "UTF-8", null); Assert.assertTrue("COMPANY".equals(tablename)); Assert.assertTrue(rrs.getNodes().length == nodeSize); } /** * ddl deal test * * @throws Exception */ @Test public void testDDL() throws Exception { SchemaConfig schema = schemaMap.get("TESTDB"); CacheService cacheService = new CacheService(); RouteService routerService = new RouteService(cacheService); // create table/view/function/.. String sql = " create table company(idd int)"; sql = RouterUtil.getFixedSql(sql); String upsql = sql.toUpperCase(); //TODO : modify by zhuam // 小写表名，需要额外转为 大写 做比较 String tablename = RouterUtil.getTableName(sql, RouterUtil.getCreateTablePos(upsql, 0)); tablename = tablename.toUpperCase(); List dataNodes = new ArrayList<>(); Map tables = schema.getTables(); TableConfig tc; if (tables != null && (tc = tables.get(tablename)) != null) { dataNodes = tc.getDataNodes(); } int nodeSize = dataNodes.size(); int rs = ServerParse.parse(sql); int sqlType = rs & 0xff; RouteResultset rrs = routerService.route(new SystemConfig(), schema, sqlType, sql, "UTF-8", null); Assert.assertTrue("COMPANY".equals(tablename)); Assert.assertTrue(rrs.getNodes().length == nodeSize); // drop table test sql = " drop table COMPANY"; sql = RouterUtil.getFixedSql(sql); upsql = sql.toUpperCase(); tablename = RouterUtil.getTableName(sql, RouterUtil.getDropTablePos(upsql, 0)); tables = schema.getTables(); if (tables != null && (tc = tables.get(tablename)) != null) { dataNodes = tc.getDataNodes(); } nodeSize = dataNodes.size(); rs = ServerParse.parse(sql); sqlType = rs & 0xff; rrs = routerService.route(new SystemConfig(), schema, sqlType, sql, "UTF-8", null); Assert.assertTrue("COMPANY".equals(tablename)); Assert.assertTrue(rrs.getNodes().length == nodeSize); //alter table sql = " alter table COMPANY add COLUMN name int ;"; sql = RouterUtil.getFixedSql(sql); upsql = sql.toUpperCase(); tablename = RouterUtil.getTableName(sql, RouterUtil.getAlterTablePos(upsql, 0)); tables = schema.getTables(); if (tables != null && (tc = tables.get(tablename)) != null) { dataNodes = tc.getDataNodes(); } nodeSize = dataNodes.size(); rs = ServerParse.parse(sql); sqlType = rs & 0xff; rrs = routerService.route(new SystemConfig(), schema, sqlType, sql, "UTF-8", null); Assert.assertTrue("COMPANY".equals(tablename)); Assert.assertTrue(rrs.getNodes().length == nodeSize); //truncate table; sql = " truncate table COMPANY"; sql = RouterUtil.getFixedSql(sql); upsql = sql.toUpperCase(); tablename = RouterUtil.getTableName(sql, RouterUtil.getTruncateTablePos(upsql, 0)); tables = schema.getTables(); if (tables != null && (tc = tables.get(tablename)) != null) { dataNodes = tc.getDataNodes(); } nodeSize = dataNodes.size(); rs = ServerParse.parse(sql); sqlType = rs & 0xff; rrs = routerService.route(new SystemConfig(), schema, sqlType, sql, "UTF-8", null); Assert.assertTrue("COMPANY".equals(tablename)); Assert.assertTrue(rrs.getNodes().length == nodeSize); } @Test public void testDDLDefaultNode() throws Exception { SchemaConfig schema = schemaMap.get("solo1"); CacheService cacheService = new CacheService(); RouteService routerService = new RouteService(cacheService); // create table/view/function/.. String sql = " create table company(idd int)"; sql = RouterUtil.getFixedSql(sql); String upsql = sql.toUpperCase(); //TODO：modify by zhuam 小写表名，转为大写比较 String tablename = RouterUtil.getTableName(sql, RouterUtil.getCreateTablePos(upsql, 0)); tablename = tablename.toUpperCase(); List dataNodes = new ArrayList<>(); Map tables = schema.getTables(); TableConfig tc; if (tables != null && (tc = tables.get(tablename)) != null) { dataNodes = tc.getDataNodes(); } int nodeSize = dataNodes.size(); if (nodeSize==0&& schema.getDataNode()!=null){ nodeSize = 1; } int rs = ServerParse.parse(sql); int sqlType = rs & 0xff; RouteResultset rrs = routerService.route(new SystemConfig(), schema, sqlType, sql, "UTF-8", null); Assert.assertTrue("COMPANY".equals(tablename)); Assert.assertTrue(rrs.getNodes().length == nodeSize); // drop table test sql = " drop table COMPANY"; sql = RouterUtil.getFixedSql(sql); upsql = sql.toUpperCase(); tablename = RouterUtil.getTableName(sql, RouterUtil.getDropTablePos(upsql, 0)); tables = schema.getTables(); if (tables != null && (tc = tables.get(tablename)) != null) { dataNodes = tc.getDataNodes(); } nodeSize = dataNodes.size(); if (nodeSize==0&& schema.getDataNode()!=null){ nodeSize = 1; } rs = ServerParse.parse(sql); sqlType = rs & 0xff; rrs = routerService.route(new SystemConfig(), schema, sqlType, sql, "UTF-8", null); Assert.assertTrue("COMPANY".equals(tablename)); Assert.assertTrue(rrs.getNodes().length == nodeSize); // drop table test sql = " drop table if exists COMPANY"; sql = RouterUtil.getFixedSql(sql); upsql = sql.toUpperCase(); tablename = RouterUtil.getTableName(sql, RouterUtil.getDropTablePos(upsql, 0)); tables = schema.getTables(); if (tables != null && (tc = tables.get(tablename)) != null) { dataNodes = tc.getDataNodes(); } nodeSize = dataNodes.size(); if (nodeSize==0&& schema.getDataNode()!=null){ nodeSize = 1; } rs = ServerParse.parse(sql); sqlType = rs & 0xff; rrs = routerService.route(new SystemConfig(), schema, sqlType, sql, "UTF-8", null); Assert.assertTrue("COMPANY".equals(tablename)); Assert.assertTrue(rrs.getNodes().length == nodeSize); //alter table sql = " alter table COMPANY add COLUMN name int ;"; sql = RouterUtil.getFixedSql(sql); upsql = sql.toUpperCase(); tablename = RouterUtil.getTableName(sql, RouterUtil.getAlterTablePos(upsql, 0)); tables = schema.getTables(); if (tables != null && (tc = tables.get(tablename)) != null) { dataNodes = tc.getDataNodes(); } nodeSize = dataNodes.size(); if (nodeSize==0&& schema.getDataNode()!=null){ nodeSize = 1; } rs = ServerParse.parse(sql); sqlType = rs & 0xff; rrs = routerService.route(new SystemConfig(), schema, sqlType, sql, "UTF-8", null); Assert.assertTrue("COMPANY".equals(tablename)); Assert.assertTrue(rrs.getNodes().length == nodeSize); //truncate table; sql = " truncate table COMPANY"; sql = RouterUtil.getFixedSql(sql); upsql = sql.toUpperCase(); tablename = RouterUtil.getTableName(sql, RouterUtil.getTruncateTablePos(upsql, 0)); tables = schema.getTables(); if (tables != null && (tc = tables.get(tablename)) != null) { dataNodes = tc.getDataNodes(); } nodeSize = dataNodes.size(); if (nodeSize==0&& schema.getDataNode()!=null){ nodeSize = 1; } rs = ServerParse.parse(sql); sqlType = rs & 0xff; rrs = routerService.route(new SystemConfig(), schema, sqlType, sql, "UTF-8", null); Assert.assertTrue("COMPANY".equals(tablename)); Assert.assertTrue(rrs.getNodes().length == nodeSize); } // @Test // public void testTableMetaRead() throws Exception { // final SchemaConfig schema = schemaMap.get("cndb"); // // String sql = "desc offer"; // RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.DESCRIBE, sql, null, null, // cachePool); // Assert.assertEquals(false, rrs.isCacheAble()); // Assert.assertEquals(-1L, rrs.getLimitSize()); // Assert.assertEquals(1, rrs.getNodes().length); // // random return one node // // Assert.assertEquals("offer_dn[0]", rrs.getNodes()[0].getName()); // Assert.assertEquals("desc offer", rrs.getNodes()[0].getStatement()); // // sql = " desc cndb.offer"; // rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.DESCRIBE, sql, null, null, cachePool); // Assert.assertEquals(false, rrs.isCacheAble()); // Assert.assertEquals(-1L, rrs.getLimitSize()); // Assert.assertEquals(1, rrs.getNodes().length); // // random return one node // // Assert.assertEquals("offer_dn[0]", rrs.getNodes()[0].getName()); // Assert.assertEquals("desc offer", rrs.getNodes()[0].getStatement()); // // sql = " desc cndb.offer col1"; // rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.DESCRIBE, sql, null, null, cachePool); // Assert.assertEquals(false, rrs.isCacheAble()); // Assert.assertEquals(-1L, rrs.getLimitSize()); // Assert.assertEquals(1, rrs.getNodes().length); // // random return one node // // Assert.assertEquals("offer_dn[0]", rrs.getNodes()[0].getName()); // Assert.assertEquals("desc offer col1", rrs.getNodes()[0].getStatement()); // // sql = "SHOW FULL COLUMNS FROM offer IN db_name WHERE true"; // rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SHOW, sql, null, null, // cachePool); // Assert.assertEquals(false, rrs.isCacheAble()); // Assert.assertEquals(-1L, rrs.getLimitSize()); // Assert.assertEquals(1, rrs.getNodes().length); // // random return one node // // Assert.assertEquals("offer_dn[0]", rrs.getNodes()[0].getName()); // Assert.assertEquals("SHOW FULL COLUMNS FROM offer WHERE true", // rrs.getNodes()[0].getStatement()); // // sql = "SHOW FULL COLUMNS FROM db.offer IN db_name WHERE true"; // rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SHOW, sql, null, null, // cachePool); // Assert.assertEquals(-1L, rrs.getLimitSize()); // Assert.assertEquals(false, rrs.isCacheAble()); // Assert.assertEquals(1, rrs.getNodes().length); // // random return one node // // Assert.assertEquals("offer_dn[0]", rrs.getNodes()[0].getName()); // Assert.assertEquals("SHOW FULL COLUMNS FROM offer WHERE true", // rrs.getNodes()[0].getStatement()); // // // sql = "SHOW FULL TABLES FROM `TESTDB` WHERE Table_type != 'VIEW'"; // rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SHOW, sql, null, null, // cachePool); // Assert.assertEquals(-1L, rrs.getLimitSize()); // Assert.assertEquals(false, rrs.isCacheAble()); // Assert.assertEquals("SHOW FULL TABLES WHERE Table_type != 'VIEW'", rrs.getNodes()[0].getStatement()); // // sql = "SHOW INDEX IN offer FROM db_name"; // rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SHOW, sql, null, null, // cachePool); // Assert.assertEquals(false, rrs.isCacheAble()); // Assert.assertEquals(-1L, rrs.getLimitSize()); // Assert.assertEquals(1, rrs.getNodes().length); // // random return one node // // Assert.assertEquals("offer_dn[0]", rrs.getNodes()[0].getName()); // Assert.assertEquals("SHOW INDEX FROM offer", // rrs.getNodes()[0].getStatement()); // } } ================================================ FILE: src/test/java/io/mycat/route/DQLRouteTest.java ================================================ package io.mycat.route; import java.lang.reflect.Method; import java.sql.SQLSyntaxErrorException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import org.junit.Test; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; import com.alibaba.druid.sql.parser.SQLStatementParser; import com.alibaba.druid.stat.TableStat.Condition; import io.mycat.MycatServer; import io.mycat.SimpleCachePool; import io.mycat.cache.LayerCachePool; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.SchemaConfig; import io.mycat.route.factory.RouteStrategyFactory; import io.mycat.route.parser.druid.DruidShardingParseInfo; import io.mycat.route.parser.druid.MycatSchemaStatVisitor; import io.mycat.route.parser.druid.MycatStatementParser; import io.mycat.route.parser.druid.RouteCalculateUnit; import junit.framework.Assert; public class DQLRouteTest { protected Map schemaMap; protected LayerCachePool cachePool = new SimpleCachePool(); protected RouteStrategy routeStrategy; private Map tableAliasMap = new HashMap(); protected DruidShardingParseInfo ctx; public DQLRouteTest() { String schemaFile = "/route/schema.xml"; String ruleFile = "/route/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schemaMap = schemaLoader.getSchemas(); MycatServer.getInstance().getConfig().getSchemas().putAll(schemaMap); RouteStrategyFactory.init(); routeStrategy = RouteStrategyFactory.getRouteStrategy("druidparser"); } @Test public void test() throws Exception { String stmt = "select * from `offer` where id = 100"; SchemaConfig schema = schemaMap.get("mysqldb"); RouteResultset rrs = new RouteResultset(stmt, 7); SQLStatementParser parser = null; if (schema.isNeedSupportMultiDBType()) { parser = new MycatStatementParser(stmt); } else { parser = new MySqlStatementParser(stmt); } SQLStatement statement; MycatSchemaStatVisitor visitor = null; try { statement = parser.parseStatement(); visitor = new MycatSchemaStatVisitor(); } catch (Exception t) { throw new SQLSyntaxErrorException(t); } ctx = new DruidShardingParseInfo(); ctx.setSql(stmt); List taskList = visitorParse(rrs, statement, visitor); Assert.assertEquals(true, !taskList.get(0).getTablesAndConditions().isEmpty()); } @SuppressWarnings("unchecked") private List visitorParse(RouteResultset rrs, SQLStatement stmt, MycatSchemaStatVisitor visitor) throws Exception { stmt.accept(visitor); List> mergedConditionList = new ArrayList>(); if (visitor.hasOrCondition()) {// 包含or语句 // TODO // 根据or拆分 mergedConditionList = visitor.splitConditions(); } else {// 不包含OR语句 mergedConditionList.add(visitor.getConditions()); } if (visitor.getAliasMap() != null) { for (Map.Entry entry : visitor.getAliasMap().entrySet()) { String key = entry.getKey(); String value = entry.getValue(); if (key != null && key.indexOf("`") >= 0) { key = key.replaceAll("`", ""); } if (value != null && value.indexOf("`") >= 0) { value = value.replaceAll("`", ""); } // 表名前面带database的，去掉 if (key != null) { int pos = key.indexOf("."); if (pos > 0) { key = key.substring(pos + 1); } } if (key.equals(value)) { ctx.addTable(key.toUpperCase()); } // else { // tableAliasMap.put(key, value); // } tableAliasMap.put(key.toUpperCase(), value); } visitor.getAliasMap().putAll(tableAliasMap); ctx.setTableAliasMap(tableAliasMap); } //利用反射机制单元测试DefaultDruidParser类的私有方法buildRouteCalculateUnits Class clazz = Class.forName("io.mycat.route.parser.druid.impl.DefaultDruidParser"); Method buildRouteCalculateUnits = clazz.getDeclaredMethod("buildRouteCalculateUnits", new Class[] { MycatSchemaStatVisitor.class, List.class }); //System.out.println("buildRouteCalculateUnits:\t" + buildRouteCalculateUnits); Object newInstance = clazz.newInstance(); buildRouteCalculateUnits.setAccessible(true); Object returnValue = buildRouteCalculateUnits.invoke(newInstance, new Object[] { visitor, mergedConditionList }); List retList = new ArrayList(); if (returnValue instanceof ArrayList) { retList.add(((ArrayList)returnValue).get(0)); //retList = (ArrayList)returnValue; //System.out.println(taskList.get(0).getTablesAndConditions().values()); } return retList; } } ================================================ FILE: src/test/java/io/mycat/route/DeleteSqlParseTest.java ================================================ package io.mycat.route; import java.sql.SQLNonTransientException; import java.util.Map; import org.junit.Test; import io.mycat.MycatServer; import io.mycat.SimpleCachePool; import io.mycat.cache.LayerCachePool; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.route.factory.RouteStrategyFactory; import junit.framework.Assert; /** * 测试删除 * * @author huangyiming * */ public class DeleteSqlParseTest { protected Map schemaMap; protected LayerCachePool cachePool = new SimpleCachePool(); protected RouteStrategy routeStrategy; public DeleteSqlParseTest() { String schemaFile = "/route/schema.xml"; String ruleFile = "/route/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schemaMap = schemaLoader.getSchemas(); MycatServer.getInstance().getConfig().getSchemas().putAll(schemaMap); RouteStrategyFactory.init(); routeStrategy = RouteStrategyFactory.getRouteStrategy("druidparser"); } @Test public void testDeleteToRoute() throws SQLNonTransientException { String sql = "delete t from offer as t "; SchemaConfig schema = schemaMap.get("config"); RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(128, rrs.getNodes().length); } } ================================================ FILE: src/test/java/io/mycat/route/DruidDb2SqlParserTest.java ================================================ package io.mycat.route; import java.sql.SQLNonTransientException; import java.util.Map; import org.junit.Test; import io.mycat.MycatServer; import io.mycat.SimpleCachePool; import io.mycat.cache.LayerCachePool; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.route.factory.RouteStrategyFactory; import junit.framework.Assert; public class DruidDb2SqlParserTest { protected Map schemaMap; protected LayerCachePool cachePool = new SimpleCachePool(); protected RouteStrategy routeStrategy; public DruidDb2SqlParserTest() { String schemaFile = "/route/schema.xml"; String ruleFile = "/route/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schemaMap = schemaLoader.getSchemas(); MycatServer.getInstance().getConfig().getSchemas().putAll(schemaMap); RouteStrategyFactory.init(); routeStrategy = RouteStrategyFactory.getRouteStrategy("druidparser"); } @Test public void testLimitToDb2Page() throws SQLNonTransientException { String sql = "select * from offer order by id desc limit 5,10"; SchemaConfig schema = schemaMap.get("db2db"); RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(15, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals("db2_1", rrs.getNodes()[0].getName()); Assert.assertEquals("db2_2", rrs.getNodes()[1].getName()); sql= rrs.getNodes()[0].getStatement() ; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(15, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals(0, rrs.getLimitStart()); Assert.assertEquals(15, rrs.getLimitSize()); sql="select * from offer1 order by id desc limit 5,10" ; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(5, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(10, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals("db2_1", rrs.getNodes()[0].getName()); } @Test public void testDb2PageSQL() throws SQLNonTransientException { String sql = "SELECT *\n" + "FROM (SELECT sid, ROW_NUMBER() OVER (ORDER BY sid DESC) AS ROWNUM\n" + "\tFROM offer \n" + "\tWHERE sts <> 'N'\n" + "\t\t\t) XX\n" + "WHERE ROWNUM > 5\n" + "\tAND ROWNUM <= 15\n"; SchemaConfig schema = schemaMap.get("db2db"); RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(15, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals("db2_1", rrs.getNodes()[0].getName()); Assert.assertEquals("db2_2", rrs.getNodes()[1].getName()); sql = "SELECT *\n" + "FROM (SELECT sid, ROW_NUMBER() OVER (ORDER BY sid DESC) AS ROWNUM\n" + "\tFROM offer1 \n" + "\tWHERE sts <> 'N'\n" + "\t\t\t) XX\n" + "WHERE ROWNUM > 5\n" + "\tAND ROWNUM <= 15\n"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(5, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(10, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals(sql, rrs.getNodes()[0].getStatement()); Assert.assertEquals("db2_1", rrs.getNodes()[0].getName()); sql = "SELECT sid\n" + "FROM offer \n" + "ORDER BY sid desc\n" + "FETCH FIRST 10 ROWS ONLY"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); Assert.assertEquals(0, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(10, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals(sql, rrs.getNodes()[0].getStatement()); Assert.assertEquals("db2_1", rrs.getNodes()[0].getName()); Assert.assertEquals("db2_2", rrs.getNodes()[1].getName()); } } ================================================ FILE: src/test/java/io/mycat/route/DruidMysqlCreateTableTest.java ================================================ package io.mycat.route; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLName; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition; import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement; import com.alibaba.druid.sql.ast.statement.SQLTableElement; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlInsertStatement; import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; import io.mycat.MycatServer; import io.mycat.SimpleCachePool; import io.mycat.cache.LayerCachePool; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.route.factory.RouteStrategyFactory; import io.mycat.server.interceptor.impl.GlobalTableUtil; import io.mycat.util.StringUtil; import junit.framework.Assert; import org.junit.Test; import java.sql.SQLNonTransientException; import java.util.List; import java.util.Map; public class DruidMysqlCreateTableTest { protected Map schemaMap; protected LayerCachePool cachePool = new SimpleCachePool(); protected RouteStrategy routeStrategy; private static final String originSql1 = "CREATE TABLE autoslot" + "(" + " ID BIGINT AUTO_INCREMENT," + " CHANNEL_ID INT(11)," + " CHANNEL_INFO varchar(128)," + " CONSTRAINT RETL_MARK_ID PRIMARY KEY (ID)" + ") ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;"; public DruidMysqlCreateTableTest() { String schemaFile = "/route/schema.xml"; String ruleFile = "/route/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schemaMap = schemaLoader.getSchemas(); MycatServer.getInstance().getConfig().getSchemas().putAll(schemaMap); RouteStrategyFactory.init(); routeStrategy = RouteStrategyFactory.getRouteStrategy("druidparser"); } @Test public void testCreate() throws SQLNonTransientException { SchemaConfig schema = schemaMap.get("mysqldb"); RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, -1, originSql1, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); String sql= rrs.getNodes()[0].getStatement(); Assert.assertTrue(parseSql(sql)); } // @Test public void testInsert() throws SQLNonTransientException { SchemaConfig schema = schemaMap.get("mysqldb"); RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, -1, "insert into autoslot (id,sid) values(1,2) ", null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertTrue(isInsertHasSlot(rrs.getStatement())); } private boolean isInsertHasSlot(String sql) { MySqlStatementParser parser = new MySqlStatementParser(sql); MySqlInsertStatement insertStatement= (MySqlInsertStatement)parser.parseStatement(); List cc= insertStatement.getColumns(); for (SQLExpr sqlExpr : cc) { SQLIdentifierExpr c= (SQLIdentifierExpr) sqlExpr; if("_slot".equalsIgnoreCase(c.getName()) &&cc.size()==insertStatement.getValues().getValues().size()) return true; } return false; } public boolean parseSql(String sql) { MySqlStatementParser parser = new MySqlStatementParser(sql); SQLStatement statement = parser.parseStatement(); return hasColumn(statement); } private static boolean hasColumn(SQLStatement statement){ for (SQLTableElement tableElement : ((SQLCreateTableStatement)statement).getTableElementList()) { SQLName sqlName = null; if (tableElement instanceof SQLColumnDefinition) { sqlName = ((SQLColumnDefinition)tableElement).getName(); } if (sqlName != null) { String simpleName = sqlName.getSimpleName(); simpleName = StringUtil.removeBackquote(simpleName); if (tableElement instanceof SQLColumnDefinition && "_slot".equalsIgnoreCase(simpleName)) { return true; } } } return false; } } ================================================ FILE: src/test/java/io/mycat/route/DruidMysqlHavingTest.java ================================================ package io.mycat.route; import java.sql.SQLNonTransientException; import java.util.Map; import org.junit.Test; import io.mycat.MycatServer; import io.mycat.SimpleCachePool; import io.mycat.cache.LayerCachePool; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.route.factory.RouteStrategyFactory; import junit.framework.Assert; public class DruidMysqlHavingTest { protected Map schemaMap; protected LayerCachePool cachePool = new SimpleCachePool(); protected RouteStrategy routeStrategy; public DruidMysqlHavingTest() { String schemaFile = "/route/schema.xml"; String ruleFile = "/route/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schemaMap = schemaLoader.getSchemas(); MycatServer.getInstance().getConfig().getSchemas().putAll(schemaMap); RouteStrategyFactory.init(); routeStrategy = RouteStrategyFactory.getRouteStrategy("druidparser"); } @Test public void testHaving() throws SQLNonTransientException { String sql = "select avg(offer_id) avgofferid, member_id from offer_detail group by member_id having avgofferid > 100"; SchemaConfig schema = schemaMap.get("cndb"); RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(3, rrs.getSqlMerge().getHavingColsName().length); sql = "select avg(offer_id) avgofferid, member_id from offer_detail group by member_id having avg(offer_id) > 100"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(3, rrs.getSqlMerge().getHavingColsName().length); sql = "select count(offer_id) countofferid, member_id from offer_detail group by member_id having countofferid > 100"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(3, rrs.getSqlMerge().getHavingColsName().length); sql = "select count(offer_id) countofferid, member_id from offer_detail group by member_id having count(offer_id) > 100"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(3, rrs.getSqlMerge().getHavingColsName().length); } } ================================================ FILE: src/test/java/io/mycat/route/DruidMysqlRouteStrategyTest.java ================================================ package io.mycat.route; import java.sql.SQLNonTransientException; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; import org.junit.Test; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; import io.mycat.MycatServer; import io.mycat.SimpleCachePool; import io.mycat.cache.LayerCachePool; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.route.factory.RouteStrategyFactory; import io.mycat.server.parser.ServerParse; import junit.framework.Assert; import junit.framework.TestCase; public class DruidMysqlRouteStrategyTest extends TestCase { protected Map schemaMap; protected LayerCachePool cachePool = new SimpleCachePool(); protected RouteStrategy routeStrategy; public DruidMysqlRouteStrategyTest() { String schemaFile = "/route/schema.xml"; String ruleFile = "/route/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schemaMap = schemaLoader.getSchemas(); MycatServer.getInstance().getConfig().getSchemas().putAll(schemaMap); RouteStrategyFactory.init(); routeStrategy = RouteStrategyFactory.getRouteStrategy("druidparser"); } protected void setUp() throws Exception { // super.setUp(); // schemaMap = CobarServer.getInstance().getConfig().getSchemas(); } // public void testAlias() throws Exception { // String sql = "SELECT UM.UserId , UM.MenuId ,SM.ParentId ,SM.FullName , SM.Description , SM.Img , SM.NavigateUrl ,SM.FormName ,SM.Target ,SM.IsUnfold FROM Lever_SysMenu SM INNER JOIN ( SELECT UR.UserId AS UserId , RM.MenuId AS MenuId FROM Lever_RoleMenu RM INNER JOIN Lever_UserRole UR ON RM.RoleId = UR.RoleId UNION SELECT UserId , MenuId FROM Lever_UserMenu UNION SELECT U.UserId , RM.MenuId FROM Lever_User U LEFT JOIN Lever_RoleMenu RM ON U.RoleId = RM.RoleId WHERE U.UserId = '8d28533f-1762-4e79-b71f-64eb1a50cb8b' ) UM ON SM.MenuId = UM.MenuId WHERE UM.UserId = '8d28533f-1762-4e79-b71f-64eb1a50cb8b' AND SM.Enabled = 1 ORDER BY SM.SortCode"; // SchemaConfig schema = schemaMap.get("wdw"); // RouteResultset rrs = routeStrategy.route(new SystemConfig(),schema, -1, sql, null, // null, cachePool); // } public void testRouteInsertShort() throws Exception { String sql = "inSErt into offer_detail (`offer_id`, gmt) values ('123',now())"; SchemaConfig schema = schemaMap.get("cndb"); RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(false, rrs.isCacheAble()); Assert.assertEquals(-1l, rrs.getLimitSize()); Assert.assertEquals("detail_dn15", rrs.getNodes()[0].getName()); String expect = "INSERT INTO offer_detail (`offer_id`, gmt)\n" + "VALUES ('123', now())"; Assert.assertEquals(expect, rrs.getNodes()[0].getStatement()); sql = "inSErt into offer_detail ( gmt) values (now())"; schema = schemaMap.get("cndb"); try { rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); } catch (Exception e) { String msg = "bad insert sql (sharding column:"; Assert.assertTrue(e.getMessage().contains(msg)); } sql = "inSErt into offer_detail (offer_id, gmt) values (123,now())"; schema = schemaMap.get("cndb"); rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(false, rrs.isCacheAble()); Assert.assertEquals(-1l, rrs.getLimitSize()); Assert.assertEquals("detail_dn15", rrs.getNodes()[0].getName()); Assert.assertEquals( "INSERT INTO offer_detail (offer_id, gmt)\nVALUES ('123', now())", rrs.getNodes()[0].getStatement()); sql = "insert into offer(group_id,offer_id,member_id)values(234,123,'abc')"; schema = schemaMap.get("cndb"); rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(false, rrs.isCacheAble()); Assert.assertEquals(-1l, rrs.getLimitSize()); Assert.assertEquals("offer_dn12", rrs.getNodes()[0].getName()); Assert.assertEquals( "INSERT INTO offer (group_id, offer_id, member_id)\n" + "VALUES (234, 123, 'abc')", rrs.getNodes()[0].getStatement()); sql = "\n" + " INSERT INTO \n" + "`offer` \n" + "(`asf`,member_id) \n" + "VALUES \n" + "(' the articles sfroms user selection ','abc')"; schema = schemaMap.get("cndb"); rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); } public void testGlobalTableroute() throws Exception { String sql = null; SchemaConfig schema = schemaMap.get("TESTDB"); RouteResultset rrs = null; // select of global table route to only one datanode defined sql = "select * from company where company.name like 'aaa'"; schema = schemaMap.get("TESTDB"); rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(false, rrs.isCacheAble()); // query of global table only route to one datanode sql = "insert into company (id,name,level) values(111,'company1',3)"; schema = schemaMap.get("TESTDB"); rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(3, rrs.getNodes().length); Assert.assertEquals(false, rrs.isCacheAble()); // update of global table route to every datanode defined sql = "update company set name=name+aaa"; schema = schemaMap.get("TESTDB"); rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(3, rrs.getNodes().length); Assert.assertEquals(false, rrs.isCacheAble()); // delete of global table route to every datanode defined sql = "delete from company where id = 1"; schema = schemaMap.get("TESTDB"); rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(3, rrs.getNodes().length); Assert.assertEquals(false, rrs.isCacheAble()); // company is global table ,will route to differnt tables schema = schemaMap.get("TESTDB"); sql = "select * from company A where a.sharding_id=10001 union select * from company B where B.sharding_id =10010"; Set nodeSet = new HashSet(); for (int i = 0; i < 10; i++) { rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(false, rrs.isCacheAble()); Assert.assertEquals(1, rrs.getNodes().length); nodeSet.add(rrs.getNodes()[0].getName()); } Assert.assertEquals(true, nodeSet.size() > 1); } public void testMoreGlobalTableroute() throws Exception { String sql = null; SchemaConfig schema = schemaMap.get("TESTDB"); RouteResultset rrs = null; // select of global table route to only one datanode defined sql = "select * from company,area where area.company_id=company.id "; schema = schemaMap.get("TESTDB"); rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(false, rrs.isCacheAble()); // 全局表涉及到多个节点时,不缓存路由结果 } public void testRouteMultiTables() throws Exception { // company is global table ,route to 3 datanode and ignored in route String sql = "select * from company,customer ,orders where customer.company_id=company.id and orders.customer_id=customer.id and company.name like 'aaa' limit 10"; SchemaConfig schema = schemaMap.get("TESTDB"); RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); Assert.assertEquals(true, rrs.isCacheAble()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals("dn1", rrs.getNodes()[0].getName()); Assert.assertEquals("dn2", rrs.getNodes()[1].getName()); } public void testRouteCache() throws Exception { // select cache ID this.cachePool.putIfAbsent("TESTDB_EMPLOYEE", "88", "dn2"); SchemaConfig schema = schemaMap.get("TESTDB"); String sql = "select * from employee where id=88"; RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(false, rrs.isCacheAble());//已经缓存了,不必再缓存了 Assert.assertEquals(null, rrs.getPrimaryKey()); Assert.assertEquals(-1, rrs.getLimitSize()); Assert.assertEquals("dn2", rrs.getNodes()[0].getName()); // select cache ID not found ,return all node and rrst not cached sql = "select * from employee where id=89"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); Assert.assertEquals(false, rrs.isCacheAble()); Assert.assertEquals("TESTDB_EMPLOYEE.ID", rrs.getPrimaryKey()); Assert.assertEquals(-1, rrs.getLimitSize()); // update cache ID found sql = "update employee set name='aaa' where id=88"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(false, rrs.isCacheAble()); Assert.assertEquals(null, rrs.getPrimaryKey()); Assert.assertEquals("dn2", rrs.getNodes()[0].getName()); // delete cache ID should be not founded sql = "delete from employee where id=88"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(false, rrs.isCacheAble()); Assert.assertEquals("dn2", rrs.getNodes()[0].getName()); // Assert.assertEquals("dn2", rrs.getNodes()[1].getName()); } private static Map getNodeMap( RouteResultset rrs, int expectSize) { RouteResultsetNode[] routeNodes = rrs.getNodes(); Assert.assertEquals(expectSize, routeNodes.length); Map nodeMap = new HashMap( expectSize, 1); for (int i = 0; i < expectSize; i++) { RouteResultsetNode routeNode = routeNodes[i]; nodeMap.put(routeNode.getName(), routeNode); } Assert.assertEquals(expectSize, nodeMap.size()); return nodeMap; } private static interface NodeNameDeconstructor { public int getNodeIndex(String name); } private static class NodeNameAsserter implements NodeNameDeconstructor { private String[] expectNames; public NodeNameAsserter() { } public NodeNameAsserter(String... expectNames) { Assert.assertNotNull(expectNames); this.expectNames = expectNames; } protected void setNames(String[] expectNames) { Assert.assertNotNull(expectNames); this.expectNames = expectNames; } public void assertRouteNodeNames(Collection nodeNames) { Assert.assertNotNull(nodeNames); Assert.assertEquals(expectNames.length, nodeNames.size()); for (String name : expectNames) { Assert.assertTrue(nodeNames.contains(name)); } } @Override public int getNodeIndex(String name) { for (int i = 0; i < expectNames.length; ++i) { if (name.equals(expectNames[i])) { return i; } } throw new NoSuchElementException("route node " + name + " dosn't exist!"); } } private static class IndexedNodeNameAsserter extends NodeNameAsserter { /** * @param from included * @param to excluded */ public IndexedNodeNameAsserter(String prefix, int from, int to) { super(); String[] names = new String[to - from]; for (int i = 0; i < names.length; ++i) { names[i] = prefix + (i + from) ; } setNames(names); } } private static class RouteNodeAsserter { private NodeNameDeconstructor deconstructor; private SQLAsserter sqlAsserter; public RouteNodeAsserter(NodeNameDeconstructor deconstructor, SQLAsserter sqlAsserter) { this.deconstructor = deconstructor; this.sqlAsserter = sqlAsserter; } public void assertNode(RouteResultsetNode node) throws Exception { int nodeIndex = deconstructor.getNodeIndex(node.getName()); sqlAsserter.assertSQL(node.getStatement(), nodeIndex); } } private static interface SQLAsserter { public void assertSQL(String sql, int nodeIndex) throws Exception; } private static class SimpleSQLAsserter implements SQLAsserter { private Map> map = new HashMap>(); public SimpleSQLAsserter addExpectSQL(int nodeIndex, String sql) { Set set = map.get(nodeIndex); if (set == null) { set = new HashSet(); map.put(nodeIndex, set); } set.add(sql); return this; } @Override public void assertSQL(String sql, int nodeIndex) throws Exception { Assert.assertNotNull(map.get(nodeIndex)); Assert.assertTrue(map.get(nodeIndex).contains(sql)); } } public void testroute() throws Exception { SchemaConfig schema = schemaMap.get("cndb"); String sql = "select * from independent where member='abc'"; RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(true, rrs.isCacheAble()); Map nodeMap = getNodeMap(rrs, 128); IndexedNodeNameAsserter nameAsserter = new IndexedNodeNameAsserter( "independent_dn", 0, 128); nameAsserter.assertRouteNodeNames(nodeMap.keySet()); SimpleSQLAsserter sqlAsserter = new SimpleSQLAsserter(); for (int i = 0; i < 128; ++i) { sqlAsserter.addExpectSQL(i, "select * from independent where member='abc'"); } RouteNodeAsserter asserter = new RouteNodeAsserter(nameAsserter, sqlAsserter); for (RouteResultsetNode node : nodeMap.values()) { asserter.assertNode(node); } // include database schema ,should remove sql = "select * from cndb.independent A where a.member='abc'"; schema = schemaMap.get("cndb"); rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(true, rrs.isCacheAble()); nodeMap = getNodeMap(rrs, 128); nameAsserter = new IndexedNodeNameAsserter("independent_dn", 0, 128); nameAsserter.assertRouteNodeNames(nodeMap.keySet()); sqlAsserter = new SimpleSQLAsserter(); for (int i = 0; i < 128; ++i) { sqlAsserter.addExpectSQL(i, "select * from independent A where a.member='abc'"); } asserter = new RouteNodeAsserter(nameAsserter, sqlAsserter); for (RouteResultsetNode node : nodeMap.values()) { asserter.assertNode(node); } } public void testERroute() throws Exception { SchemaConfig schema = schemaMap.get("TESTDB"); String sql = "insert into orders (id,name,customer_id) values(1,'testonly',1)"; RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(false, rrs.isCacheAble()); Assert.assertEquals("dn1", rrs.getNodes()[0].getName()); sql = "insert into orders (id,name,customer_id) values(1,'testonly',2000001)"; rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(false, rrs.isCacheAble()); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals("dn2", rrs.getNodes()[0].getName()); // can't update join key sql = "update orders set id=1 ,name='aaa' , customer_id=2000001"; String err = null; try { rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); } catch (SQLNonTransientException e) { err = e.getMessage(); } Assert.assertEquals( true, err.startsWith("Parent relevant column can't be updated ORDERS->CUSTOMER_ID")); // route by parent rule ,update sql sql = "update orders set id=1 ,name='aaa' where customer_id=2000001"; rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(true, rrs.isCacheAble()); Assert.assertEquals("dn2", rrs.getNodes()[0].getName()); // route by parent rule but can't find datanode sql = "update orders set id=1 ,name='aaa' where customer_id=-1"; try { rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); } catch (Exception e) { err = e.getMessage(); } Assert.assertEquals(true, err.startsWith("can't find datanode for sharding column:")); // route by parent rule ,select sql sql = "select * from orders where customer_id=2000001"; rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(true, rrs.isCacheAble()); Assert.assertEquals("dn2", rrs.getNodes()[0].getName()); // route by parent rule ,delete sql sql = "delete from orders where customer_id=2000001"; rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals("dn2", rrs.getNodes()[0].getName()); //test alias in column sql = "select name as order_name from orders order by order_name limit 10,5"; rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); MySqlStatementParser parser = new MySqlStatementParser("SELECT name AS order_name FROM orders ORDER BY order_name LIMIT 0,15"); SQLStatement statement = parser.parseStatement(); // Assert.assertEquals(sql, rrs.getNodes()[0].getStatement()); } public void testDuplicatePartitionKey() throws Exception { String sql = null; SchemaConfig schema = schemaMap.get("cndb"); RouteResultset rrs = null; sql = "select * from cndb.offer where (offer_id, group_id ) In (123,234)"; schema = schemaMap.get("cndb"); rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(true, rrs.isCacheAble()); Assert.assertEquals(-1l, rrs.getLimitSize()); Assert.assertEquals(128, rrs.getNodes().length); for (int i = 0; i < 128; i++) { // Assert.assertEquals("offer_dn" + i , // rrs.getNodes()[i].getName());//node的排序有变化，所以此处不强求 Assert.assertEquals( "select * from offer where (offer_id, group_id ) In (123,234)", rrs.getNodes()[i].getStatement()); } sql = "SELECT * FROM offer WHERE FALSE OR offer_id = 123 AND member_id = 123 OR member_id = 123 AND member_id = 234 OR member_id = 123 AND member_id = 345 OR member_id = 123 AND member_id = 456 OR offer_id = 234 AND group_id = 123 OR offer_id = 234 AND group_id = 234 OR offer_id = 234 AND group_id = 345 OR offer_id = 234 AND group_id = 456 OR offer_id = 345 AND group_id = 123 OR offer_id = 345 AND group_id = 234 OR offer_id = 345 AND group_id = 345 OR offer_id = 345 AND group_id = 456 OR offer_id = 456 AND group_id = 123 OR offer_id = 456 AND group_id = 234 OR offer_id = 456 AND group_id = 345 OR offer_id = 456 AND group_id = 456"; schema = schemaMap.get("cndb"); rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(true, rrs.isCacheAble()); getNodeMap(rrs, 128); sql = "select * from offer where false" + " or offer_id=123 and group_id=123" + " or group_id=123 and offer_id=234" + " or offer_id=123 and group_id=345" + " or offer_id=123 and group_id=456 "; schema = schemaMap.get("cndb"); rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(true, rrs.isCacheAble()); Assert.assertEquals(-1l, rrs.getLimitSize()); } public void testAddLimitToSQL() throws Exception { final SchemaConfig schema = schemaMap.get("TESTDB"); String sql = null; RouteResultset rrs = null; sql = "select * from orders"; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertEquals(true, rrs.isCacheAble()); Map nodeMap = getNodeMap(rrs, 2); NodeNameAsserter nameAsserter = new NodeNameAsserter("dn2", "dn1"); nameAsserter.assertRouteNodeNames(nodeMap.keySet()); Assert.assertEquals(schema.getDefaultMaxLimit(), rrs.getLimitSize()); // Assert.assertEquals("SELECT * FROM orders LIMIT 100", rrs.getNodes()[0].getStatement()); MySqlStatementParser parser = new MySqlStatementParser("SELECT * FROM orders LIMIT 100"); SQLStatement statement = parser.parseStatement(); Assert.assertEquals(statement.toString(), rrs.getNodes()[0].getStatement()); sql = "select * from goods"; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertEquals(false, rrs.isCacheAble()); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(schema.getDefaultMaxLimit(), rrs.getLimitSize()); // Assert.assertEquals("select * from goods", rrs.getNodes()[0].getStatement()); parser = new MySqlStatementParser("SELECT * FROM goods LIMIT 100"); statement = parser.parseStatement(); Assert.assertEquals(statement.toString(), rrs.getNodes()[0].getStatement()); sql = "select * from goods limit 2 ,3"; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertEquals(false, rrs.isCacheAble()); Assert.assertEquals(1, rrs.getNodes().length); // Assert.assertEquals(-1, rrs.getLimitSize()); Assert.assertEquals("select * from goods limit 2 ,3", rrs.getNodes()[0].getStatement()); sql = "select * from notpartionTable limit 2 ,3"; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertEquals(true, rrs.isCacheAble()); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(3, rrs.getLimitSize()); Assert.assertEquals("select * from notpartionTable limit 2 ,3", rrs.getNodes()[0].getStatement()); } public void testModifySQLLimit() throws Exception { final SchemaConfig schema = schemaMap.get("TESTDB"); String sql = null; RouteResultset rrs = null; //SQL span multi datanode sql = "select * from orders limit 2,3"; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertEquals(true, rrs.isCacheAble()); Map nodeMap = getNodeMap(rrs, 2); NodeNameAsserter nameAsserter = new NodeNameAsserter("dn2", "dn1"); nameAsserter.assertRouteNodeNames(nodeMap.keySet()); Assert.assertEquals(3, rrs.getLimitSize()); MySqlStatementParser parser = new MySqlStatementParser("SELECT * FROM orders LIMIT 0,5"); SQLStatement statement = parser.parseStatement(); Assert.assertEquals(statement.toString(), rrs.getNodes()[0].getStatement()); //SQL not span multi datanode sql = "select * from customer where id=10000 limit 2,3"; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertEquals(true, rrs.isCacheAble()); nodeMap = getNodeMap(rrs, 1); nameAsserter = new NodeNameAsserter("dn1"); nameAsserter.assertRouteNodeNames(nodeMap.keySet()); Assert.assertEquals(3, rrs.getLimitSize()); Assert.assertEquals("select * from customer where id=10000 limit 2,3", rrs.getNodes()[0].getStatement()); } public void testGroupLimit() throws Exception { final SchemaConfig schema = schemaMap.get("cndb"); String sql = null; RouteResultset rrs = null; sql = "select count(*) from (select * from(select * from offer_detail where offer_id='123' or offer_id='234' limit 88)offer where offer.member_id='abc' limit 60) w " + " where w.member_id ='pavarotti17' limit 99"; rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(true, rrs.isCacheAble()); // Assert.assertEquals(88L, rrs.getLimitSize()); // Assert.assertEquals(RouteResultset.SUM_FLAG, rrs.getFlag()); Map nodeMap = getNodeMap(rrs, 2); NodeNameAsserter nameAsserter = new NodeNameAsserter("detail_dn29", "detail_dn15"); nameAsserter.assertRouteNodeNames(nodeMap.keySet()); sql = "select count(*) from (select * from(select max(id) from offer_detail where offer_id='123' or offer_id='234' limit 88)offer where offer.member_id='abc' limit 60) w " + " where w.member_id ='pavarotti17' limit 99"; rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(true, rrs.isCacheAble()); nodeMap = getNodeMap(rrs, 2); nameAsserter = new NodeNameAsserter("detail_dn29", "detail_dn15"); nameAsserter.assertRouteNodeNames(nodeMap.keySet()); sql = "select * from (select * from(select max(id) from offer_detail where offer_id='123' or offer_id='234' limit 88)offer where offer.member_id='abc' limit 60) w " + " where w.member_id ='pavarotti17' limit 99"; rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(true, rrs.isCacheAble()); nodeMap = getNodeMap(rrs, 2); nameAsserter = new NodeNameAsserter("detail_dn29", "detail_dn15"); nameAsserter.assertRouteNodeNames(nodeMap.keySet()); sql = "select * from (select count(*) from(select * from offer_detail where offer_id='123' or offer_id='234' limit 88)offer where offer.member_id='abc' limit 60) w " + " where w.member_id ='pavarotti17' limit 99"; rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(true, rrs.isCacheAble()); // Assert.assertEquals(88L, rrs.getLimitSize()); // Assert.assertEquals(RouteResultset.SUM_FLAG, rrs.getFlag()); nodeMap = getNodeMap(rrs, 2); nameAsserter = new NodeNameAsserter("detail_dn29", "detail_dn15"); nameAsserter.assertRouteNodeNames(nodeMap.keySet()); } // public void testTableMetaRead() throws Exception { // final SchemaConfig schema = schemaMap.get("cndb"); // // String sql = " desc offer"; // RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.DESCRIBE, sql, null, null, // cachePool); // Assert.assertEquals(false, rrs.isCacheAble()); // Assert.assertEquals(-1L, rrs.getLimitSize()); // Assert.assertEquals(1, rrs.getNodes().length); // // random return one node // // Assert.assertEquals("offer_dn[0]", rrs.getNodes()[0].getName()); // Assert.assertEquals("desc offer", rrs.getNodes()[0].getStatement()); // // sql = "desc cndb.offer"; // rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.DESCRIBE, sql, null, null, cachePool); // Assert.assertEquals(false, rrs.isCacheAble()); // Assert.assertEquals(-1L, rrs.getLimitSize()); // Assert.assertEquals(1, rrs.getNodes().length); // // random return one node // // Assert.assertEquals("offer_dn[0]", rrs.getNodes()[0].getName()); // Assert.assertEquals("desc offer", rrs.getNodes()[0].getStatement()); // // sql = "desc cndb.offer col1"; // rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.DESCRIBE, sql, null, null, cachePool); // Assert.assertEquals(false, rrs.isCacheAble()); // Assert.assertEquals(-1L, rrs.getLimitSize()); // Assert.assertEquals(1, rrs.getNodes().length); // // random return one node // // Assert.assertEquals("offer_dn[0]", rrs.getNodes()[0].getName()); // Assert.assertEquals("desc offer col1", rrs.getNodes()[0].getStatement()); // // sql = "SHOW FULL COLUMNS FROM offer IN db_name WHERE true"; // rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SHOW, sql, null, null, // cachePool); // Assert.assertEquals(false, rrs.isCacheAble()); // Assert.assertEquals(-1L, rrs.getLimitSize()); // Assert.assertEquals(1, rrs.getNodes().length); // // random return one node // // Assert.assertEquals("offer_dn[0]", rrs.getNodes()[0].getName()); // Assert.assertEquals("SHOW FULL COLUMNS FROM offer WHERE true", // rrs.getNodes()[0].getStatement()); // // sql = "SHOW FULL COLUMNS FROM db.offer IN db_name WHERE true"; // rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SHOW, sql, null, null, // cachePool); // Assert.assertEquals(-1L, rrs.getLimitSize()); // Assert.assertEquals(false, rrs.isCacheAble()); // Assert.assertEquals(1, rrs.getNodes().length); // // random return one node // // Assert.assertEquals("offer_dn[0]", rrs.getNodes()[0].getName()); // Assert.assertEquals("SHOW FULL COLUMNS FROM offer WHERE true", // rrs.getNodes()[0].getStatement()); // // // sql = "SHOW FULL TABLES FROM `TESTDB` WHERE Table_type != 'VIEW'"; // rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SHOW, sql, null, null, // cachePool); // Assert.assertEquals(-1L, rrs.getLimitSize()); // Assert.assertEquals(false, rrs.isCacheAble()); // Assert.assertEquals("SHOW FULL TABLES WHERE Table_type != 'VIEW'", rrs.getNodes()[0].getStatement()); // // sql = "SHOW INDEX IN offer FROM db_name"; // rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SHOW, sql, null, null, // cachePool); // Assert.assertEquals(false, rrs.isCacheAble()); // Assert.assertEquals(-1L, rrs.getLimitSize()); // Assert.assertEquals(1, rrs.getNodes().length); // // random return one node // // Assert.assertEquals("offer_dn[0]", rrs.getNodes()[0].getName()); // Assert.assertEquals("SHOW INDEX FROM offer", // rrs.getNodes()[0].getStatement()); // // sql = "SHOW TABLES from db_name like 'solo'"; // rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SHOW, sql, null, null, // cachePool); // Assert.assertEquals(false, rrs.isCacheAble()); // Assert.assertEquals(-1L, rrs.getLimitSize()); // Map nodeMap = getNodeMap(rrs, 3); // NodeNameAsserter nameAsserter = new NodeNameAsserter("detail_dn0", // "offer_dn0", "independent_dn0"); // nameAsserter.assertRouteNodeNames(nodeMap.keySet()); // SimpleSQLAsserter sqlAsserter = new SimpleSQLAsserter(); // sqlAsserter.addExpectSQL(0, "SHOW TABLES like 'solo'") // .addExpectSQL(1, "SHOW TABLES like 'solo'") // .addExpectSQL(2, "SHOW TABLES like 'solo'") // .addExpectSQL(3, "SHOW TABLES like 'solo'"); // RouteNodeAsserter asserter = new RouteNodeAsserter(nameAsserter, // sqlAsserter); // for (RouteResultsetNode node : nodeMap.values()) { // asserter.assertNode(node); // } // // sql = "SHOW TABLES in db_name "; // rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SHOW, sql, null, null, // cachePool); // Assert.assertEquals(false, rrs.isCacheAble()); // Assert.assertEquals(-1L, rrs.getLimitSize()); // nodeMap = getNodeMap(rrs, 3); // nameAsserter = new NodeNameAsserter("detail_dn0", "offer_dn0", // "independent_dn0"); // nameAsserter.assertRouteNodeNames(nodeMap.keySet()); // sqlAsserter = new SimpleSQLAsserter(); // sqlAsserter.addExpectSQL(0, "SHOW TABLES") // .addExpectSQL(1, "SHOW TABLES").addExpectSQL(2, "SHOW TABLES") // .addExpectSQL(3, "SHOW TABLES"); // asserter = new RouteNodeAsserter(nameAsserter, sqlAsserter); // for (RouteResultsetNode node : nodeMap.values()) { // asserter.assertNode(node); // } // // sql = "SHOW TABLeS "; // rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SHOW, sql, null, null, // cachePool); // Assert.assertEquals(false, rrs.isCacheAble()); // Assert.assertEquals(-1L, rrs.getLimitSize()); // nodeMap = getNodeMap(rrs, 3); // nameAsserter = new NodeNameAsserter("offer_dn0", "detail_dn0", // "independent_dn0"); // nameAsserter.assertRouteNodeNames(nodeMap.keySet()); // sqlAsserter = new SimpleSQLAsserter(); // sqlAsserter.addExpectSQL(0, "SHOW TABLeS ") // .addExpectSQL(1, "SHOW TABLeS ").addExpectSQL(2, "SHOW TABLeS "); // asserter = new RouteNodeAsserter(nameAsserter, sqlAsserter); // for (RouteResultsetNode node : nodeMap.values()) { // asserter.assertNode(node); // } // } public void testConfigSchema() throws Exception { try { SchemaConfig schema = schemaMap.get("config"); String sql = "select * from offer where offer_id=1"; routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertFalse(true); } catch (Exception e) { Assert.assertEquals("route rule for table OFFER is required: select * from offer where offer_id=1", e.getMessage()); } try { SchemaConfig schema = schemaMap.get("config"); String sql = "select * from offer where col11111=1"; routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertFalse(true); } catch (Exception e) { } try { SchemaConfig schema = schemaMap.get("config"); String sql = "select * from offer "; routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertFalse(true); } catch (Exception e) { } } public void testIgnoreSchema() throws Exception { SchemaConfig schema = schemaMap.get("ignoreSchemaTest"); String sql = "select * from offer where offer_id=1"; RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(false, rrs.isCacheAble()); Assert.assertEquals("cndb_dn", rrs.getNodes()[0].getName()); Assert.assertEquals(sql, rrs.getNodes()[0].getStatement()); sql = "select * from ignoreSchemaTest.offer1 where ignoreSchemaTest.offer1.offer_id=1"; rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(false, rrs.isCacheAble()); Assert.assertEquals("select * from offer1 where offer1.offer_id=1", rrs.getNodes()[0].getStatement()); sql = "select * from ignoreSchemaTest2.offer where ignoreSchemaTest2.offer.offer_id=1"; rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(false, rrs.isCacheAble()); Assert.assertEquals(sql, rrs.getNodes()[0].getStatement(), sql); sql = "select * from ignoreSchemaTest2.offer a,offer b where ignoreSchemaTest2.offer.offer_id=1"; rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(false, rrs.isCacheAble()); Assert.assertEquals( "select * from ignoreSchemaTest2.offer a,offer b where ignoreSchemaTest2.offer.offer_id=1", rrs.getNodes()[0].getStatement()); } // public void testNonPartitionSQL() throws Exception { // // SchemaConfig schema = schemaMap.get("cndb"); // String sql = null; // RouteResultset rrs = null; // // schema = schemaMap.get("dubbo2"); // sql = "SHOW TABLES from db_name like 'solo'"; // rrs = routeStrategy.route(new SystemConfig(), schema, 9, sql, null, null, cachePool); // Assert.assertEquals(false, rrs.isCacheAble()); // Assert.assertEquals(-1L, rrs.getLimitSize()); // Assert.assertEquals(1, rrs.getNodes().length); // Assert.assertEquals("dn1", rrs.getNodes()[0].getName()); // Assert.assertEquals("SHOW TABLES like 'solo'", // rrs.getNodes()[0].getStatement()); // // schema = schemaMap.get("dubbo"); // sql = "SHOW TABLES from db_name like 'solo'"; // rrs = routeStrategy.route(new SystemConfig(), schema, 9, sql, null, null, cachePool); // Assert.assertEquals(false, rrs.isCacheAble()); // Assert.assertEquals(-1L, rrs.getLimitSize()); // Assert.assertEquals(1, rrs.getNodes().length); // Assert.assertEquals("dubbo_dn", rrs.getNodes()[0].getName()); // Assert.assertEquals("SHOW TABLES like 'solo'", // rrs.getNodes()[0].getStatement()); // // // // sql = "desc cndb.offer"; // rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); // Assert.assertEquals(false, rrs.isCacheAble()); // Assert.assertEquals(-1L, rrs.getLimitSize()); // Assert.assertEquals(1, rrs.getNodes().length); // Assert.assertEquals("dubbo_dn", rrs.getNodes()[0].getName()); // Assert.assertEquals("desc cndb.offer", rrs.getNodes()[0].getStatement()); // // schema = schemaMap.get("cndb"); // sql = "SHOW fulL TaBLES from db_name like 'solo'"; // rrs = routeStrategy.route(new SystemConfig(), schema, 9, sql, null, null, cachePool); // Assert.assertEquals(false, rrs.isCacheAble()); // Map nodeMap = getNodeMap(rrs, 3); // NodeNameAsserter nameAsserter = new NodeNameAsserter("detail_dn0", // "offer_dn0", "independent_dn0"); // nameAsserter.assertRouteNodeNames(nodeMap.keySet()); // SimpleSQLAsserter sqlAsserter = new SimpleSQLAsserter(); // sqlAsserter.addExpectSQL(0, "SHOW FULL TABLES like 'solo'") // .addExpectSQL(1, "SHOW FULL TABLES like 'solo'") // .addExpectSQL(2, "SHOW FULL TABLES like 'solo'") // .addExpectSQL(3, "SHOW FULL TABLES like 'solo'"); // RouteNodeAsserter asserter = new RouteNodeAsserter(nameAsserter, // sqlAsserter); // for (RouteResultsetNode node : nodeMap.values()) { // asserter.assertNode(node); // } // } public void testGlobalTableSingleNodeLimit() throws Exception { SchemaConfig schema = schemaMap.get("TESTDB"); String sql = "select * from globalsn"; RouteResultset rrs = null; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertEquals(100L, rrs.getLimitSize()); } /** * select 1 * select 1 union all select 2 * * @throws Exception */ public void testSelectNoTable() throws Exception { SchemaConfig schema = schemaMap.get("TESTDB"); String sql = "select 1"; RouteResultset rrs = null; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); sql = "select 1 union select 2"; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); } /** * 支持insert into ... values (),()... * 不支持insert into ... select... * * @throws Exception */ public void testBatchInsert() throws Exception { SchemaConfig schema = schemaMap.get("TESTDB"); RouteResultset rrs = null; //不支持childtable 批量插入 String sql = "insert into orders (id,name,customer_id) values(1,'testonly',1),(2,'testonly',2000001)"; try { rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); } catch (Exception e) { Assert.assertEquals("ChildTable multi insert not provided", e.getMessage()); } sql = "insert into employee (id,name,customer_id) select id,name,customer_id from customer"; try { rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); } catch (Exception e) { Assert.assertEquals("TODO:insert into .... select .... not supported!", e.getMessage()); } //分片表批量插入正常 employee sql = "insert into employee (id,name,sharding_id) values(1,'testonly',10000),(2,'testonly','10010')"; rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); Assert.assertEquals(false, rrs.isCacheAble()); Assert.assertEquals("dn1", rrs.getNodes()[0].getName()); Assert.assertEquals("dn2", rrs.getNodes()[1].getName()); String node1Sql = formatSql("insert into employee (id,name,sharding_id) values(1,'testonly','10000')"); String node2Sql = formatSql("insert into employee (id,name,sharding_id) values(2,'testonly','10010')"); RouteResultsetNode[] nodes = rrs.getNodes(); Assert.assertEquals(node1Sql, nodes[0].getStatement()); Assert.assertEquals(node2Sql, nodes[1].getStatement()); } /** * insert ... on duplicate key ... update... * * @throws Exception */ public void testInsertOnDuplicateKey() throws Exception { SchemaConfig schema = schemaMap.get("TESTDB"); String sql = "insert into employee (id,name,sharding_id) values(1,'testonly',10000) on duplicate key update name='nihao'"; RouteResultset rrs = null; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals("dn1", rrs.getNodes()[0].getName()); //insert ... on duplicate key ... update col1 = VALUES(col1),col2 = VALUES(col2) sql = "insert into employee (id,name,sharding_id) values(1,'testonly',10000) " + "on duplicate key update name=VALUES(name),id = VALUES(id)"; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals("dn1", rrs.getNodes()[0].getName()); //insert ... on duplicate key ,sharding key can't be updated sql = "insert into employee (id,name,sharding_id) values(1,'testonly',10000) " + "on duplicate key update name=VALUES(name),id = VALUES(id),sharding_id = VALUES(sharding_id)"; try { rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); } catch (Exception e) { Assert.assertEquals("Sharding column can't be updated: EMPLOYEE -> SHARDING_ID", e.getMessage()); } } /** * 测试函数COUNT * * @throws Exception */ @Test public void testAggregateExpr() throws Exception { SchemaConfig schema = schemaMap.get("TESTDB"); String sql = "select id, name, count(name) from employee group by name;"; RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertTrue(rrs.getMergeCols().containsKey("count2")); sql = "select id, name, count(name) as c from employee group by name;"; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertTrue(rrs.getMergeCols().containsKey("c")); sql = "select id, name, count(name) c from employee group by name;"; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertTrue(rrs.getMergeCols().containsKey("c")); } /** * 测试between语句的路由 * * @throws Exception */ @Test public void testBetweenExpr() throws Exception { // 0-200M=0 // 200M1-400M=1 // 400M1-600M=2 // 600M1-800M=3 // 800M1-1000M=4 SchemaConfig schema = schemaMap.get("TESTDB"); String sql = "select * from customer where id between 1 and 5;"; RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertTrue(rrs.getNodes().length == 1); Assert.assertTrue(rrs.getNodes()[0].getName().equals("dn1")); sql = "select * from customer where id between 1 and 2000001;"; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertTrue(rrs.getNodes().length == 2); sql = "select * from customer where id between 2000001 and 3000001;"; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertTrue(rrs.getNodes().length == 1); Assert.assertTrue(rrs.getNodes()[0].getName().equals("dn2")); sql = "delete from customer where id between 2000001 and 3000001;"; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertTrue(rrs.getNodes().length == 1); Assert.assertTrue(rrs.getNodes()[0].getName().equals("dn2")); sql = "update customer set name='newName' where id between 2000001 and 3000001;"; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertTrue(rrs.getNodes().length == 1); Assert.assertTrue(rrs.getNodes()[0].getName().equals("dn2")); } /** * 测试or语句的路由 * * @throws Exception */ @Test public void testOr() throws Exception { // 0-200M=0 // 200M1-400M=1 // 400M1-600M=2 // 600M1-800M=3 // 800M1-1000M=4 SchemaConfig schema = schemaMap.get("TESTDB"); String sql = "select * from customer where sharding_id=10000 or 1=1;"; RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertTrue(rrs.getNodes().length == 2); Assert.assertTrue(rrs.getNodes()[0].getName().equals("dn1")); Assert.assertTrue(rrs.getNodes()[1].getName().equals("dn2")); sql = "select * from customer where sharding_id = 10000 or sharding_id = 10010"; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertTrue(rrs.getNodes()[0].getName().equals("dn1")); Assert.assertTrue(rrs.getNodes()[1].getName().equals("dn2")); sql = "select * from customer where sharding_id = 10000 or user_id = 'wangwu'"; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertTrue(rrs.getNodes()[0].getName().equals("dn1")); Assert.assertTrue(rrs.getNodes()[1].getName().equals("dn2")); // #2694 验证(name=1 or name=2) and id=11 的路由等价于id=11 and (name=1 or name=2) 路由 sql = "select * from customer where sharding_id = 10000 and (user_id = 'zhangsan' or user_id='lisi')"; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertTrue(rrs.getNodes()[0].getName().equals("dn1")); sql = "select * from customer where (user_id = 'zhangsan' or user_id='lisi') and sharding_id = 10000"; RouteResultset rrs2 = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertTrue(rrs2.getNodes()[0].getName().equals(rrs.getNodes()[0].getName())); } /** * 测试父子表，查询子表的语句路由到多个节点 * @throws Exception */ @Test public void testERRouteMutiNode() throws Exception { SchemaConfig schema = schemaMap.get("TESTDB"); String sql = "select * from orders where customer_id in(1,2000001);"; RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertTrue(rrs.getNodes().length == 2); Assert.assertTrue(rrs.getNodes()[0].getName().equals("dn1")); Assert.assertTrue(rrs.getNodes()[1].getName().equals("dn2")); } /** * 测试多层or语句 * * @throws Exception */ @Test public void testMultiLevelOr() throws Exception { SchemaConfig schema = schemaMap.get("TESTDB"); String sql = "select id from travelrecord " + " where id = 1 and ( fee=3 or days=5 or (traveldate = '2015-05-04 00:00:07.375' " + " and (user_id=2 or fee=days or fee = 0))) and name = 'zhangsan'" ; RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertTrue(rrs.getNodes().length == 1); sql = "select id from travelrecord " + " where id = 1 and ( fee=3 or days=5 or (traveldate = '2015-05-04 00:00:07.375' " + " and (user_id=2 or fee=days or fee = 0))) and name = 'zhangsan' or id = 2000001" ; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertTrue(rrs.getNodes().length == 2); sql = "select id from travelrecord " + " where id = 1 and ( fee=3 or days=5 or (traveldate = '2015-05-04 00:00:07.375' " + " and (user_id=2 or fee=days or fee = 0))) and name = 'zhangsan' or id = 2000001 or id = 4000001" ; rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertTrue(rrs.getNodes().length == 3); } /** * 测试 global table 的or语句 * * * @throws Exception */ @Test public void testGlobalTableOr() throws Exception { SchemaConfig schema = schemaMap.get("TESTDB"); String sql = "select id from company where 1 = 1 and name ='company1' or name = 'company2'" ; for(int i = 0; i < 20; i++) { RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.SELECT, sql, null, null, cachePool); Assert.assertTrue(rrs.getNodes().length == 1); } } /** * 测试别名路由 * * @throws Exception */ public void testAlias() throws Exception { SchemaConfig schema = schemaMap.get("TESTDB"); RouteResultset rrs = null; //不支持childtable 批量插入 //update 全局表 String sql = "update company a set name = '' where a.id = 1;"; rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(3, rrs.getNodes().length); //update带别名时的路由 sql = "update travelrecord a set name = '' where a.id = 1;"; rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); //别名大小写路由 sql = "select * from travelrecord A where a.id = 1;"; rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); } /** * 测试"schema.table" * * @throws Exception */ public void testSchemaTable() throws Exception { SchemaConfig schema = schemaMap.get("schema_table_test"); RouteResultset rrs = null; // 1 schema==当前逻辑schema String sql = "select * from schema_table_test.offer where id = 1;"; rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertTrue(rrs.getNodes()[0].getName().equals("dn1")); // 2 schema !=当前schema,路由到默认节点。带alias sql = "select * from noExistSchema.offer a where a.id = 1;"; rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertTrue(rrs.getNodes()[0].getName().equals("dn2")); // 不带alias sql = "select * from noExistSchema.offer where id = 1;"; rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertTrue(rrs.getNodes()[0].getName().equals("dn2")); // 3 默认节点不存在，抛错 schema = schemaMap.get("TESTDB"); sql = "select * from schema_table_test.offer where id = 1;"; try { rrs = routeStrategy.route(new SystemConfig(), schema, 1, sql, null, null, cachePool); } catch (Exception e) { Assert.assertTrue(e.getMessage().contains( "can't find table define in schema SCHEMA_TABLE_TEST.OFFER alias：schema_table_test.offer, schema:TESTDB")); } } private String formatSql(String sql) { MySqlStatementParser parser = new MySqlStatementParser(sql); SQLStatement stmt = parser.parseStatement(); return stmt.toString(); } } ================================================ FILE: src/test/java/io/mycat/route/DruidMysqlSqlParserTest.java ================================================ package io.mycat.route; import java.sql.SQLNonTransientException; import java.util.Map; import org.junit.Test; import io.mycat.MycatServer; import io.mycat.SimpleCachePool; import io.mycat.cache.LayerCachePool; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.route.factory.RouteStrategyFactory; import io.mycat.server.parser.ServerParse; import junit.framework.Assert; public class DruidMysqlSqlParserTest { protected Map schemaMap; protected LayerCachePool cachePool = new SimpleCachePool(); protected RouteStrategy routeStrategy; public DruidMysqlSqlParserTest() { String schemaFile = "/route/schema.xml"; String ruleFile = "/route/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schemaMap = schemaLoader.getSchemas(); MycatServer.getInstance().getConfig().getSchemas().putAll(schemaMap); RouteStrategyFactory.init(); routeStrategy = RouteStrategyFactory.getRouteStrategy("druidparser"); } @Test public void testLimitPage() throws SQLNonTransientException { String sql = "select * from offer order by id desc limit 5,10"; SchemaConfig schema = schemaMap.get("mysqldb"); RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(15, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals("dn1", rrs.getNodes()[0].getName()); Assert.assertEquals("dn2", rrs.getNodes()[1].getName()); sql= rrs.getNodes()[0].getStatement() ; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(15, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals(0, rrs.getLimitStart()); Assert.assertEquals(15, rrs.getLimitSize()); sql="select * from offer1 order by id desc limit 5,10" ; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(5, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(10, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals("dn1", rrs.getNodes()[0].getName()); sql="select * from offer1 order by id desc limit 10" ; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(0, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(10, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals("dn1", rrs.getNodes()[0].getName()); } @Test public void testLockTableSql() throws SQLNonTransientException{ String sql = "lock tables goods write"; SchemaConfig schema = schemaMap.get("TESTDB"); RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, ServerParse.LOCK, sql, null, null, cachePool); Assert.assertEquals(3, rrs.getNodes().length); } } ================================================ FILE: src/test/java/io/mycat/route/DruidMysqlSqlSubqueriesParserTest.java ================================================ package io.mycat.route; import java.sql.SQLNonTransientException; import java.util.Map; import org.junit.Test; import io.mycat.MycatServer; import io.mycat.SimpleCachePool; import io.mycat.cache.LayerCachePool; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.route.factory.RouteStrategyFactory; import junit.framework.Assert; @SuppressWarnings("deprecation") public class DruidMysqlSqlSubqueriesParserTest { protected Map schemaMap; protected LayerCachePool cachePool = new SimpleCachePool(); protected RouteStrategy routeStrategy; public DruidMysqlSqlSubqueriesParserTest() { String schemaFile = "/route/schema.xml"; String ruleFile = "/route/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schemaMap = schemaLoader.getSchemas(); MycatServer.getInstance().getConfig().getSchemas().putAll(schemaMap); RouteStrategyFactory.init(); routeStrategy = RouteStrategyFactory.getRouteStrategy("druidparser"); } @Test public void testSubQueries() throws SQLNonTransientException { //子查询测试需要构建ServerConnection. 暂时不在单元测试中体现.以测试报告的形式体现 } } ================================================ FILE: src/test/java/io/mycat/route/DruidOracleSqlParserTest.java ================================================ package io.mycat.route; import java.sql.SQLNonTransientException; import java.util.Map; import org.junit.Test; import io.mycat.MycatServer; import io.mycat.SimpleCachePool; import io.mycat.cache.LayerCachePool; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.route.factory.RouteStrategyFactory; import junit.framework.Assert; public class DruidOracleSqlParserTest { protected Map schemaMap; protected LayerCachePool cachePool = new SimpleCachePool(); protected RouteStrategy routeStrategy; public DruidOracleSqlParserTest() { String schemaFile = "/route/schema.xml"; String ruleFile = "/route/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schemaMap = schemaLoader.getSchemas(); MycatServer.getInstance().getConfig().getSchemas().putAll(schemaMap); RouteStrategyFactory.init(); routeStrategy = RouteStrategyFactory.getRouteStrategy("druidparser"); } @Test public void testInsertUpdate() throws Exception { SchemaConfig schema = schemaMap.get("oracledb"); String sql = "insert into offer1(group_id,offer_id,member_id)values(234,123,'abc')"; RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(false, rrs.isCacheAble()); Assert.assertEquals(-1, rrs.getLimitSize()); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(-1, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals("d_oracle1", rrs.getNodes()[0].getName()); Assert.assertEquals( "insert into offer1(group_id,offer_id,member_id)values(234,123,'abc')", rrs.getNodes()[0].getStatement()); sql = "update offer set name='x'"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); Assert.assertEquals(-1, rrs.getLimitSize()); Assert.assertEquals(false, rrs.isCacheAble()); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(-1, rrs.getNodes()[0].getLimitSize()); } @Test public void testLimitToOraclePage() throws SQLNonTransientException { String sql = "select * from offer order by id desc limit 5,10"; SchemaConfig schema = schemaMap.get("oracledb"); RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(15, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals("d_oracle1", rrs.getNodes()[0].getName()); Assert.assertEquals("d_oracle2", rrs.getNodes()[1].getName()); // 语法解析报错，先屏蔽 // sql= rrs.getNodes()[0].getStatement() ; // rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, // null, cachePool); // Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); // Assert.assertEquals(15, rrs.getNodes()[0].getLimitSize()); // Assert.assertEquals(0, rrs.getLimitStart()); // Assert.assertEquals(15, rrs.getLimitSize()); sql="select * from offer1 order by id desc limit 5,10" ; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(5, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(10, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals("d_oracle1", rrs.getNodes()[0].getName()); } @Test public void testOraclePageSQL() throws SQLNonTransientException { String sql = "SELECT *\n" + "FROM (SELECT XX.*, ROWNUM AS RN \n" + " FROM (\n" + "SELECT * FROM offer\n" + " ) XX\n" + " WHERE ROWNUM <= 15\n" + " ) XXX\n" + "WHERE RN > 5 \n"; SchemaConfig schema = schemaMap.get("oracledb"); RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(15, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals("d_oracle1", rrs.getNodes()[0].getName()); Assert.assertEquals("d_oracle2", rrs.getNodes()[1].getName()); sql = "SELECT *\n" + "FROM (SELECT XX.*, ROWNUM AS RN \n" + " FROM (\n" + "SELECT * FROM offer1" + " ) XX\n" + " WHERE ROWNUM <= 15\n" + " ) XXX\n" + "WHERE RN > 5 \n"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(5, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(10, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals(sql,rrs.getNodes()[0].getStatement()) ; Assert.assertEquals("d_oracle1", rrs.getNodes()[0].getName()); sql="SELECT *\n" + "FROM (SELECT t.*, ROW_NUMBER() OVER (ORDER BY sid DESC) AS ROWNUM1\n" + "\tFROM offer t\n" + "\tWHERE sts <> 'N'\n" + "\t\t\n" + "\t) XX\n" + "WHERE ROWNUM1 > 5\n" + "\tAND ROWNUM1 <= 15\n"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(15, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals("d_oracle1", rrs.getNodes()[0].getName()); Assert.assertEquals("d_oracle2", rrs.getNodes()[1].getName()); sql="SELECT *\n" + "FROM (SELECT t.*, ROW_NUMBER() OVER (ORDER BY sid DESC) AS ROWNUM1\n" + "\tFROM offer1 t\n" + "\tWHERE sts <> 'N'\n" + "\t\t\n" + "\t) XX\n" + "WHERE ROWNUM1 > 5\n" + "\tAND ROWNUM1 <= 15\n"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(5, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(10, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals(sql,rrs.getNodes()[0].getStatement()) ; Assert.assertEquals("d_oracle1", rrs.getNodes()[0].getName()); sql="select sid from (select sid from offer ) where rownum<=10" ; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); Assert.assertEquals(0, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(10, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals(sql,rrs.getNodes()[0].getStatement()) ; Assert.assertEquals("d_oracle1", rrs.getNodes()[0].getName()); Assert.assertEquals("d_oracle2", rrs.getNodes()[1].getName()); } } ================================================ FILE: src/test/java/io/mycat/route/DruidPostgresqlSqlParserTest.java ================================================ package io.mycat.route; import java.sql.SQLNonTransientException; import java.util.Map; import org.junit.Test; import io.mycat.MycatServer; import io.mycat.SimpleCachePool; import io.mycat.cache.LayerCachePool; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.route.factory.RouteStrategyFactory; import junit.framework.Assert; public class DruidPostgresqlSqlParserTest { protected Map schemaMap; protected LayerCachePool cachePool = new SimpleCachePool(); protected RouteStrategy routeStrategy; public DruidPostgresqlSqlParserTest() { String schemaFile = "/route/schema.xml"; String ruleFile = "/route/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schemaMap = schemaLoader.getSchemas(); MycatServer.getInstance().getConfig().getSchemas().putAll(schemaMap); RouteStrategyFactory.init(); routeStrategy = RouteStrategyFactory.getRouteStrategy("druidparser"); } @Test public void testLimitToPgPage() throws SQLNonTransientException { String sql = "select * from offer order by id desc limit 5,10"; SchemaConfig schema = schemaMap.get("pgdb"); RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(15, rrs.getNodes()[0].getLimitSize()); sql= rrs.getNodes()[0].getStatement() ; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(15, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals(0, rrs.getLimitStart()); Assert.assertEquals(15, rrs.getLimitSize()); sql="select * from offer1 order by id desc limit 5,10" ; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(5, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(10, rrs.getNodes()[0].getLimitSize()); } @Test public void testPGPageSQL() throws SQLNonTransientException { String sql = "select sid from offer order by sid limit 10 offset 5"; SchemaConfig schema = schemaMap.get("pgdb"); RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(15, rrs.getNodes()[0].getLimitSize()); sql = "select sid from offer1 order by sid limit 10 offset 5"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(5, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(10, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals("SELECT sid\n" + "FROM offer1\n" + "ORDER BY sid\n" + "LIMIT 10 OFFSET 5",rrs.getNodes()[0].getStatement()) ; } } ================================================ FILE: src/test/java/io/mycat/route/DruidSqlServerSqlParserTest.java ================================================ package io.mycat.route; import java.sql.SQLNonTransientException; import java.util.Map; import org.junit.Test; import io.mycat.MycatServer; import io.mycat.SimpleCachePool; import io.mycat.cache.LayerCachePool; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.route.factory.RouteStrategyFactory; import junit.framework.Assert; public class DruidSqlServerSqlParserTest { protected Map schemaMap; protected LayerCachePool cachePool = new SimpleCachePool(); protected RouteStrategy routeStrategy; public DruidSqlServerSqlParserTest() { String schemaFile = "/route/schema.xml"; String ruleFile = "/route/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schemaMap = schemaLoader.getSchemas(); MycatServer.getInstance().getConfig().getSchemas().putAll(schemaMap); RouteStrategyFactory.init(); routeStrategy = RouteStrategyFactory.getRouteStrategy("druidparser"); } @Test public void testLimitToSqlServerPage() throws SQLNonTransientException { String sql = "select * from offer order by id desc limit 5,10"; SchemaConfig schema = schemaMap.get("sqlserverdb"); RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(15, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals("sqlserver_1", rrs.getNodes()[0].getName()); Assert.assertEquals("sqlserver_2", rrs.getNodes()[1].getName()); ///语句解析报错，先注释掉 // sql= rrs.getNodes()[0].getStatement() ; // rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, // null, cachePool); // Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); // Assert.assertEquals(15, rrs.getNodes()[0].getLimitSize()); // Assert.assertEquals(0, rrs.getLimitStart()); // Assert.assertEquals(15, rrs.getLimitSize()); sql="select * from offer1 order by id desc limit 5,10" ; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(5, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(10, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals("sqlserver_1", rrs.getNodes()[0].getName()); } @Test public void testSqlServerPageSQL() throws SQLNonTransientException { String sql = "SELECT *\n" + "FROM (SELECT sid, ROW_NUMBER() OVER (ORDER BY sid DESC) AS ROWNUM\n" + "\tFROM offer \n" + "\tWHERE sts <> 'N'\n" + "\t\t\t) XX\n" + "WHERE ROWNUM > 5\n" + "\tAND ROWNUM <= 15\n"; SchemaConfig schema = schemaMap.get("sqlserverdb"); RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(15, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals("sqlserver_1", rrs.getNodes()[0].getName()); Assert.assertEquals("sqlserver_2", rrs.getNodes()[1].getName()); sql = "SELECT *\n" + "FROM (SELECT sid, ROW_NUMBER() OVER (ORDER BY sid DESC) AS ROWNUM\n" + "\tFROM offer1 \n" + "\tWHERE sts <> 'N'\n" + "\t\t\t) XX\n" + "WHERE ROWNUM > 5\n" + "\tAND ROWNUM <= 15\n"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(5, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(10, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals(sql, rrs.getNodes()[0].getStatement()); Assert.assertEquals("sqlserver_1", rrs.getNodes()[0].getName()); sql = "select * from ( select row_number()over(order by tempColumn)tempRowNumber,* from ( select top \n" + "15 tempColumn=0, sid \n" + "from offer where sts<>'N' and asf like '%'+'akka'+'%' order by sid )t )tt where tempRowNumber>5"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(15, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals("sqlserver_1", rrs.getNodes()[0].getName()); Assert.assertEquals("sqlserver_2", rrs.getNodes()[1].getName()); sql = "select * from ( select row_number()over(order by tempColumn)tempRowNumber,* from ( select top \n" + "15 tempColumn=0, sid \n" + "from offer1 where sts<>'N' and asf like '%'+'akka'+'%' order by sid )t )tt where tempRowNumber>5"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(5, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(5, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(10, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals(sql, rrs.getNodes()[0].getStatement()); Assert.assertEquals("sqlserver_1", rrs.getNodes()[0].getName()); sql = "SELECT TOP 10 sid \n" + " FROM offer where sts<>'N' and asf like '%'+'akka'+'%' \n" + " ORDER BY sid desc"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); Assert.assertEquals(0, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(10, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals(sql, rrs.getNodes()[0].getStatement()); Assert.assertEquals("sqlserver_1", rrs.getNodes()[0].getName()); Assert.assertEquals("sqlserver_2", rrs.getNodes()[1].getName()); } @Test public void testTopPageSQL() throws SQLNonTransientException { SchemaConfig schema = schemaMap.get("sqlserverdb"); RouteResultset rrs = null; String sql = "SELECT TOP 10 * \n" + " FROM offer1 where sts<>'N' and asf like '%'+'akka'+'%' \n" + " ORDER BY sid desc"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(0, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(10, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals(sql, rrs.getNodes()[0].getStatement()); Assert.assertEquals("sqlserver_1", rrs.getNodes()[0].getName()); sql = "SELECT TOP 10 offer1.name,offer1.id \n" + " FROM offer1 where sts<>'N' and asf like '%'+'akka'+'%' \n" + " ORDER BY sid desc"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(1, rrs.getNodes().length); Assert.assertEquals(0, rrs.getLimitStart()); Assert.assertEquals(10, rrs.getLimitSize()); Assert.assertEquals(0, rrs.getNodes()[0].getLimitStart()); Assert.assertEquals(10, rrs.getNodes()[0].getLimitSize()); Assert.assertEquals(sql, rrs.getNodes()[0].getStatement()); Assert.assertEquals("sqlserver_1", rrs.getNodes()[0].getName()); } } ================================================ FILE: src/test/java/io/mycat/route/HintDBTypeTest.java ================================================ package io.mycat.route; import io.mycat.MycatServer; import io.mycat.SimpleCachePool; import io.mycat.cache.CacheService; import io.mycat.cache.LayerCachePool; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.route.factory.RouteStrategyFactory; import io.mycat.server.parser.ServerParse; import java.util.Map; import junit.framework.Assert; import org.junit.Test; public class HintDBTypeTest { protected Map schemaMap; protected LayerCachePool cachePool = new SimpleCachePool(); protected RouteStrategy routeStrategy; public HintDBTypeTest() { String schemaFile = "/route/schema.xml"; String ruleFile = "/route/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schemaMap = schemaLoader.getSchemas(); MycatServer.getInstance().getConfig().getSchemas().putAll(schemaMap); RouteStrategyFactory.init(); routeStrategy = RouteStrategyFactory.getRouteStrategy("druidparser"); } /** * 测试注解 * * @throws Exception */ @Test public void testHint() throws Exception { SchemaConfig schema = schemaMap.get("TESTDB"); //使用注解（新注解，/*!mycat*/），runOnSlave=false 强制走主节点 String sql = "/*!mycat:db_type=master*/select * from employee where sharding_id=1"; CacheService cacheService = new CacheService(); RouteService routerService = new RouteService(cacheService); RouteResultset rrs = routerService.route(new SystemConfig(), schema, ServerParse.SELECT, sql, "UTF-8", null); Assert.assertTrue(!rrs.getRunOnSlave()); //使用注解（新注解，/*#mycat*/），runOnSlave=false 强制走主节点 sql = "/*#mycat:db_type=master*/select * from employee where sharding_id=1"; rrs = routerService.route(new SystemConfig(), schema, ServerParse.SELECT, sql, "UTF-8", null); Assert.assertTrue(!rrs.getRunOnSlave()); //使用注解（新注解，/*mycat*/），runOnSlave=false 强制走主节点 sql = "/*mycat:db_type=master*/select * from employee where sharding_id=1"; rrs = routerService.route(new SystemConfig(), schema, ServerParse.SELECT, sql, "UTF-8", null); Assert.assertTrue(!rrs.getRunOnSlave()); //不使用注解，runOnSlave=null, 根据读写分离策略走主从库 sql = "select * from employee where sharding_id=1"; rrs = routerService.route(new SystemConfig(), schema, ServerParse.SELECT, sql, "UTF-8", null); Assert.assertTrue(rrs.getRunOnSlave()==null); } } ================================================ FILE: src/test/java/io/mycat/route/HintTest.java ================================================ package io.mycat.route; import java.util.Map; import junit.framework.Assert; import org.junit.Test; import io.mycat.MycatServer; import io.mycat.SimpleCachePool; import io.mycat.cache.CacheService; import io.mycat.cache.LayerCachePool; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.route.RouteResultset; import io.mycat.route.RouteService; import io.mycat.route.RouteStrategy; import io.mycat.route.factory.RouteStrategyFactory; import io.mycat.server.parser.ServerParse; public class HintTest { protected Map schemaMap; protected LayerCachePool cachePool = new SimpleCachePool(); protected RouteStrategy routeStrategy; public HintTest() { String schemaFile = "/route/schema.xml"; String ruleFile = "/route/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schemaMap = schemaLoader.getSchemas(); MycatServer.getInstance().getConfig().getSchemas().putAll(schemaMap); RouteStrategyFactory.init(); routeStrategy = RouteStrategyFactory.getRouteStrategy("fdbparser"); } /** * 测试注解 * * @throws Exception */ @Test public void testHint() throws Exception { SchemaConfig schema = schemaMap.get("TESTDB"); //使用注解（新注解，/*后面没有空格），路由到1个节点 String sql = "/*!mycat: sql = select * from employee where sharding_id = 10010 */select * from employee"; CacheService cacheService = new CacheService(); RouteService routerService = new RouteService(cacheService); RouteResultset rrs = routerService.route(new SystemConfig(), schema, ServerParse.SELECT, sql, "UTF-8", null); Assert.assertTrue(rrs.getNodes().length == 1); //使用注解（新注解，/*后面有空格），路由到1个节点 sql = "/*#mycat: sql = select * from employee where sharding_id = 10000 */select * from employee"; rrs = routerService.route(new SystemConfig(), schema, ServerParse.SELECT, sql, "UTF-8", null); Assert.assertTrue(rrs.getNodes().length == 1); // 支持hint里面带有”，“ sql = "/*!mycat: sql = select * from employee where sharding_id in ('10000','10010') */select * from employee"; rrs = routerService.route(new SystemConfig(), schema, ServerParse.SELECT, sql, "UTF-8", null); Assert.assertTrue(rrs.getNodes().length == 2); //不用注解，路由到2个节点 sql = "select * from employee"; rrs = routerService.route(new SystemConfig(), schema, ServerParse.SELECT, sql, "UTF-8", null); Assert.assertTrue(rrs.getNodes().length == 2); } } ================================================ FILE: src/test/java/io/mycat/route/TestSelectBetweenSqlParser.java ================================================ package io.mycat.route; import java.io.IOException; import java.sql.SQLNonTransientException; import java.util.Map; import junit.framework.Assert; import org.junit.Test; import io.mycat.MycatServer; import io.mycat.SimpleCachePool; import io.mycat.cache.LayerCachePool; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.route.factory.RouteStrategyFactory; import io.mycat.server.ServerConnection; /** * 修改内容 * * @author lxy * */ public class TestSelectBetweenSqlParser { protected Map schemaMap; protected LayerCachePool cachePool = new SimpleCachePool(); public TestSelectBetweenSqlParser() { String schemaFile = "/route/schema.xml"; String ruleFile = "/route/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schemaMap = schemaLoader.getSchemas(); MycatServer.getInstance().getConfig().getSchemas().putAll(schemaMap); RouteStrategyFactory.init(); } @Test public void testBetweenSqlRoute() throws SQLNonTransientException, IOException { String sql = "select * from offer_detail where offer_id between 1 and 33"; SchemaConfig schema = schemaMap.get("cndb"); RouteResultset rrs = RouteStrategyFactory.getRouteStrategy().route(new SystemConfig(),schema, -1, sql, null, null, cachePool); Assert.assertEquals(5, rrs.getNodes().length); sql = "select * from offer_detail where col_1 = 33 and offer_id between 1 and 33 and col_2 = 18"; schema = schemaMap.get("cndb"); rrs = RouteStrategyFactory.getRouteStrategy().route(new SystemConfig(),schema, -1, sql, null, null, cachePool); Assert.assertEquals(5, rrs.getNodes().length); // sql = "select b.* from offer_date b join offer_detail a on a.id=b.id " + // "where b.col_date between '2014-02-02' and '2014-04-12' and col_1 = 3 and offer_id between 1 and 33"; sql = "select b.* from offer_detail a join offer_date b on a.id=b.id " + "where b.col_date between '2014-02-02' and '2014-04-12' and col_1 = 3 and offer_id between 1 and 33"; // sql = "select a.* from offer_detail a join offer_date b on a.id=b.id " + // "where b.col_date = '2014-04-02' and col_1 = 33 and offer_id =1"; schema = schemaMap.get("cndb"); // 两个路由规则不一样的表现在 走catlet. 不再取交集, catlet 测试时需要前端连接.这里注释掉. // rrs = RouteStrategyFactory.getRouteStrategy().route(new SystemConfig(),schema, -1, sql, null, // null, cachePool); // Assert.assertEquals(2, rrs.getNodes().length); //这里2个表都有条件路由，取的是交集, //确认大于小于操作符 sql = "select b.* from offer_date b " + "where b.col_date > '2014-02-02'"; // sql = "select a.* from offer_detail a join offer_date b on a.id=b.id " + // "where b.col_date = '2014-04-02' and col_1 = 33 and offer_id =1"; schema = schemaMap.get("cndb"); rrs = RouteStrategyFactory.getRouteStrategy().route(new SystemConfig(),schema, -1, sql, null, null, cachePool); Assert.assertEquals(128, rrs.getNodes().length); sql = "select * from offer_date where col_1 = 33 and col_date between '2014-01-02' and '2014-01-12'"; schema = schemaMap.get("cndb"); rrs = RouteStrategyFactory.getRouteStrategy().route(new SystemConfig(),schema, -1, sql, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); sql = "select * from offer_date a where col_1 = 33 and a.col_date between '2014-01-02' and '2014-01-12'"; schema = schemaMap.get("cndb"); rrs = RouteStrategyFactory.getRouteStrategy().route(new SystemConfig(),schema, -1, sql, null, null, cachePool); Assert.assertEquals(2, rrs.getNodes().length); } } ================================================ FILE: src/test/java/io/mycat/route/function/AutoPartitionByLongTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.route.function; import junit.framework.Assert; import org.junit.Test; public class AutoPartitionByLongTest { @Test public void test() { AutoPartitionByLong autoPartition=new AutoPartitionByLong(); autoPartition.setMapFile("autopartition-long.txt"); autoPartition.init(); String idVal="0"; Assert.assertEquals(true, 0==autoPartition.calculate(idVal)); idVal="2000000"; Assert.assertEquals(true, 0==autoPartition.calculate(idVal)); idVal="2000001"; Assert.assertEquals(true, 1==autoPartition.calculate(idVal)); idVal="4000000"; Assert.assertEquals(true, 1==autoPartition.calculate(idVal)); idVal="4000001"; Assert.assertEquals(true, 2==autoPartition.calculate(idVal)); } } ================================================ FILE: src/test/java/io/mycat/route/function/PartitionByCRC32PreSlotTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.route.function; import io.mycat.config.model.TableConfig; import io.mycat.config.model.rule.RuleConfig; import org.junit.Assert; import org.junit.Test; import static io.mycat.route.function.PartitionByCRC32PreSlot.genDataNodesString; public class PartitionByCRC32PreSlotTest { static TableConfig genTableConfig(int count) { RuleConfig rule = new RuleConfig("id", "crc32slot"); String sb = genDataNodesString(count); TableConfig tableConf = new TableConfig("test", "id", true, false, -1, sb, null, rule, true, null, false, null, null, null, false); return tableConf; } @Test public void test() { PartitionByCRC32PreSlot partition = new PartitionByCRC32PreSlot(); partition.setRuleName("test"); partition.setTableConfig(genTableConfig(1000)); partition.reInit(); Assert.assertEquals(true, 521 == partition.calculate("1000316")); Assert.assertEquals(true, 637 == partition.calculate("2")); partition.setTableConfig(genTableConfig(2)); partition.reInit(); Assert.assertEquals(true, 0 == partition.calculate("1")); Assert.assertEquals(true, 1 == partition.calculate("2")); Assert.assertEquals(true, 0 == partition.calculate("3")); Assert.assertEquals(true, 1 == partition.calculate("4")); Assert.assertEquals(true, 0 == partition.calculate("5")); Assert.assertEquals(true, 0 == partition.calculate("6")); Assert.assertEquals(true, 0 == partition.calculate("7")); Assert.assertEquals(true, 0 == partition.calculate("8")); Assert.assertEquals(true, 0 == partition.calculate("9")); Assert.assertEquals(true, 0 == partition.calculate("9999")); Assert.assertEquals(true, 1 == partition.calculate("123456789")); Assert.assertEquals(true, 1 == partition.calculate("35565")); partition.setTableConfig(genTableConfig(3)); partition.reInit(); Assert.assertEquals(true, 1 == partition.calculate("1")); Assert.assertEquals(true, 1 == partition.calculate("2")); Assert.assertEquals(true, 0 == partition.calculate("3")); Assert.assertEquals(true, 2 == partition.calculate("4")); Assert.assertEquals(true, 0 == partition.calculate("5")); Assert.assertEquals(true, 1 == partition.calculate("6")); Assert.assertEquals(true, 1 == partition.calculate("7")); Assert.assertEquals(true, 0 == partition.calculate("8")); Assert.assertEquals(true, 0 == partition.calculate("9")); Assert.assertEquals(true, 0 == partition.calculate("9999")); Assert.assertEquals(true, 2 == partition.calculate("123456789")); Assert.assertEquals(true, 2 == partition.calculate("35565")); } public static void main(String[] args) { for (int i = 0; i < 20; i++) { int y = 9; int count = 3; long slot = i % y; int slotSize = y / count; Long index = slot / slotSize; if (slotSize * count != y && index > count - 1) { index = index - 1; } System.out.println(slot + " " + index); } } } ================================================ FILE: src/test/java/io/mycat/route/function/PartitionByDateTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.route.function; import org.junit.Assert; import org.junit.Test; public class PartitionByDateTest { @Test public void test() { PartitionByDate partition=new PartitionByDate(); partition.setDateFormat("yyyy-MM-dd"); partition.setsBeginDate("2014-01-01"); partition.setsPartionDay("10"); partition.init(); Assert.assertEquals(true, 0 == partition.calculate("2014-01-01")); Assert.assertEquals(true, 0 == partition.calculate("2014-01-10")); Assert.assertEquals(true, 1 == partition.calculate("2014-01-11")); Assert.assertEquals(true, 12 == partition.calculate("2014-05-01")); partition.setDateFormat("yyyy-MM-dd"); partition.setsBeginDate("2014-01-01"); partition.setsEndDate("2014-01-31"); partition.setsPartionDay("10"); partition.init(); // // /** // * 0 : 01.01-01.10,02.10-02.19 // * 1 : 01.11-01.20,02.20-03.01 // * 2 : 01.21-01.30,03.02-03.12 // * 3 ： 01.31-02-09,03.13-03.23 // */ Assert.assertEquals(true, 0 == partition.calculate("2014-01-01")); Assert.assertEquals(true, 0 == partition.calculate("2014-01-10")); Assert.assertEquals(true, 1 == partition.calculate("2014-01-11")); Assert.assertEquals(true, 3 == partition.calculate("2014-02-01")); Assert.assertEquals(true, 0 == partition.calculate("2014-02-19")); Assert.assertEquals(true, 1 == partition.calculate("2014-02-20")); Assert.assertEquals(true, 1 == partition.calculate("2014-03-01")); Assert.assertEquals(true, 2 == partition.calculate("2014-03-02")); Assert.assertEquals(true, 2 == partition.calculate("2014-03-11")); Assert.assertEquals(true, 3 == partition.calculate("2014-03-20")); //测试默认1 partition.setDateFormat("yyyy-MM-dd"); partition.setsBeginDate("2014-01-01"); partition.setsEndDate("2014-01-31"); partition.setsPartionDay("1"); partition.init(); Assert.assertEquals(true, 0 == partition.calculate("2014-01-01")); Assert.assertEquals(true, 9 == partition.calculate("2014-01-10")); Assert.assertEquals(true, 10 == partition.calculate("2014-01-11")); Assert.assertEquals(true, 0 == partition.calculate("2014-02-01")); System.out.println(partition.calculate("2014-02-19")); //自然日测试 //1、只开启自然日分表开关 PartitionByDate partition2=new PartitionByDate(); partition2.setDateFormat("yyyy-MM-dd"); partition2.setsNaturalDay("1"); partition2.init(); //Assert.assertEquals(true, 6 == partition2.calculate("2014-01-20")); Assert.assertEquals(true, 19 == partition2.calculate("2014-01-20")); Assert.assertEquals(true, 0 == partition2.calculate("2014-03-01")); Assert.assertEquals(true, 30 == partition2.calculate("2018-03-31")); //2、顺便开启开始时间 partition2.setDateFormat("yyyy-MM-dd"); partition2.setsNaturalDay("1"); partition2.setsPartionDay("1"); partition2.setsBeginDate("2014-01-02"); partition2.init(); Assert.assertEquals(true, 19 == partition2.calculate("2014-01-20")); Assert.assertEquals(true, 0 == partition2.calculate("2014-03-01")); Assert.assertEquals(true, 30 == partition2.calculate("2018-03-31")); //2、顺便开启开始时间,结束时间不足28天（开启自然日失败，默认间隔模式）PartionDay=1 PartitionByDate partition3=new PartitionByDate(); partition3.setDateFormat("yyyy-MM-dd"); partition3.setsNaturalDay("1"); partition3.setsPartionDay("1"); partition3.setsBeginDate("2014-01-02"); partition3.setsEndDate("2014-01-20"); partition3.init(); Assert.assertEquals(true, 0 == partition3.calculate("2014-01-02")); Assert.assertEquals(true, 1 == partition3.calculate("2014-01-03")); Assert.assertEquals(true, 2 == partition3.calculate("2014-01-04")); Assert.assertEquals(true, 6 == partition3.calculate("2014-01-08")); Assert.assertEquals(true, 8 == partition3.calculate("2014-01-10")); Assert.assertEquals(true, 12 == partition3.calculate("2014-01-14")); Assert.assertEquals(true, 18 == partition3.calculate("2014-01-20")); System.out.println(partition3.calculate("2014-03-01")); //Assert.assertEquals(true, 0 == partition3.calculate("2014-03-01")); //3、顺便开启开始时间,结束时间不足28天（开启自然日失败，默认间隔模式）PartionDay=10 恢复间隔模式 partition.setsNaturalDay("1"); partition.setsBeginDate("2014-01-01"); partition.setsEndDate("2014-01-24"); partition.setsPartionDay("10"); partition.init(); Assert.assertEquals(true, 0 == partition.calculate("2014-01-01")); Assert.assertEquals(true, 0 == partition.calculate("2014-01-10")); Assert.assertEquals(true, 0 == partition.calculate("2014-01-05")); Assert.assertEquals(true, 1 == partition.calculate("2014-01-20")); System.out.println("------------success!----"); //4、顺便开启开始时间,结束时间超过29天 PartionDay=1 partition.setsNaturalDay("1"); partition.setsBeginDate("2014-01-01"); partition.setsEndDate("2014-01-29"); partition.setsPartionDay("10"); partition.init(); Assert.assertEquals(true, 0 == partition.calculate("2014-01-01")); Assert.assertEquals(true, 9 == partition.calculate("2014-01-10")); Assert.assertEquals(true, 4 == partition.calculate("2014-01-05")); Assert.assertEquals(true, 19 == partition.calculate("2014-01-20")); Assert.assertEquals(true, 30 == partition.calculate("2018-01-31")); //4、顺便开启开始时间,结束时间超过29天 PartionDay=1 partition.setsNaturalDay("1"); partition.setsBeginDate("2014-01-01"); partition.setsEndDate("2018-01-29"); partition.setsPartionDay("1"); partition.init(); Assert.assertEquals(true, 0 == partition.calculate("2014-01-01")); Assert.assertEquals(true, 9 == partition.calculate("2014-01-10")); Assert.assertEquals(true, 4 == partition.calculate("2014-01-05")); Assert.assertEquals(true, 19 == partition.calculate("2014-01-20")); Assert.assertEquals(true, 30 == partition.calculate("2018-01-31")); System.out.println("------------success!----"); } } ================================================ FILE: src/test/java/io/mycat/route/function/PartitionByHashModTest.java ================================================ package io.mycat.route.function; import junit.framework.Assert; import org.junit.Test; import java.util.Random; import java.util.concurrent.CountDownLatch; /** * 哈希值取模单元测试 * * @author Hash Zhang */ public class PartitionByHashModTest { String allChar = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; @Test public void test() throws InterruptedException { CountDownLatch countDownLatch = new CountDownLatch(1); Task task1 = new Task(countDownLatch,63); Task task2 = new Task(countDownLatch,64); task1.start(); task2.start(); countDownLatch.countDown(); task1.join(); task2.join(); } private class Task extends Thread{ CountDownLatch countDownLatch; int count; public Task(CountDownLatch countDownLatch,int count) { this.countDownLatch = countDownLatch; this.count = count; } @Override public void run() { try { countDownLatch.await(); } catch (InterruptedException e) { e.printStackTrace(); } PartitionByHashMod partitionByHashMod = new PartitionByHashMod(); partitionByHashMod.setCount(count); Random random = new Random(); StringBuffer sb = new StringBuffer(); long start = System.currentTimeMillis(); for (int i = 0; i < 1000000; i++) { for (int j = 0; j < 32; j++) { sb.append(allChar.charAt(random.nextInt(allChar.length()))); } int result = partitionByHashMod.calculate(sb.toString()); sb = new StringBuffer(); Assert.assertTrue(0<=result && result schemaMap; protected LayerCachePool cachePool = new SimpleCachePool(); protected RouteStrategy routeStrategy; public PartitionByRangeDateHashTest() { String schemaFile = "/route/schema.xml"; String ruleFile = "/route/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schemaMap = schemaLoader.getSchemas(); MycatServer.getInstance().getConfig().getSchemas().putAll(schemaMap); RouteStrategyFactory.init(); routeStrategy = RouteStrategyFactory.getRouteStrategy("druidparser"); } @Test public void testRange() throws SQLNonTransientException { String sql = "select * from offer1 where col_date between '2014-01-01 00:00:00' and '2014-01-03 23:59:59' order by id desc limit 100"; SchemaConfig schema = schemaMap.get("TESTDB"); RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); junit.framework.Assert.assertEquals(6, rrs.getNodes().length); sql = "select * from offer1 where col_date between '2014-01-01 00:00:00' and '2014-01-04 00:00:59' order by id desc limit 100"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); junit.framework.Assert.assertEquals(12, rrs.getNodes().length); sql = "select * from offer1 where col_date between '2014-01-04 00:00:00' and '2014-01-06 23:59:59' order by id desc limit 100"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); junit.framework.Assert.assertEquals(6, rrs.getNodes().length); } public static int hash(long str,int size) { return Hashing.consistentHash(str,size) ; } public static void main(String[] args) throws ParseException { Map map=new HashMap<>() ; Date beginDate = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse("2014-01-04 00:00:01"); for (int i = 0; i < 60*60*24*10; i++) { Calendar cal = Calendar.getInstance(); cal.setTime(beginDate); cal.add(Calendar.SECOND, 1); beginDate = cal.getTime(); int hash = hash(beginDate.getTime(), 3); if(map.containsKey(hash)) { map.put(hash, (int)map.get(hash)+1); } else { map.put(hash,1); } // System.out.println(hash); } System.out.println(map.values()); } } ================================================ FILE: src/test/java/io/mycat/route/function/PartitionByRangeModTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.route.function; import junit.framework.Assert; import org.junit.Test; import io.mycat.MycatServer; import io.mycat.SimpleCachePool; import io.mycat.cache.LayerCachePool; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.route.RouteResultset; import io.mycat.route.RouteStrategy; import io.mycat.route.factory.RouteStrategyFactory; import java.math.BigInteger; import java.sql.SQLNonTransientException; import java.util.Map; public class PartitionByRangeModTest { @Test public void test() { PartitionByRangeMod autoPartition = new PartitionByRangeMod(); autoPartition.setMapFile("partition-range-mod.txt"); autoPartition.init(); String idVal = "0"; Assert.assertEquals(true, 0 == autoPartition.calculate(idVal)); idVal = "1"; Assert.assertEquals(true, 1 == autoPartition.calculate(idVal)); idVal = "2"; Assert.assertEquals(true, 2 == autoPartition.calculate(idVal)); idVal = "3"; Assert.assertEquals(true, 3 == autoPartition.calculate(idVal)); idVal = "4"; Assert.assertEquals(true, 4 == autoPartition.calculate(idVal)); idVal = "5"; Assert.assertEquals(true, 0 == autoPartition.calculate(idVal)); idVal="2000000"; Assert.assertEquals(true, 0==autoPartition.calculate(idVal)); idVal="2000001"; Assert.assertEquals(true, 5==autoPartition.calculate(idVal)); idVal="4000000"; Assert.assertEquals(true, 5==autoPartition.calculate(idVal)); idVal="4000001"; Assert.assertEquals(true, 7==autoPartition.calculate(idVal)); } private static int mod(long v, int size) { BigInteger bigNum = BigInteger.valueOf(v).abs(); return (bigNum.mod(BigInteger.valueOf(size))).intValue(); } protected Map schemaMap; protected LayerCachePool cachePool = new SimpleCachePool(); protected RouteStrategy routeStrategy; public PartitionByRangeModTest() { String schemaFile = "/route/schema.xml"; String ruleFile = "/route/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schemaMap = schemaLoader.getSchemas(); MycatServer.getInstance().getConfig().getSchemas().putAll(schemaMap); RouteStrategyFactory.init(); routeStrategy = RouteStrategyFactory.getRouteStrategy("druidparser"); } @Test public void testRange() throws SQLNonTransientException { String sql = "select * from offer where id between 2000000 and 4000001 order by id desc limit 100"; SchemaConfig schema = schemaMap.get("TESTDB"); RouteResultset rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(10, rrs.getNodes().length); sql = "select * from offer where id between 9 and 2000 order by id desc limit 100"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(5, rrs.getNodes().length); sql = "select * from offer where id between 4000001 and 6005001 order by id desc limit 100"; rrs = routeStrategy.route(new SystemConfig(), schema, -1, sql, null, null, cachePool); Assert.assertEquals(8, rrs.getNodes().length); } } ================================================ FILE: src/test/java/io/mycat/route/function/PartitionByStringTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.route.function; import org.junit.Assert; import org.junit.Test; import io.mycat.route.function.PartitionByString; public class PartitionByStringTest { @Test public void test() { PartitionByString rule = new PartitionByString(); String idVal=null; rule.setPartitionLength("512"); rule.setPartitionCount("2"); rule.init(); rule.setHashSlice("0:2"); // idVal = "0"; // Assert.assertEquals(true, 0 == rule.calculate(idVal)); // idVal = "45a"; // Assert.assertEquals(true, 1 == rule.calculate(idVal)); //last 4 rule = new PartitionByString(); rule.setPartitionLength("512"); rule.setPartitionCount("2"); rule.init(); //last 4 characters rule.setHashSlice("-4:0"); idVal = "aaaabbb0000"; Assert.assertEquals(true, 0 == rule.calculate(idVal)); idVal = "aaaabbb2359"; Assert.assertEquals(true, 0 == rule.calculate(idVal)); } } ================================================ FILE: src/test/java/io/mycat/route/function/RuleFunctionSuitTableTest.java ================================================ package io.mycat.route.function; import java.util.Arrays; import org.junit.Assert; import org.junit.Test; import io.mycat.config.model.TableConfig; import io.mycat.config.model.rule.RuleConfig; import io.mycat.util.SplitUtil; /** * 测试分片算法定义是否符合分片表的定义, 主要测试分区数是否符合分片表分片数 * * @author CrazyPig * */ public class RuleFunctionSuitTableTest { @Test public void testAutoPartitionByLong() { AutoPartitionByLong autoPartition=new AutoPartitionByLong(); autoPartition.setMapFile("autopartition-long.txt"); autoPartition.init(); // partition = 3 Assert.assertEquals(3, autoPartition.getPartitionNum()); RuleConfig rule = new RuleConfig("id", "auto-partition-long"); rule.setRuleAlgorithm(autoPartition); TableConfig tableConf = new TableConfig("test", "id", true, false, -1, "dn1,dn2", null, rule, true, null, false, null, null, null, false); int suit1 = autoPartition.suitableFor(tableConf); Assert.assertEquals(-1, suit1); tableConf.getDataNodes().clear(); tableConf.getDataNodes().addAll(Arrays.asList("dn1", "dn2", "dn3")); int suit2 = autoPartition.suitableFor(tableConf); Assert.assertEquals(0, suit2); tableConf.getDataNodes().clear(); tableConf.getDataNodes().addAll(Arrays.asList("dn1", "dn2", "dn3", "dn4")); int suit3 = autoPartition.suitableFor(tableConf); Assert.assertEquals(1, suit3); /* * autopartition-long-dupl.txt * 0-1000=0 * 1001-2000=1 * 2001-3000=0 * 3001-4000=1 */ AutoPartitionByLong autoPartition2 = new AutoPartitionByLong(); autoPartition2.setMapFile("autopartition-long-dupl.txt"); autoPartition2.init(); Assert.assertEquals(2, autoPartition2.getPartitionNum()); RuleConfig rule2 = new RuleConfig("id", "auto-partition-long-dupl"); rule2.setRuleAlgorithm(autoPartition2); TableConfig tableConf2 = new TableConfig("test2", "id", true, false, -1, "dn1,dn2", null, rule, true, null, false, null, null, null,false); Assert.assertEquals(0, autoPartition2.suitableFor(tableConf2)); Assert.assertEquals(0, autoPartition2.calculate("500").intValue()); Assert.assertEquals(1, autoPartition2.calculate("1500").intValue()); Assert.assertEquals(1, autoPartition2.calculate("2000").intValue()); Assert.assertEquals(0, autoPartition2.calculate("3000").intValue()); Assert.assertEquals(1, autoPartition2.calculate("3001").intValue()); } @Test public void testPartitionByDate() { PartitionByDate partition = new PartitionByDate(); partition.setDateFormat("yyyy-MM-dd"); partition.setsBeginDate("2014-01-01"); partition.setsEndDate("2014-01-31"); partition.setsPartionDay("10"); partition.init(); // partition = 4 Assert.assertEquals(4, partition.getPartitionNum()); RuleConfig rule = new RuleConfig("col_date", "partition-date"); rule.setRuleAlgorithm(partition); TableConfig tableConf = new TableConfig("test", "id", true, false, -1, "dn1,dn2,dn3", null, rule, true, null, false, null, null, null, false); int suit1 = partition.suitableFor(tableConf); Assert.assertEquals(-1, suit1); tableConf.getDataNodes().clear(); tableConf.getDataNodes().addAll(Arrays.asList("dn1", "dn2", "dn3", "dn4")); int suit2 = partition.suitableFor(tableConf); Assert.assertEquals(0, suit2); tableConf.getDataNodes().clear(); tableConf.getDataNodes().addAll(Arrays.asList("dn1", "dn2", "dn3", "dn4", "dn5")); int suit3 = partition.suitableFor(tableConf); Assert.assertEquals(1, suit3); PartitionByDate partition1 = new PartitionByDate(); partition.setDateFormat("yyyy-MM-dd"); partition.setsBeginDate("2014-01-01"); partition.setsPartionDay("10"); partition.init(); // partition no limit int suit4 = partition1.suitableFor(tableConf); Assert.assertEquals(0, suit4); } @Test public void testPartitionByHashMod() { PartitionByHashMod partition = new PartitionByHashMod(); partition.setCount(3); // partition = 3; Assert.assertEquals(3, partition.getPartitionNum()); RuleConfig rule = new RuleConfig("id", "partition-hash-mod"); rule.setRuleAlgorithm(partition); TableConfig tableConf = new TableConfig("test", "id", true, false, -1, "dn1,dn2,dn3", null, rule, true, null, false, null, null, null, false); int suit1 = partition.suitableFor(tableConf); Assert.assertEquals(0, suit1); tableConf.getDataNodes().clear(); tableConf.getDataNodes().addAll(Arrays.asList("dn1", "dn2", "dn3", "dn4")); int suit2 = partition.suitableFor(tableConf); Assert.assertEquals(1, suit2); tableConf.getDataNodes().clear(); tableConf.getDataNodes().addAll(Arrays.asList("dn1", "dn2")); int suit3 = partition.suitableFor(tableConf); Assert.assertEquals(-1, suit3); } @Test public void testPartitionByRangeMod() { PartitionByRangeMod partition = new PartitionByRangeMod(); partition.setMapFile("partition-range-mod.txt"); partition.init(); Assert.assertEquals(20, partition.getPartitionNum()); // partition = 20 RuleConfig rule = new RuleConfig("id", "partition-range-mod"); rule.setRuleAlgorithm(partition); TableConfig tableConf = new TableConfig("test", "id", true, false, -1, "dn$1-10", null, rule, true, null, false, null, null, null, false); int suit1 = partition.suitableFor(tableConf); Assert.assertEquals(-1, suit1); tableConf.getDataNodes().clear(); String[] dataNodes = SplitUtil.split("dn$1-20", ',', '$', '-'); tableConf.getDataNodes().addAll(Arrays.asList(dataNodes)); int suit2 = partition.suitableFor(tableConf); Assert.assertEquals(0, suit2); tableConf.getDataNodes().clear(); dataNodes = SplitUtil.split("dn$1-30", ',', '$', '-'); tableConf.getDataNodes().addAll(Arrays.asList(dataNodes)); int suit3 = partition.suitableFor(tableConf); Assert.assertEquals(1, suit3); } @Test public void testPartitionByPattern() { PartitionByPattern partition = new PartitionByPattern(); partition.setMapFile("partition-pattern.txt"); partition.init(); /* * partition-pattern.txt * 1-32=0 * 33-64=1 * 65-96=2 * 97-128=3 * 129-160=4 * 161-192=5 * 193-224=6 * 225-256=7 * 0-0=7 */ Assert.assertEquals(8, partition.getPartitionNum()); } @Test public void testPartitionByPrefixPattern() { PartitionByPrefixPattern partition = new PartitionByPrefixPattern(); partition.setMapFile("partition_prefix_pattern.txt"); partition.init(); /* * partition_prefix_pattern.txt * 1-4=0 * 5-8=1 * 9-12=2 * 13-16=3 * 17-20=4 * 21-24=5 * 25-28=6 * 29-32=7 * 0-0=7 */ Assert.assertEquals(8, partition.getPartitionNum()); RuleConfig rule = new RuleConfig("id", "partition-prefix-pattern"); rule.setRuleAlgorithm(partition); TableConfig tableConf = new TableConfig("test", "id", true, false, -1, "dn1,dn2", null, rule, true, null, false, null, null, null, false); int suit1 = partition.suitableFor(tableConf); Assert.assertEquals(-1, suit1); tableConf.getDataNodes().clear(); String[] dataNodes = SplitUtil.split("dn$1-8", ',', '$', '-'); tableConf.getDataNodes().addAll(Arrays.asList(dataNodes)); int suit2 = partition.suitableFor(tableConf); Assert.assertEquals(0, suit2); tableConf.getDataNodes().clear(); dataNodes = SplitUtil.split("dn$1-10", ',', '$', '-'); tableConf.getDataNodes().addAll(Arrays.asList(dataNodes)); int suit3 = partition.suitableFor(tableConf); Assert.assertEquals(1, suit3); } } ================================================ FILE: src/test/java/io/mycat/route/function/TestLatestMonthPartion.java ================================================ package io.mycat.route.function; import static org.junit.Assert.assertTrue; import org.junit.Test; public class TestLatestMonthPartion { @Test public void testSetDataNodes() { LatestMonthPartion partion = new LatestMonthPartion(); partion.setSplitOneDay(24); Integer val = partion.calculate("2015020100"); assertTrue(val == 0); val = partion.calculate("2015020216"); assertTrue(val == 40); val = partion.calculate("2015022823"); assertTrue(val == 27 * 24 + 23); Integer[] span = partion.calculateRange("2015020100", "2015022823"); assertTrue(span.length == 27 * 24 + 23 + 1); assertTrue(span[0] == 0 && span[span.length - 1] == 27 * 24 + 23); span = partion.calculateRange("2015020100", "2015020123"); assertTrue(span.length == 24); assertTrue(span[0] == 0 && span[span.length - 1] == 23); } } ================================================ FILE: src/test/java/io/mycat/route/function/TestNumberParseUtil.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.route.function; import org.junit.Test; import io.mycat.route.function.NumberParseUtil; import junit.framework.Assert; public class TestNumberParseUtil { @Test public void test() { String val = "2000"; Assert.assertEquals(2000, NumberParseUtil.parseLong(val)); val = "2M"; Assert.assertEquals(20000, NumberParseUtil.parseLong(val)); val = "2M1"; Assert.assertEquals(20001, NumberParseUtil.parseLong(val)); val = "1000M"; Assert.assertEquals(10000000, NumberParseUtil.parseLong(val)); val = "30K"; Assert.assertEquals(30000, NumberParseUtil.parseLong(val)); val = "30K1"; Assert.assertEquals(30001, NumberParseUtil.parseLong(val)); val = "30K09"; Assert.assertEquals(30009, NumberParseUtil.parseLong(val)); } } ================================================ FILE: src/test/java/io/mycat/route/parser/druid/impl/DefaultDruidParserTest.java ================================================ package io.mycat.route.parser.druid.impl; import static org.junit.Assert.assertArrayEquals; import static org.mockito.Mockito.mock; import org.junit.Before; import org.junit.Test; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; import com.alibaba.druid.sql.parser.SQLStatementParser; import io.mycat.cache.LayerCachePool; import io.mycat.config.model.SchemaConfig; import io.mycat.route.RouteResultset; import io.mycat.route.parser.druid.DruidParser; import io.mycat.route.parser.druid.DruidShardingParseInfo; import io.mycat.route.parser.druid.MycatSchemaStatVisitor; import io.mycat.server.parser.ServerParse; /** * sql解析单元测试 * @author lian * @date 2016年12月2日 */ public class DefaultDruidParserTest { private SchemaConfig schema; private DruidParser druidParser; @Before public void setUp(){ schema = mock(SchemaConfig.class); druidParser = new DefaultDruidParser(); } @Test public void testParser() throws Exception { assertArrayEquals(getParseTables("select id as id from company t;"), getArr("company".toUpperCase())); assertArrayEquals(getParseTables("select 1 from (select 1 from company) company;"), getArr("company".toUpperCase())); assertArrayEquals(getParseTables("select 1 from company,customer where company.id = customer.cid"), getArr("company".toUpperCase(),"customer".toUpperCase())); assertArrayEquals(getParseTables("select 1 from db1.company,db1.customer where company.id = customer.cid"), getArr("db1.company".toUpperCase(), "db1.customer".toUpperCase())); assertArrayEquals(getParseTables("select 1 from mysql.company,db1.customer where company.id = customer.cid"), getArr("db1.customer".toUpperCase())); } private Object[] getParseTables(String sql) throws Exception{ SQLStatementParser parser = new MySqlStatementParser(sql); SQLStatement statement = parser.parseStatement(); MycatSchemaStatVisitor visitor = new MycatSchemaStatVisitor(); LayerCachePool cachePool = mock(LayerCachePool.class); RouteResultset rrs = new RouteResultset(sql, ServerParse.SELECT); druidParser.parser(schema, rrs, statement, sql, cachePool, visitor); DruidShardingParseInfo ctx = druidParser.getCtx(); return ctx.getTables().toArray(); } private Object[] getArr(String...strings){ return strings; } } ================================================ FILE: src/test/java/io/mycat/route/perf/NoShardingSpace.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.route.perf; import java.sql.SQLNonTransientException; import io.mycat.SimpleCachePool; import io.mycat.cache.LayerCachePool; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.route.factory.RouteStrategyFactory; /** * @author mycat */ public class NoShardingSpace { private SchemaConfig schema; private static int total=1000000; protected LayerCachePool cachePool = new SimpleCachePool(); public NoShardingSpace() { String schemaFile = "/route/schema.xml"; String ruleFile = "/route/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schema = schemaLoader.getSchemas().get("dubbo"); } public void testDefaultSpace() throws SQLNonTransientException { SchemaConfig schema = this.schema; String stmt = "insert into offer (member_id, gmt_create) values ('1','2001-09-13 20:20:33')"; for (int i = 0; i < total; i++) { RouteStrategyFactory.getRouteStrategy().route(new SystemConfig(),schema, -1,stmt, null, null,cachePool); } } public static void main(String[] args) throws SQLNonTransientException { NoShardingSpace test = new NoShardingSpace(); System.currentTimeMillis(); long start = System.currentTimeMillis(); test.testDefaultSpace(); long end = System.currentTimeMillis(); System.out.println("take " + (end - start) + " ms. avg "+(end-start+0.0)/total); } } ================================================ FILE: src/test/java/io/mycat/route/perf/ShardingDefaultSpace.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.route.perf; import java.sql.SQLNonTransientException; import io.mycat.SimpleCachePool; import io.mycat.cache.LayerCachePool; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.route.factory.RouteStrategyFactory; /** * @author mycat */ public class ShardingDefaultSpace { private SchemaConfig schema; private static int total=1000000; protected LayerCachePool cachePool = new SimpleCachePool(); public ShardingDefaultSpace() throws InterruptedException { String schemaFile = "/route/schema.xml"; String ruleFile = "/route/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schema = schemaLoader.getSchemas().get("cndb"); } /** * 路由到defaultSpace的性能测试 */ public void testDefaultSpace() throws SQLNonTransientException { SchemaConfig schema = this.getSchema(); String sql = "insert into offer (member_id, gmt_create) values ('1','2001-09-13 20:20:33')"; for (int i = 0; i < total; i++) { RouteStrategyFactory.getRouteStrategy().route(new SystemConfig(),schema,-1, sql, null, null,cachePool); } } protected SchemaConfig getSchema() { return schema; } public static void main(String[] args) throws Exception { ShardingDefaultSpace test = new ShardingDefaultSpace(); System.currentTimeMillis(); long start = System.currentTimeMillis(); test.testDefaultSpace(); long end = System.currentTimeMillis(); System.out.println("take " + (end - start) + " ms. avg "+(end-start+0.0)/total); } } ================================================ FILE: src/test/java/io/mycat/route/perf/ShardingMultiTableSpace.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.route.perf; import java.sql.SQLNonTransientException; import io.mycat.SimpleCachePool; import io.mycat.cache.LayerCachePool; import io.mycat.config.loader.SchemaLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; import io.mycat.route.factory.RouteStrategyFactory; /** * @author mycat */ public class ShardingMultiTableSpace { private SchemaConfig schema; private static int total=1000000; protected LayerCachePool cachePool = new SimpleCachePool(); public ShardingMultiTableSpace() throws InterruptedException { String schemaFile = "/route/schema.xml"; String ruleFile = "/route/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schema = schemaLoader.getSchemas().get("cndb"); } /** * 路由到tableSpace的性能测试 * * @throws SQLNonTransientException */ public void testTableSpace() throws SQLNonTransientException { SchemaConfig schema = getSchema(); String sql = "select id,member_id,gmt_create from offer where member_id in ('1','22','333','1124','4525')"; for (int i = 0; i < total; i++) { RouteStrategyFactory.getRouteStrategy().route(new SystemConfig(),schema, -1,sql, null, null,cachePool); } } protected SchemaConfig getSchema() { return schema; } public static void main(String[] args) throws Exception { ShardingMultiTableSpace test = new ShardingMultiTableSpace(); System.currentTimeMillis(); long start = System.currentTimeMillis(); test.testTableSpace(); long end = System.currentTimeMillis(); System.out.println("take " + (end - start) + " ms. avg "+(end-start+0.0)/total); } } ================================================ FILE: src/test/java/io/mycat/route/util/PartitionForSingle.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.route.util; /** * 数据分区工具单独版本，请使用singleton的模式调用。 * * @author mycat */ public final class PartitionForSingle { // 分区长度:数据段分布定义，其中取模的数一定要是2^n， 因为这里使用x % 2^n == x & (2^n - 1)等式，来优化性能。 private static final int PARTITION_LENGTH = 1024; private static final int DEFAULT_HASH_LENGTH = 8; // %转换为&操作的换算数值 private static final long AND_VALUE = PARTITION_LENGTH - 1; // 分区线段 private final int[] segment = new int[PARTITION_LENGTH]; /** * @param count * 表示定义的分区数 * @param length * 表示对应每个分区的取值长度 *

* 注意：其中count,length两个数组的长度必须是一致的。 *

*/ public PartitionForSingle(int[] count, int[] length) { if (count == null || length == null || (count.length != length.length)) { throw new RuntimeException("error,check your scope & scopeLength definition."); } int segmentLength = 0; for (int i = 0; i < count.length; i++) { segmentLength += count[i]; } int[] scopeSegment = new int[segmentLength + 1]; int index = 0; for (int i = 0; i < count.length; i++) { for (int j = 0; j < count[i]; j++) { scopeSegment[++index] = scopeSegment[index - 1] + length[i]; } } if (scopeSegment[scopeSegment.length - 1] != PARTITION_LENGTH) { throw new RuntimeException("error,check your partitionScope definition."); } // 数据映射操作 for (int i = 1; i < scopeSegment.length; i++) { for (int j = scopeSegment[i - 1]; j < scopeSegment[i]; j++) { segment[j] = (i - 1); } } } public int partition(long h) { return segment[(int) (h & AND_VALUE)]; } public int partition(String key) { return segment[(int) (hash(key) & AND_VALUE)]; } private static long hash(String s) { long h = 0; int len = s.length(); for (int i = 0; (i < DEFAULT_HASH_LENGTH && i < len); i++) { h = (h << 5) - h + s.charAt(i); } return h; } // for test public static void main(String[] args) { // 拆分为16份，每份长度均为：64。 // Scope scope = new Scope(new int[] { 16 }, new int[] { 64 }); // // 拆分为23份，前8份长度为：8，后15份长度为：64。 // Scope scope = new Scope(new int[] { 8, 15 }, new int[] { 8, 64 }); // // 拆分为128份，每份长度均为：8。 // Scope scope = new Scope(new int[] { 128 }, new int[] { 8 }); PartitionForSingle p = new PartitionForSingle(new int[] { 8, 15 }, new int[] { 8, 64 }); String memberId = "xianmao.hexm"; int value = 0; long st = System.currentTimeMillis(); for (int i = 0; i < 10000000; i++) { value = p.partition(memberId); } long et = System.currentTimeMillis(); System.out.println("value:" + value + ",take time:" + (et - st) + " ms."); } } ================================================ FILE: src/test/java/io/mycat/route/util/PartitionUtilTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.route.util; import org.junit.Assert; import org.junit.Test; import io.mycat.route.util.PartitionUtil; /** * @author mycat() */ public class PartitionUtilTest { @Test public void testPartition() { // 本例的分区策略：希望将数据水平分成3份，前两份各占25%，第三份占50%。（故本例非均匀分区） // |<---------------------1024------------------------>| // |<----256--->|<----256--->|<----------512---------->| // | partition0 | partition1 | partition2 | // | 共2份,故count[0]=2 | 共1份，故count[1]=1 | int[] count = new int[] { 2, 1 }; int[] length = new int[] { 256, 512 }; PartitionUtil pu = new PartitionUtil(count, length); // 下面代码演示分别以offerId字段或memberId字段根据上述分区策略拆分的分配结果 int DEFAULT_STR_HEAD_LEN = 8; // cobar默认会配置为此值 long offerId = 12345; String memberId = "qiushuo"; // 若根据offerId分配，partNo1将等于0，即按照上述分区策略，offerId为12345时将会被分配到partition0中 int partNo1 = pu.partition(offerId); // 若根据memberId分配，partNo2将等于2，即按照上述分区策略，memberId为qiushuo时将会被分到partition2中 int partNo2 = pu.partition(memberId, 0, DEFAULT_STR_HEAD_LEN); Assert.assertEquals(0, partNo1); Assert.assertEquals(2, partNo2); } } ================================================ FILE: src/test/java/io/mycat/route/util/RouterUtilTest.java ================================================ package io.mycat.route.util; import io.mycat.util.StringUtil; import org.junit.Assert; import org.junit.Test; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; /** * @author Hash Zhang * @version 1.0.0 * @date 2016/7/19 */ public class RouterUtilTest { public static void main(String[] args) { String sql = "insert into hotnews(title,name) values('test1',\"name\"),('(test)',\"(test)\"),('\\\"',\"\\'\"),(\")\",\"\\\"\\')\");"; List values = RouterUtil.handleBatchInsert(sql, sql.toUpperCase().indexOf("VALUES")); System.out.println(values); } @Test public void testBatchInsert() { String sql = "insert into hotnews(title,name) values('test1',\"name\"),('(test)',\"(test)\"),('\\\"',\"\\'\"),(\")\",\"\\\"\\')\");"; List values = RouterUtil.handleBatchInsert(sql, sql.toUpperCase().indexOf("VALUES")); Assert.assertTrue(values.get(0).equals("insert into hotnews(title,name) values('test1',\"name\")")); Assert.assertTrue(values.get(1).equals("insert into hotnews(title,name) values('(test)',\"(test)\")")); Assert.assertTrue(values.get(2).equals("insert into hotnews(title,name) values('\\\"',\"\\'\")")); Assert.assertTrue(values.get(3).equals("insert into hotnews(title,name) values(\")\",\"\\\"\\')\")")); } @Test public void testParseSqlValueArrayAndSuffixStr() { String sql = "insert into hotnews(title,name) values('test1',\"name\"),('(test)',\"(test)\"),('\\\"',\"\\'\"),(\")\",\"\\\"\\')\"),(left(upper('test'), 2),\"left(upper('test'), 2)\") on duplicate key update name = values(name)"; Object[] valueArrayAndSuffixStr = RouterUtil.parseSqlValueArrayAndSuffixStr(sql, sql.toUpperCase().indexOf("VALUES")); Assert.assertTrue(valueArrayAndSuffixStr.length == 2); List> valueArray = (List>) valueArrayAndSuffixStr[0]; String suffixStr = (String) valueArrayAndSuffixStr[1]; Assert.assertTrue(valueArray.size() == 5); Assert.assertTrue(valueArray.get(0).equals(new ArrayList<>(Arrays.asList("'test1'", "\"name\"")))); Assert.assertTrue(valueArray.get(1).equals(new ArrayList<>(Arrays.asList("'(test)'", "\"(test)\"")))); Assert.assertTrue(valueArray.get(2).equals(new ArrayList<>(Arrays.asList("'\\\"'", "\"\\'\"")))); Assert.assertTrue(valueArray.get(3).equals(new ArrayList<>(Arrays.asList("\")\"", "\"\\\"\\')\"")))); Assert.assertTrue(valueArray.get(4).equals(new ArrayList<>(Arrays.asList("left(upper('test'), 2)", "\"left(upper('test'), 2)\"")))); Assert.assertTrue(suffixStr.equals("on duplicate key update name = values(name)")); } @Test public void testRemoveSchema() { String sql = "update test set name='abcdtestx.aa' where id=1 and testx=123"; String afterAql= RouterUtil.removeSchema(sql,"testx"); Assert.assertEquals(sql,afterAql); System.out.println(afterAql); } @Test public void testRemoveSchemaSelect() { String sql = "select id as 'aa' from test where name='abcdtestx.aa' and id=1 and testx=123"; String afterAql= RouterUtil.removeSchema(sql,"testx"); Assert.assertEquals(sql,afterAql); } @Test public void testRemoveSchemaSelect2() { String sql = "select id as 'aa' from testx.test where name='abcd testx.aa' and id=1 and testx=123"; String afterAql= RouterUtil.removeSchema(sql,"testx"); Assert.assertNotSame(sql.indexOf("testx."),afterAql.indexOf("testx.")); } @Test public void testRemoveSchema2(){ String sql = "update testx.test set name='abcd \\' testx.aa' where id=1"; String sqltrue = "update test set name='abcd \\' testx.aa' where id=1"; String sqlnew = RouterUtil.removeSchema(sql, "testx"); Assert.assertEquals("处理错误：", sqltrue, sqlnew); } @Test public void testRemoveSchema3(){ String sql = "update testx.test set testx.name='abcd testx.aa' where testx.id=1"; String sqltrue = "update test set name='abcd testx.aa' where id=1"; String sqlnew = RouterUtil.removeSchema(sql, "testx"); Assert.assertEquals("处理错误：", sqltrue, sqlnew); } @Test public void testRemoveSchema4(){ String sql = "update testx.test set testx.name='abcd testx.aa' and testx.name2='abcd testx.aa' where testx.id=1"; String sqltrue = "update test set name='abcd testx.aa' and name2='abcd testx.aa' where id=1"; String sqlnew = RouterUtil.removeSchema(sql, "testx"); Assert.assertEquals("处理错误：", sqltrue, sqlnew); } /** * @modification 修改支持createTable语句中包含“IF NOT EXISTS”的情况,这里测试下 * @date 2016/12/8 * @modifiedBy Hash Zhang */ @Test public void testGetCreateTableStmtTableName(){ String sql1 = StringUtil.makeString("create table if not exists producer(\n", "\tid int(11) primary key,\n", "\tname varchar(32)\n", ");").toUpperCase(); String sql2 = StringUtil.makeString("create table good(\n", "\tid int(11) primary key,\n", "\tcontent varchar(32),\n", "\tproducer_id int(11) key\n", ");").toUpperCase(); Assert.assertTrue("producer".equalsIgnoreCase(RouterUtil.getTableName(sql1, RouterUtil.getCreateTablePos(sql1, 0)))); Assert.assertTrue("good".equalsIgnoreCase(RouterUtil.getTableName(sql2, RouterUtil.getCreateTablePos(sql2, 0)))); } /** * @modification 针对修改RouterUtil的去除schema的方法支持` 进行测试 * @date 2016/12/29 * @modifiedBy Hash Zhang */ @Test public void testRemoveSchemaWithHypha(){ String sql1 = StringUtil.makeString("select `testdb`.`orders`.`id`, `testdb`.`orders`.`customer_id`, `testdb`.`orders`.`goods_id` from `testdb`.`orders` where testdb.`orders`.`id` = 1;").toUpperCase(); String sql2 = StringUtil.makeString("select `testdb`.`orders`.`id`, testdb.`orders`.`customer_id`, `testdb`.`orders`.`goods_id` from testdb.`orders` where `testdb`.`orders`.`id` = 1;").toUpperCase(); String sql3 = StringUtil.makeString("select testdb.`orders`.`id`, `testdb`.`orders`.`customer_id`, testdb.`orders`.`goods_id` from `testdb`.`orders` where testdb.`orders`.`id` = 1;").toUpperCase(); String sql4 = StringUtil.makeString("select testdb.`orders`.`id`, testdb.`orders`.`customer_id`, testdb.`orders`.`goods_id` from testdb.`orders` where testdb.`orders`.`id` = 1;").toUpperCase(); String result = "SELECT `ORDERS`.`ID`, `ORDERS`.`CUSTOMER_ID`, `ORDERS`.`GOODS_ID` FROM `ORDERS` WHERE `ORDERS`.`ID` = 1;"; Assert.assertTrue(result.equals(RouterUtil.removeSchema(sql1,"testdb"))); Assert.assertTrue(result.equals(RouterUtil.removeSchema(sql2,"testdb"))); Assert.assertTrue(result.equals(RouterUtil.removeSchema(sql3,"testdb"))); Assert.assertTrue(result.equals(RouterUtil.removeSchema(sql4,"testdb"))); } } ================================================ FILE: src/test/java/io/mycat/sequence/DistributedSequenceHandlerTest.java ================================================ package io.mycat.sequence; import io.mycat.config.MycatConfig; import io.mycat.route.sequence.handler.DistributedSequenceHandler; import junit.framework.Assert; import org.apache.curator.test.TestingServer; import org.junit.Before; import org.junit.Test; import java.io.IOException; import java.util.HashSet; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; /** * 基于ZK与本地配置的分布式ID生成器 * 无悲观锁，吞吐量更高 * * @author Hash Zhang * @version 1.0 * @time 00:12:05 2016/5/3 */ public class DistributedSequenceHandlerTest { TestingServer testingServer = null; DistributedSequenceHandler distributedSequenceHandler[]; @Before public void initialize() throws Exception { distributedSequenceHandler = new DistributedSequenceHandler[16]; MycatConfig mycatConfig = new MycatConfig(); testingServer = new TestingServer(); testingServer.start(); for (int i = 0; i < 16; i++) { distributedSequenceHandler[i] = new DistributedSequenceHandler(mycatConfig.getSystem()); distributedSequenceHandler[i].initializeZK(testingServer.getConnectString()); distributedSequenceHandler[i].nextId(""); } } /** * 测试获取的唯一InstanceId * * @throws Exception */ @Test public void testUniqueInstanceID() throws Exception { Set idSet = new HashSet<>(); for (int i = 0; i < 16; i++) { idSet.add(distributedSequenceHandler[i].getInstanceId()); } Assert.assertEquals(idSet.size(), 16); } /** * 测试获取的唯一id * * @throws Exception */ @Test public void testUniqueID() throws Exception { final ConcurrentHashMap idSet = new ConcurrentHashMap<>(); Thread thread[] = new Thread[10]; long start = System.currentTimeMillis(); //多少线程，注意线程数不能超过最大线程数（1< idSet = new HashSet<>(); try { int leader = failLeader(17); System.out.println("***断掉一个leader节点后（curator会抛对应的异常断链异常，不用在意）***："); for (int i = 0; i < 16; i++) { if (i == leader) { System.out.println("Node [" + i + "] used to be leader"); continue; } distributedSequenceHandler[i].nextId(""); System.out.println("Node [" + i + "]is leader:" + distributedSequenceHandler[i].getLeaderSelector().hasLeadership() ); System.out.println(" InstanceID:" + distributedSequenceHandler[i].getInstanceId()); idSet.add(distributedSequenceHandler[i].getInstanceId()); } Assert.assertEquals(idSet.size(), 15); idSet = new HashSet<>(); int leader2 = failLeader(leader); System.out.println("***断掉两个leader节点后（curator会抛对应的异常断链异常，不用在意）***："); for (int i = 0; i < 16; i++) { if (i == leader || i == leader2) { System.out.println("Node ["+i + " used to be leader"); continue; } distributedSequenceHandler[i].nextId(""); System.out.println("Node ["+i + "]is leader:" + distributedSequenceHandler[i].getLeaderSelector().hasLeadership()); System.out.println(" InstanceID:" + distributedSequenceHandler[i].getInstanceId()); idSet.add(distributedSequenceHandler[i].getInstanceId()); } Assert.assertEquals(idSet.size(), 14); idSet = new HashSet<>(); MycatConfig mycatConfig = new MycatConfig(); distributedSequenceHandler[leader] = new DistributedSequenceHandler(mycatConfig.getSystem()); distributedSequenceHandler[leader].initializeZK(testingServer.getConnectString()); distributedSequenceHandler[leader].nextId(""); distributedSequenceHandler[leader2] = new DistributedSequenceHandler(mycatConfig.getSystem()); distributedSequenceHandler[leader2].initializeZK(testingServer.getConnectString()); distributedSequenceHandler[leader2].nextId(""); System.out.println("新加入两个节点后"); for (int i = 0; i < 16; i++) { System.out.println("Node ["+i + "]is leader:" + distributedSequenceHandler[i].getLeaderSelector().hasLeadership() ); System.out.println(" InstanceID:" + distributedSequenceHandler[i].getInstanceId()); idSet.add(distributedSequenceHandler[i].getInstanceId()); } } catch (Exception e) { } finally { Assert.assertEquals(idSet.size(), 16); } } private int failLeader(int p) { int leader = 0, follower = 0; for (int i = 0; i < 16; i++) { if (i == p) { continue; } if (distributedSequenceHandler[i].getLeaderSelector().hasLeadership()) { leader = i; } else { follower = i; } System.out.println("Node ["+i + "]is leader:" + distributedSequenceHandler[i].getLeaderSelector().hasLeadership() ); System.out.println(" InstanceID:" + distributedSequenceHandler[i].getInstanceId()); } try { distributedSequenceHandler[leader].close(); } catch (IOException e) { } while (true) { follower++; if (follower >= 16) { follower = 0; } if (follower == leader || follower == p) { continue; } if (distributedSequenceHandler[follower].getLeaderSelector().hasLeadership()) { break; } } return leader; } } ================================================ FILE: src/test/java/io/mycat/sequence/IncrSequenceZKHandlerTest.java ================================================ package io.mycat.sequence; import io.mycat.route.sequence.handler.IncrSequenceZKHandler; import io.mycat.route.util.PropertiesUtil; import junit.framework.Assert; import org.apache.curator.test.TestingServer; import org.junit.Before; import org.junit.Test; import java.util.Properties; import java.util.concurrent.ConcurrentSkipListSet; /** * zookeeper 实现递增序列号 * 默认测试模拟60个进程，每个进程内20个线程。每个线程调用50次参数为GLOBAL的nextid * 默认GLOBAL.MINID=1 * 默认GLOBAL.MAXID=10 * 表示当前线程内id用光时，每次会取GLOBAL.MINID-GLOBAL.MAXID9个ID * * @author Hash Zhang * @version 1.0 * @time 23:35 2016/5/6 */ public class IncrSequenceZKHandlerTest { private final static int MAX_CONNECTION = 5; private final static int threadCount = 5; private final static int LOOP = 5; TestingServer testingServer = null; IncrSequenceZKHandler incrSequenceZKHandler[]; ConcurrentSkipListSet results; @Before public void initialize() throws Exception { testingServer = new TestingServer(); testingServer.start(); incrSequenceZKHandler = new IncrSequenceZKHandler[MAX_CONNECTION]; results = new ConcurrentSkipListSet(); } @Test public void testCorrectnessAndEfficiency() throws InterruptedException { final Thread threads[] = new Thread[MAX_CONNECTION]; for (int i = 0; i < MAX_CONNECTION; i++) { final int a = i; threads[i] = new Thread() { @Override public void run() { incrSequenceZKHandler[a] = new IncrSequenceZKHandler(); Properties props = PropertiesUtil.loadProps("sequence_conf.properties"); try { incrSequenceZKHandler[a].initializeZK(props, testingServer.getConnectString()); } catch (Exception e) { e.printStackTrace(); } Thread threads[] = new Thread[threadCount]; for (int j = 0; j < threadCount; j++) { threads[j] = new Thread() { @Override public void run() { for (int k = 0; k < LOOP; k++) { long key = incrSequenceZKHandler[a].nextId("GLOBAL"); results.add(key); } } }; threads[j].start(); } for (int j = 0; j < threadCount; j++) { try { threads[j].join(); } catch (InterruptedException e) { e.printStackTrace(); } } } }; } long start = System.currentTimeMillis(); for (int i = 0; i < MAX_CONNECTION; i++) { threads[i].start(); } for (int i = 0; i < MAX_CONNECTION; i++) { threads[i].join(); } long end = System.currentTimeMillis(); Assert.assertEquals(MAX_CONNECTION * LOOP * threadCount, results.size()); // Assert.assertTrue(results.pollLast().equals(MAX_CONNECTION * LOOP * threadCount + 1L)); // Assert.assertTrue(results.pollFirst().equals(2L)); System.out.println("Time elapsed:" + ((double) (end - start + 1) / 1000.0) + "s\n TPS:" + ((double) (MAX_CONNECTION * LOOP * threadCount) / (double) (end - start + 1) * 1000.0) + "/s"); } } ================================================ FILE: src/test/java/io/mycat/sequence/SequenceHandlerTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.sequence; import junit.framework.Assert; import org.junit.Test; import io.mycat.route.sequence.handler.IncrSequencePropHandler; import io.mycat.route.sequence.handler.SequenceHandler; /** * 全局序列号单元测试 * * @author Michael * @time Create on 2013-12-30 上午12:07:51 * @version 1.0 */ public class SequenceHandlerTest { //@Test public void testPropSequence() { SequenceHandler hander = IncrSequencePropHandler.getInstance(); Assert.assertEquals(hander.nextId("DEF") - hander.nextId("DEF"), -1); } /** * @param args */ public static void main(String[] args) { SequenceHandler hander = IncrSequencePropHandler.getInstance(); System.out.println(hander.nextId("DEF")); } } ================================================ FILE: src/test/java/io/mycat/sequence/SequenceTest.java ================================================ package io.mycat.sequence; import io.mycat.MycatServer; import org.junit.After; import org.junit.Before; import org.junit.Test; import java.util.Set; import java.util.TreeSet; import java.util.UUID; /** * 全局序列号测试 * * @author Hash Zhang * @version 1.0 * @time 00:12:05 2016/5/6 */ public class SequenceTest { private Set sequenceSet; private long startTime; private long endTime; @Before public void initialize() { sequenceSet = new TreeSet<>(); startTime = System.nanoTime(); } // @Test // public void testIncrement(){ // System.out.print("Increment "); // for (int i = 0; i < 1000000; i++) { // sequenceSet.add(i+""); // } // } // @Test public void testUUID(){ System.out.print("UUID "); for (int i = 0; i < 100; i++) { sequenceSet.add(UUID.randomUUID().toString()); } } @Test public void testRandom(){ TreeSet treeSet= new TreeSet<>(); System.out.println(Long.toBinaryString(Long.valueOf(System.currentTimeMillis()+"")).length()); } @Test public void testRandom2(){ System.out.print("UUID "); for (int i = 0; i < 100; i++) { sequenceSet.add("aaassscccddd"+i); } } @Test public void testXAXID(){ String xid = MycatServer.getInstance().getXATXIDGLOBAL(); System.out.println(xid); } @After public void end() { endTime = System.nanoTime(); System.out.println("Time elapsed: " + (endTime - startTime)/(1000000L) + "ms"); } } ================================================ FILE: src/test/java/io/mycat/server/handler/ServerHandlerTest.java ================================================ package io.mycat.server.handler; public class ServerHandlerTest { } ================================================ FILE: src/test/java/io/mycat/server/interceptor/impl/GlobalTableUtilTest.java ================================================ package io.mycat.server.interceptor.impl; import org.junit.Test; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; import junit.framework.Assert; public class GlobalTableUtilTest { private static final String originSql1 = "CREATE TABLE retl_mark" + "(" + " ID BIGINT AUTO_INCREMENT," + " CHANNEL_ID INT(11)," + " CHANNEL_INFO varchar(128)," + " CONSTRAINT RETL_MARK_ID PRIMARY KEY (ID)" + ") ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;"; private static final String originSql2 = "CREATE TABLE retl_mark" + "(" + " ID BIGINT AUTO_INCREMENT," + " CHANNEL_ID INT(11)," + " CHANNEL_INFO varchar(128)," + " _MYCAT_OP_TIME int," + " CONSTRAINT RETL_MARK_ID PRIMARY KEY (ID)" + ") ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;"; @Test public void addColumnIfCreate() { String sql = parseSql(originSql1); System.out.println(sql); boolean contains = sql.contains("_mycat_op_time "); Assert.assertTrue(contains); sql = parseSql(originSql2); System.out.println(sql); Assert.assertFalse(sql.contains("_mycat_op_time int COMMENT '全局表保存修改时间戳的字段名'")); } public String parseSql(String sql) { MySqlStatementParser parser = new MySqlStatementParser(sql); SQLStatement statement = parser.parseStatement(); return GlobalTableUtil.addColumnIfCreate(sql, statement); } } ================================================ FILE: src/test/java/io/mycat/sqlexecute/BaseSQLExeTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.sqlexecute; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import junit.framework.Assert; /** * base sql test cases,some data should in database first * * @author wuzhih * */ public class BaseSQLExeTest { private static boolean driverLoaded = false; public static Connection getCon(String url, String user, String passwd) throws SQLException { Connection theCon = DriverManager.getConnection(url, user, passwd); return theCon; } private static void testMultiNodeNormalSQL(Connection theCon) throws SQLException { theCon.setAutoCommit(true); System.out.println("testMultiNodeNormalSQL begin"); String[] sqls = { "select * from travelrecord where id=1", "select * from travelrecord order by fee limit 200,100", "select * from travelrecord limit 100", "select sum(fee) total_fee, days,count(id),max(fee),min(fee) from travelrecord group by days order by days desc limit 99 ", "update travelrecord set user_id=user_id where id =1", "delete from travelrecord where id =1 ", "insert into travelrecord (id,user_id,traveldate,fee,days) values(1,'wang','2014-01-05',510.5,3)" }; Statement stmt = theCon.createStatement(); for (String sql : sqls) { stmt.execute(sql); } theCon.setAutoCommit(false); for (String sql : sqls) { stmt.execute(sql); } theCon.commit(); System.out.println("testMultiNodeNormalSQL end"); } private static void testMultiNodeLargeResultset(Connection theCon) throws SQLException { theCon.setAutoCommit(true); System.out.println("testMultiNodeLargeResultset begin"); String sql = "select * from travelrecord limit 100000"; for (int i = 0; i < 100; i++) { Statement stmt = theCon.createStatement(); ResultSet rs = stmt.executeQuery(sql); int count = 0; while (rs.next()) { count++; } rs.close(); stmt.close(); System.out.println("total result " + count); } System.out.println("testMultiNodeLargeResultset end"); } private static void testSingleNodeNormalSQL(Connection theCon) throws SQLException { theCon.setAutoCommit(true); System.out.println("testSingleNodeNormalSQL begin"); String[] sqls = { "select * from company limit 100", "select name,count(id),max(id),min(id) from company group by name order by name desc limit 100 ", "update company set name=name where id =1", "delete from company where id =-1 ", "delete from company where id =1 ", "insert into company(id,name) values(1,'hp')" }; Statement stmt = theCon.createStatement(); for (String sql : sqls) { System.out.println("execute " + sql); stmt.execute(sql); } theCon.setAutoCommit(false); for (String sql : sqls) { stmt.execute(sql); } theCon.commit(); System.out.println("testSingleNodeNormalSQL end"); } private static void testBadSQL(Connection theCon) throws SQLException { System.out.println("testBadSQL begin"); theCon.setAutoCommit(true); String[] sqls = { "select sum(fee) total_fee, days,count(id),max(fee),min(fee) from travelrecord group by id order by id desc limit 99", "select a ,id,name from company limit 1", "update company set name=name where id =-1", "insert into company(id,name) values(1,'hp')", "insert into company(id,name,badname) values(1,'hp')", "insert into travelrecord (id,user_id,traveldate,fee,days) values(1,’wang’,’2014-01-05’,510.5,3)", "insert into travelrecord (id,user_id,traveldate,fee,days,badcolumn) values(1,’wang’,’2014-01-05’,510.5,3)", "select sum(fee) total_fee, days,count(id),max(fee),min(fee) from travelrecord group by count(id) order by count(id) desc limit 99 "}; for (String sql : sqls) { try { System.out.println("execute "+sql); theCon.createStatement().executeQuery(sql); } catch (Exception e) { // e.printStackTrace(); Assert.assertEquals(true, e != null); } } System.out.println("testBadSQL passed"); } private static void testTransaction(Connection theCon) throws SQLException { System.out.println("testTransaction begin"); theCon.setAutoCommit(false); String sql = "select id,name from company limit 1"; String oldName = null; String upSQL = null; ResultSet rs = theCon.createStatement().executeQuery(sql); if (rs.next()) { long id = rs.getLong(1); oldName = rs.getString(2); upSQL = "update company set name='updatedname' where id=" + id; // System.out.println(sql); } int count = theCon.createStatement().executeUpdate(upSQL); Assert.assertEquals(true, count > 0); theCon.rollback(); rs = theCon.createStatement().executeQuery(sql); String newName = null; if (rs.next()) { newName = rs.getString(2); } Assert.assertEquals(true, oldName.equals(newName)); System.out.println("testTransaction passed"); } public static void closeCon(Connection theCon) { try { theCon.close(); } catch (SQLException e) { e.printStackTrace(); } } public static Connection getCon(String[] args) throws Exception { if (driverLoaded == false) { Class.forName("com.mysql.jdbc.Driver"); driverLoaded = true; } if (args.length < 3) { System.out .println("input param,format: [jdbcurl] [user] [password] "); return null; } String url = args[0]; String user = args[1]; String password = args[2]; return getCon(url, user, password); } public static void main(String[] args) throws Exception { Connection theCon = null; try { theCon = getCon(args); testBadSQL(theCon); } catch (SQLException e) { e.printStackTrace(); } finally { closeCon(theCon); } try { theCon = getCon(args); testMultiNodeLargeResultset(theCon); } catch (SQLException e) { e.printStackTrace(); } finally { closeCon(theCon); } try { theCon = getCon(args); testSingleNodeNormalSQL(theCon); } catch (SQLException e) { e.printStackTrace(); } finally { closeCon(theCon); } try { theCon = getCon(args); testTransaction(theCon); } catch (SQLException e) { e.printStackTrace(); } finally { closeCon(theCon); } theCon = null; try { theCon = getCon(args); testMultiNodeNormalSQL(theCon); } catch (SQLException e) { e.printStackTrace(); } finally { closeCon(theCon); } } } ================================================ FILE: src/test/java/io/mycat/sqlexecute/MultiThreadSelectTest.java ================================================ package io.mycat.sqlexecute; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.List; public class MultiThreadSelectTest { private static void testSequnce(Connection theCon) throws SQLException { boolean autCommit = System.currentTimeMillis() % 2 == 1; theCon.setAutoCommit(autCommit); String sql = "select * from company "; Statement stmt = theCon.createStatement(); int charChoise = (int) (System.currentTimeMillis() % 3); if (charChoise == 0) { stmt.executeQuery("SET NAMES UTF8;"); } else if (charChoise == 1) { stmt.executeQuery("SET NAMES latin1;"); } if (charChoise == 2) { stmt.executeQuery("SET NAMES gb2312;"); } ResultSet rs = stmt.executeQuery(sql); if (rs.next()) { System.out.println(Thread.currentThread().getName() + " get seq " + rs.getLong(1)); } else { System.out.println(Thread.currentThread().getName() + " can't get seq "); } if (autCommit == false) { theCon.commit(); } stmt.close(); } private static Connection getCon(String url, String user, String passwd) throws SQLException { Connection theCon = DriverManager.getConnection(url, user, passwd); return theCon; } public static void main(String[] args) { try { Class.forName("com.mysql.jdbc.Driver"); } catch (ClassNotFoundException e1) { e1.printStackTrace(); } final String url = "jdbc:mysql://localhost:8066/TESTDB"; final String user = "test"; final String password = "test"; List threads = new ArrayList(100); for (int i = 0; i < 50; i++) { threads.add(new Thread() { public void run() { Connection con; try { con = getCon(url, user, password); for (int j = 0; j < 10000; j++) { testSequnce(con); } } catch (SQLException e) { e.printStackTrace(); } } }); } for (Thread thred : threads) { thred.start(); } boolean hasRunning = true; while (hasRunning) { hasRunning = false; for (Thread thred : threads) { if (thred.isAlive()) { try { Thread.sleep(1000); hasRunning = true; } catch (InterruptedException e) { } } } } } } ================================================ FILE: src/test/java/io/mycat/sqlexecute/MultiThreadSequnceTest.java ================================================ package io.mycat.sqlexecute; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.List; public class MultiThreadSequnceTest { private static void testSequnce(Connection theCon) throws SQLException { try { theCon.setAutoCommit(false); String sql = "select next value for MYCATSEQ_GLOBAL "; Statement stmt = theCon.createStatement(); ResultSet rs = stmt.executeQuery(sql); if (rs.next()) { System.out.println(Thread.currentThread().getName() + " get seq " + rs.getLong(1)); } else { System.out.println(Thread.currentThread().getName() + " can't get seq "); } theCon.commit(); stmt.close(); } finally { theCon.close(); } } private static Connection getCon(String url, String user, String passwd) throws SQLException { Connection theCon = DriverManager.getConnection(url, user, passwd); return theCon; } public static void main(String[] args) { try { Class.forName("com.mysql.jdbc.Driver"); } catch (ClassNotFoundException e1) { e1.printStackTrace(); } final String url = "jdbc:mysql://localhost:8066/TESTDB"; final String user = "test"; final String password = "test"; List threads = new ArrayList(100); for (int i = 0; i < 100; i++) { threads.add(new Thread() { public void run() { Connection con; try { con = getCon(url, user, password); testSequnce(con); } catch (SQLException e) { e.printStackTrace(); } } }); } for (Thread thred : threads) { thred.start(); } boolean hasRunning = true; while (hasRunning) { hasRunning = false; for (Thread thred : threads) { if (thred.isAlive()) { try { Thread.sleep(1000); hasRunning = true; } catch (InterruptedException e) { } } } } } } ================================================ FILE: src/test/java/io/mycat/sqlexecute/MycatMulitJdbcVersionTest.java ================================================ package io.mycat.sqlexecute; import java.net.URL; import java.net.URLClassLoader; import java.sql.Connection; import java.sql.Driver; import java.sql.DriverManager; import java.sql.DriverPropertyInfo; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; import java.sql.Statement; import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.logging.Logger; /** * * 测试mycat对不同版本的mysql jdbc的兼容性 * * *

* 关联issue: @see https://github.com/MyCATApache/Mycat-Server/issues/1203 * *

* Note:
* 1. 请将这个类放到新建的project独立运行, mycat pom.xml里面使用的mysql驱动会影响测试结果
* 2. 确保project新建lib子目录并且在lib子目录里面放置了各类版本的mysql jdbc驱动 * 3. 程序会动态加载不同版本的jdbc驱动, 请不要将任何mysql jdbc驱动加入classpath, 否则也可能影响测试结果 * * @author CrazyPig * @since 2016-11-13 * */ public class MycatMulitJdbcVersionTest { private static final String JDBC_URL = "jdbc:mysql://localhost:8066/TESTDB"; private static final String USER = "root"; private static final String PASSWORD = "123456"; private static final Map jdbcVersionMap = new HashMap(); private static final Map tmpDriverMap = new HashMap(); // 动态加载jdbc驱动 private static void dynamicLoadJdbc(String mysqlJdbcFile) throws Exception { URL u = new URL("jar:file:lib/" + mysqlJdbcFile + "!/"); String classname = jdbcVersionMap.get(mysqlJdbcFile); URLClassLoader ucl = new URLClassLoader(new URL[] { u }); Driver d = (Driver)Class.forName(classname, true, ucl).newInstance(); DriverShim driver = new DriverShim(d); DriverManager.registerDriver(driver); tmpDriverMap.put(mysqlJdbcFile, driver); } // 每一次测试完卸载对应版本的jdbc驱动 private static void dynamicUnLoadJdbc(String mysqlJdbcFile) throws SQLException { DriverManager.deregisterDriver(tmpDriverMap.get(mysqlJdbcFile)); } // 进行一次测试 private static void testOneVersion(String mysqlJdbcFile) { System.out.println("start test mysql jdbc version : " + mysqlJdbcFile); try { dynamicLoadJdbc(mysqlJdbcFile); } catch (Exception e1) { e1.printStackTrace(); } Connection conn = null; try { conn = DriverManager.getConnection(JDBC_URL, USER, PASSWORD); Statement stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery("select user()"); System.out.println("select user() output : "); while(rs.next()) { System.out.println(rs.getObject(1)); } rs = stmt.executeQuery("show tables"); System.out.println("show tables output : "); while(rs.next()) { System.out.println(rs.getObject(1)); } } catch (SQLException e) { e.printStackTrace(); } finally { if(conn != null) { try { conn.close(); } catch (SQLException e) { e.printStackTrace(); } } } try { dynamicUnLoadJdbc(mysqlJdbcFile); } catch (SQLException e) { e.printStackTrace(); } System.out.println("end !!!"); System.out.println(); } public static void main(String[] args) { // 多版本mysql jdbc驱动兼容性测试 // NOTE: 注意将对应的jar放到lib子目录, 不需要加入classpath!!! jdbcVersionMap.put("mysql-connector-java-6.0.3.jar", "com.mysql.cj.jdbc.Driver"); jdbcVersionMap.put("mysql-connector-java-5.1.6.jar", "com.mysql.jdbc.Driver"); jdbcVersionMap.put("mysql-connector-java-5.1.31.jar", "com.mysql.jdbc.Driver"); jdbcVersionMap.put("mysql-connector-java-5.1.35.jar", "com.mysql.jdbc.Driver"); jdbcVersionMap.put("mysql-connector-java-5.1.39.jar", "com.mysql.jdbc.Driver"); // 更多的jdbc驱动... for(String mysqlJdbcFile : jdbcVersionMap.keySet()) { testOneVersion(mysqlJdbcFile); } } } class DriverShim implements Driver { private Driver driver; DriverShim(Driver d) { this.driver = d; } public boolean acceptsURL(String u) throws SQLException { return this.driver.acceptsURL(u); } public Connection connect(String u, Properties p) throws SQLException { return this.driver.connect(u, p); } @Override public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException { return this.driver.getPropertyInfo(url, info); } @Override public int getMajorVersion() { return this.driver.getMajorVersion(); } @Override public int getMinorVersion() { return this.driver.getMinorVersion(); } @Override public boolean jdbcCompliant() { return this.driver.jdbcCompliant(); } @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException { return this.driver.getParentLogger(); } } ================================================ FILE: src/test/java/io/mycat/sqlexecute/RollbackTest.java ================================================ package io.mycat.sqlexecute; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; public class RollbackTest { private static Connection getCon(String url, String user, String passwd) throws SQLException { Connection theCon = DriverManager.getConnection(url, user, passwd); return theCon; } public static void main(String[] args) { } } ================================================ FILE: src/test/java/io/mycat/sqlexecute/ServerPrepareTest.java ================================================ package io.mycat.sqlexecute; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.sql.*; import org.junit.Assert; /** * * @author CrazyPig * */ public class ServerPrepareTest { // JDBC driver name and database URL static final String JDBC_DRIVER = "com.mysql.jdbc.Driver"; static final String DB_URL = "jdbc:mysql://localhost:8066/TESTDB?useServerPrepStmts=true"; // Database credentials static final String USER = "root"; static final String PASS = "mysql"; static { try { Class.forName("com.mysql.jdbc.Driver"); } catch (ClassNotFoundException e) { e.printStackTrace(); } } /** * create table hotnews ( * id int primary key auto_increment, * title varchar(200), * content text, * image0 blob, * image1 blob, * image2 mediumblob, * image3 longblob * ) engine = innodb default character set = 'utf8'; */ /** * 测试发送COM_STMT_SEND_LONG_DATA命令 * @throws IOException */ public static void testComStmtSendLondData() throws IOException { Connection conn = null; PreparedStatement pstmt = null; ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); // 获取待存储图片输入流 InputStream image0In = classLoader.getResourceAsStream("blob/image0.jpg"); InputStream image1In = classLoader.getResourceAsStream("blob/image1.png"); InputStream image2In = classLoader.getResourceAsStream("blob/image2.png"); InputStream image3In = classLoader.getResourceAsStream("blob/image3.png"); // 保存图片字节数据,待后面取回数据进行校验 byte[] image0Bytes = getBytes(image0In); byte[] image1Bytes = getBytes(image1In); byte[] image2Bytes = getBytes(image2In); byte[] image3Bytes = getBytes(image3In); try { conn = DriverManager.getConnection(DB_URL,USER,PASS); pstmt = conn.prepareStatement("insert into hotnews(id, title, content, image0, image1, image2, image3) values(?,?,?,?,?,?,?)"); pstmt.setInt(1, 1314); pstmt.setString(2, "hotnew"); // text字段设置 pstmt.setBinaryStream(3, new ByteArrayInputStream("this is a content of hotnew".getBytes("UTF-8"))); // blob字段构造 Blob image0Blob = conn.createBlob(); Blob image1Blob = conn.createBlob(); Blob image2Blob = conn.createBlob(); Blob image3Blob = conn.createBlob(); image0Blob.setBytes(1, image0Bytes); image1Blob.setBytes(1, image1Bytes); image2Blob.setBytes(1, image2Bytes); image3Blob.setBytes(1, image3Bytes); // blob字段设置 pstmt.setBlob(4, image0Blob); pstmt.setBlob(5, image1Blob); pstmt.setBlob(6, image2Blob); pstmt.setBlob(7, image3Blob); // 执行 pstmt.execute(); // 从表里面拿出刚插入的数据, 对blob字段进行校验 pstmt = conn.prepareStatement("select image0, image1, image2, image3 from hotnews where id = ?"); pstmt.setInt(1, 1314); ResultSet rs = pstmt.executeQuery(); if(rs.next()) { InputStream _image0In = rs.getBlob(1).getBinaryStream(); InputStream _image1In = rs.getBlob(2).getBinaryStream(); InputStream _image2In = rs.getBlob(3).getBinaryStream(); InputStream _image3In = rs.getBlob(4).getBinaryStream(); // 断言从数据库取出来的数据,与之前发送的数据是一致的(字节数组内容比较) Assert.assertArrayEquals(image0Bytes, getBytes(_image0In)); Assert.assertArrayEquals(image1Bytes, getBytes(_image1In)); Assert.assertArrayEquals(image2Bytes, getBytes(_image2In)); Assert.assertArrayEquals(image3Bytes, getBytes(_image3In)); } pstmt.close(); } catch (SQLException e) { e.printStackTrace(); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } finally { if(conn != null) { try { conn.close(); } catch (SQLException e) { e.printStackTrace(); } } } } private static byte[] getBytes(InputStream in) throws IOException { byte[] bytes = new byte[0]; byte[] buffer = new byte[1024]; ByteArrayOutputStream bout = new ByteArrayOutputStream(); int len = 0; while((len = in.read(buffer)) > -1) { bout.write(buffer, 0, len); } bytes = bout.toByteArray(); return bytes; } /** * 测试发送COM_STMT_RESET命令 */ public static void testComStmtRest() { Connection conn = null; PreparedStatement pstmt = null; try { conn = DriverManager.getConnection(DB_URL,USER,PASS); pstmt = conn.prepareStatement("insert into hotnews(id, title, content) values(?,?,?)"); pstmt.setInt(1, 1314); pstmt.setString(2, "hotnew"); pstmt.setBinaryStream(3, new ByteArrayInputStream("this is a content of hotnew".getBytes("UTF-8"))); pstmt.execute(); pstmt.clearParameters(); pstmt.setInt(1, 1315); pstmt.setString(2, "hotnew"); pstmt.setBinaryStream(3, new ByteArrayInputStream("this is a new content of hotnew".getBytes("UTF-8"))); pstmt.execute(); pstmt.close(); } catch (SQLException e) { e.printStackTrace(); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } finally { if(conn != null) { try { conn.close(); } catch (SQLException e) { e.printStackTrace(); } } } } public static void simpleTest() { Connection conn = null; PreparedStatement stmt = null; try{ System.out.println("Connecting to database..."); conn = DriverManager.getConnection(DB_URL,USER,PASS); System.out.println("Creating statement..."); String sql = "SELECT * FROM test where id> list = analyzer.getValues(); Assert.assertTrue((list.get(0).getValue().get() == (long) THREAD_COUNT * LOOP_COUNT)); Assert.assertTrue((list.get(1).getValue().get() == (long) THREAD_COUNT * LOOP_COUNT)); Assert.assertTrue((list.get(2).getValue().get() == (long) THREAD_COUNT * LOOP_COUNT)); } @Test @Ignore public void testUserSqlHighStat() throws InterruptedException { final UserSqlHighStat userSqlHighStat = new UserSqlHighStat(); Thread thread[] = new Thread[THREAD_COUNT]; Thread thread2[] = new Thread[THREAD_COUNT]; Thread thread3[] = new Thread[THREAD_COUNT]; for (int i = 0; i < THREAD_COUNT; i++) { thread[i] = new Thread() { @Override public void run() { for (int j = 0; j < LOOP_COUNT; j++) { userSqlHighStat.addSql(sql, 10L, 1L, 11L, "127.0.0.1"); } } }; thread2[i] = new Thread() { @Override public void run() { for (int j = 0; j < LOOP_COUNT; j++) { userSqlHighStat.addSql(sql2, 10L, 1L, 11L, "127.0.0.1"); } } }; thread3[i] = new Thread() { @Override public void run() { for (int j = 0; j < LOOP_COUNT; j++) { userSqlHighStat.addSql(sql4, 10L, 1L, 11L, "127.0.0.1"); } } }; } for (int i = 0; i < THREAD_COUNT; i++) { thread[i].start(); thread2[i].start(); thread3[i].start(); } for (int i = 0; i < THREAD_COUNT; i++) { thread[i].join(); thread2[i].join(); thread3[i].join(); } List sqlFrequency = userSqlHighStat.getSqlFrequency(true); Assert.assertTrue(sqlFrequency.size() == 2); Assert.assertTrue(sqlFrequency.get(0).getCount() == 2 * THREAD_COUNT *LOOP_COUNT); Assert.assertTrue(sqlFrequency.get(1).getCount() == THREAD_COUNT *LOOP_COUNT); } } ================================================ FILE: src/test/java/io/mycat/util/ArrayPerformanceMain.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import java.util.ArrayList; import java.util.List; /** * @author mycat */ public class ArrayPerformanceMain { public void tArray() { byte[] a = new byte[] { 1, 2, 3, 4, 5, 6, 7 }; System.currentTimeMillis(); long t1 = System.currentTimeMillis(); for (int x = 0; x < 1000000; x++) { byte[][] ab = new byte[10][]; for (int i = 0; i < ab.length; i++) { ab[i] = a; } } long t2 = System.currentTimeMillis(); System.out.println("array take time:" + (t2 - t1) + " ms."); } public void tList() { byte[] a = new byte[] { 1, 2, 3, 4, 5, 6, 7 }; System.currentTimeMillis(); long t1 = System.currentTimeMillis(); for (int x = 0; x < 1000000; x++) { List ab = new ArrayList(10); for (int i = 0; i < ab.size(); i++) { ab.add(a); } } long t2 = System.currentTimeMillis(); System.out.println("list take time:" + (t2 - t1) + " ms."); } } ================================================ FILE: src/test/java/io/mycat/util/BitTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import org.junit.Test; /** * @author mycat */ public class BitTest { @Test public void testNoop() { } public static void main(String[] args) { System.out.println(0xffff0001 & 0xffff);// 低16位 System.out.println(0x0002ffff >>> 16);// 高16位 } } ================================================ FILE: src/test/java/io/mycat/util/ConcurrentHashMapMain.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; /** * @author mycat */ public class ConcurrentHashMapMain { private final ConcurrentMap cm; public ConcurrentHashMapMain() { cm = new ConcurrentHashMap(); cm.put("abcdefg", "abcdefghijk"); } public void tGet() { for (int i = 0; i < 1000000; i++) { cm.get("abcdefg"); } } public void tGetNone() { for (int i = 0; i < 1000000; i++) { cm.get("abcdefghijk"); } } public void tEmpty() { for (int i = 0; i < 1000000; i++) { cm.isEmpty(); } } public void tRemove() { for (int i = 0; i < 1000000; i++) { cm.remove("abcdefg"); } } } ================================================ FILE: src/test/java/io/mycat/util/HashMapMain.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import java.util.HashMap; import java.util.Map; /** * @author mycat */ public class HashMapMain { public void t() { String[] keys = new String[] { "a", "b", "c", "d", "e" }; long t = System.currentTimeMillis(); int count = 1000000; Map m = new HashMap(); t = System.currentTimeMillis(); for (int i = 0; i < count; i++) { for (String key : keys) { m.put(key, "String.value"); } for (String key : keys) { m.remove(key); } } System.out.println((System.currentTimeMillis() - t) * 1000 * 1000 / (count * keys.length * 2) + " ns"); } } ================================================ FILE: src/test/java/io/mycat/util/HexFormatUtilMain.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import java.util.ArrayList; import java.util.List; import io.mycat.util.HexFormatUtil; /** * @author mycat */ public class HexFormatUtilMain { public static void main(String[] args) { List srcList = new ArrayList(); srcList.add("53 45 4C 45 43 54 20 4C 41 53 54 5F 49 4E 53 45 52 54 5F 49 44 28 29"); srcList.add("4C 41 53 54 5F 49 4E 53 45 52 54 5F 49 44 28 29"); srcList.add("64 65 66"); srcList.add("73 65 6C 65 63 74 20 2A 20 66 72 6F 6D 20 62 72 6D 6D 73 5F 75 73 65 72 20 6C 69 6D 69 74 20 31"); srcList.add("62 72 6D 6D 73 31"); srcList.add("62 72 6D 6D 73 5F 75 73 65 72"); srcList.add("69 64"); srcList.add("49 4E 53 45 52 54 20 49 4E 54 4F 20 62 72 6D 6D 73 5F 75 73 65 72 20 56 41 4C 55 45 53 20 28 6E 75 6C 6C 2C 27 68 65 78 69 61 6E 6D 61 6F 27 2C 30 2C 30 2C 30 2C 30 2C 27 32 30 30 39 2D 30 33 2D 30 35 27 2C 27 31 32 31 2E 33 34 2E 31 37 38 2E 33 35 27 2C 27 32 30 30 39 2D 30 33 2D 30 35 20 31 34 3A 33 38 3A 33 35 27 2C 27 32 30 30 39 2D 30 33 2D 30 35 20 31 34 3A 33 38 3A 33 35 27 2C 27 32 30 30 39 2D 30 33 2D 30 35 20 31 34 3A 33 38 3A 33 35 27 29"); srcList.add("73 65 6C 65 63 74 20 69 64 20 66 72 6F 6D 20 6F 66 66 65 72 20 6C 69 6D 69 74 20 3F"); srcList.add("73 65 6C 65 63 74 20 69 64 20 66 72 6F 6D 20 6F 66 66 65 72 20 6C 69 6D 69 74 20 31"); srcList.add("64 65 66"); srcList.add("3F"); srcList.add("6F 66 66 65 72 31"); srcList.add("6F 66 66 65 72"); srcList.add("32 39 30 34 33"); for (int i = 0; i < srcList.size(); i++) { System.out.println(HexFormatUtil.fromHex(srcList.get(i), "UTF-8")); } System.out.println(HexFormatUtil.fromHex8B("73 71 00 00 00 00 00 00")); } } ================================================ FILE: src/test/java/io/mycat/util/HexFormatUtilTest.java ================================================ package io.mycat.util; import org.junit.Assert; import org.junit.Test; /** * * @author CrazyPig * @since 2016-09-09 * */ public class HexFormatUtilTest { @Test public void testBytesToString() { byte[] bytes = new byte[]{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 }; String hexString = HexFormatUtil.bytesToHexString(bytes); String expected = "0102030405060708090A0B0C0D0E0F1011121314"; Assert.assertEquals(expected, hexString); } } ================================================ FILE: src/test/java/io/mycat/util/LockPerfMain.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; /** * @author mycat */ public class LockPerfMain { public void tReentrantLock() { System.currentTimeMillis(); ReentrantLock lock = new ReentrantLock(); long t1 = System.currentTimeMillis(); for (int i = 0; i < 10000000; i++) { if (lock.tryLock()) { try { // ... } finally { lock.unlock(); } } } long t2 = System.currentTimeMillis(); System.out.println("take time:" + (t2 - t1) + " ms."); } public void tAtomicBoolean() { System.currentTimeMillis(); AtomicBoolean atomic = new AtomicBoolean(); long t1 = System.currentTimeMillis(); for (int i = 0; i < 10000000; i++) { if (atomic.compareAndSet(false, true)) { try { // ... } finally { atomic.set(false); } } } long t2 = System.currentTimeMillis(); System.out.println("take time:" + (t2 - t1) + " ms."); } } ================================================ FILE: src/test/java/io/mycat/util/MapPerfMain.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.UUID; import org.junit.Assert; /** * @author mycat */ public class MapPerfMain { public void t1() { Map m = new HashMap(); for (int i = 0; i < 100000; i++) { m.put(UUID.randomUUID().toString(), new Date()); } remove1(m); Assert.assertEquals(0, m.size()); } public void t2() { Map m = new HashMap(); for (int i = 0; i < 100000; i++) { m.put(UUID.randomUUID().toString(), new Date()); } remove2(m); Assert.assertEquals(0, m.size()); } void remove1(Map m) { Iterator> it = m.entrySet().iterator(); while (it.hasNext()) { it.next().getValue(); it.remove(); } } void remove2(Map m) { Iterator> it = m.entrySet().iterator(); while (it.hasNext()) { it.next().getValue(); } m.clear(); } } ================================================ FILE: src/test/java/io/mycat/util/SchemaUtilTest.java ================================================ package io.mycat.util; import static org.junit.Assert.assertArrayEquals; import org.junit.Test; import io.mycat.server.util.SchemaUtil; /** * @author stones-he */ public class SchemaUtilTest { @Test public void parseShowTableTest() { String stmt = "SHOW FULL TABLES FROM schema001 WHERE Tables_in_schema001 LIKE 'tab_company'; "; String[] fields = SchemaUtil.parseShowTable(stmt); assertArrayEquals(new String[]{"1", "FULL", "FROM", "schema001", "WHERE", "Tables_in_schema001", "LIKE", "'tab_company'", "tab_company"}, fields); } @Test public void parseShowTableTest1() { String stmt = "SHOW FULL TABLES LIKE 'tab_company' "; String[] fields = SchemaUtil.parseShowTable(stmt); assertArrayEquals(new String[]{"1", "FULL", null, null, null, null, "LIKE", "'tab_company'", "tab_company"}, fields); } @Test public void parseShowTableTest2() { String stmt = "SHOW FULL TABLES IN mysql LIKE 'time%'; "; String[] fields = SchemaUtil.parseShowTable(stmt); assertArrayEquals(new String[]{"1", "FULL", "IN", "mysql", null, null, "LIKE", "'time%'", "time%"}, fields); stmt = "SHOW FULL TABLES IN mysql LIKE '%time%'; "; fields = SchemaUtil.parseShowTable(stmt); assertArrayEquals(new String[]{"1", "FULL", "IN", "mysql", null, null, "LIKE", "'%time%'", "%time%"}, fields); stmt = "SHOW FULL TABLES IN mysql LIKE '%time'; "; fields = SchemaUtil.parseShowTable(stmt); assertArrayEquals(new String[]{"1", "FULL", "IN", "mysql", null, null, "LIKE", "'%time'", "%time"}, fields); } @Test public void parseShowTableTest3() { String stmt = "SHOW TABLES WHERE table_type = 'BASE TABLE';"; String[] fields = SchemaUtil.parseShowTable(stmt); assertArrayEquals(new String[]{"1", null, null, null, "WHERE", "table_type", "=", "'BASE TABLE'", "BASE TABLE"}, fields); } @Test public void parseShowTableTest4() { String stmt = "SHOW TABLES WHERE table_type = 'BASE TABLE';"; String[] fields = SchemaUtil.parseShowTable(stmt); assertArrayEquals(new String[]{"1", null, null, null, "WHERE", "table_type", "=", "'BASE TABLE'", "BASE TABLE"}, fields); // 多个空格也能解析 stmt = "SHOW FULL TABLES WHERE table_type = 'BASE TABLE';"; fields = SchemaUtil.parseShowTable(stmt); assertArrayEquals(new String[]{"1", "FULL", null, null, "WHERE", "table_type", "=", "'BASE TABLE'", "BASE TABLE"}, fields); // stmt = "SHOW FULL TABLES in schema001 WHERE table_type = 'BASE TABLE';"; fields = SchemaUtil.parseShowTable(stmt); assertArrayEquals(new String[]{"1", "FULL", "in", "schema001", "WHERE", "table_type", "=", "'BASE TABLE'", "BASE TABLE"}, fields); // stmt = "SHOW FULL TABLES WHERE table_type LIKE '%BASE TABLE';"; fields = SchemaUtil.parseShowTable(stmt); assertArrayEquals(new String[]{"1", "FULL", null, null, "WHERE", "table_type", "LIKE", "'%BASE TABLE'", "%BASE TABLE"}, fields); // stmt = "SHOW FULL TABLES WHERE tables_in_0001 LIKE '%BASE TABLE';"; fields = SchemaUtil.parseShowTable(stmt); assertArrayEquals(new String[]{"1", "FULL", null, null, "WHERE", "tables_in_0001", "LIKE", "'%BASE TABLE'", "%BASE TABLE"}, fields); } //SHOW FULL TABLES in miccore WHERE table_type = 'BASE TABLE'; } ================================================ FILE: src/test/java/io/mycat/util/SmallSetTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import java.util.Collection; import java.util.Iterator; import io.mycat.util.SmallSet; import junit.framework.Assert; import junit.framework.TestCase; /** * @author mycat */ public class SmallSetTest extends TestCase { public void assertListEquals(Collection col, Object... objects) { if (objects == null) { Assert.assertTrue(col.isEmpty()); } Assert.assertEquals(objects.length, col.size()); int i = 0; for (Object o : col) { Assert.assertEquals(objects[i++], o); } } public void testSet() throws Exception { SmallSet sut = new SmallSet(); sut.add(1); Assert.assertEquals(1, sut.size()); Iterator iter = sut.iterator(); Assert.assertTrue(iter.hasNext()); Assert.assertTrue(iter.hasNext()); Assert.assertEquals(1, iter.next()); Assert.assertFalse(iter.hasNext()); assertListEquals(sut, 1); try { iter.next(); Assert.assertTrue(false); } catch (Exception e) { } sut = new SmallSet(); sut.add(1); Assert.assertEquals(1, sut.size()); iter = sut.iterator(); Assert.assertTrue(iter.hasNext()); Assert.assertEquals(1, iter.next()); Assert.assertFalse(iter.hasNext()); assertListEquals(sut, 1); iter.remove(); Assert.assertEquals(0, sut.size()); Assert.assertFalse(iter.hasNext()); iter = sut.iterator(); Assert.assertFalse(iter.hasNext()); sut = new SmallSet(); sut.add(1); sut.add(2); Assert.assertEquals(2, sut.size()); iter = sut.iterator(); Assert.assertTrue(iter.hasNext()); Assert.assertEquals(1, iter.next()); Assert.assertTrue(iter.hasNext()); Assert.assertEquals(2, iter.next()); Assert.assertFalse(iter.hasNext()); assertListEquals(sut, 1, 2); iter.remove(); assertListEquals(sut, 1); Assert.assertEquals(1, sut.size()); Assert.assertFalse(iter.hasNext()); sut = new SmallSet(); sut.add(1); sut.add(2); assertListEquals(sut, 1, 2); Assert.assertEquals(2, sut.size()); iter = sut.iterator(); Assert.assertTrue(iter.hasNext()); Assert.assertEquals(1, iter.next()); Assert.assertTrue(iter.hasNext()); iter.remove(); assertListEquals(sut, 2); Assert.assertEquals(1, sut.size()); Assert.assertTrue(iter.hasNext()); Assert.assertEquals(2, iter.next()); Assert.assertFalse(iter.hasNext()); sut = new SmallSet(); sut.add(1); sut.add(2); assertListEquals(sut, 1, 2); Assert.assertEquals(2, sut.size()); iter = sut.iterator(); Assert.assertTrue(iter.hasNext()); Assert.assertEquals(1, iter.next()); Assert.assertTrue(iter.hasNext()); iter.remove(); assertListEquals(sut, 2); Assert.assertEquals(1, sut.size()); iter = sut.iterator(); Assert.assertTrue(iter.hasNext()); Assert.assertEquals(2, iter.next()); Assert.assertFalse(iter.hasNext()); sut = new SmallSet(); sut.add(1); sut.add(2); assertListEquals(sut, 1, 2); Assert.assertEquals(2, sut.size()); iter = sut.iterator(); Assert.assertTrue(iter.hasNext()); Assert.assertEquals(1, iter.next()); Assert.assertTrue(iter.hasNext()); iter.remove(); assertListEquals(sut, 2); Assert.assertEquals(1, sut.size()); Assert.assertTrue(iter.hasNext()); Assert.assertEquals(2, iter.next()); iter.remove(); assertListEquals(sut); iter = sut.iterator(); Assert.assertFalse(iter.hasNext()); sut = new SmallSet(); sut.add(1); sut.add(2); sut.add(3); assertListEquals(sut, 1, 2, 3); iter = sut.iterator(); Assert.assertTrue(iter.hasNext()); Assert.assertEquals(1, iter.next()); assertListEquals(sut, 1, 2, 3); iter.remove(); assertListEquals(sut, 2, 3); Assert.assertTrue(iter.hasNext()); Assert.assertEquals(2, iter.next()); iter.remove(); assertListEquals(sut, 3); Assert.assertTrue(iter.hasNext()); Assert.assertEquals(3, iter.next()); Assert.assertFalse(iter.hasNext()); iter.remove(); assertListEquals(sut); Assert.assertFalse(iter.hasNext()); iter = sut.iterator(); Assert.assertFalse(iter.hasNext()); sut = new SmallSet(); sut.add(1); sut.add(2); sut.add(3); assertListEquals(sut, 1, 2, 3); iter = sut.iterator(); Assert.assertTrue(iter.hasNext()); Assert.assertEquals(1, iter.next()); assertListEquals(sut, 1, 2, 3); iter.remove(); assertListEquals(sut, 2, 3); Assert.assertTrue(iter.hasNext()); Assert.assertEquals(2, iter.next()); Assert.assertTrue(iter.hasNext()); Assert.assertEquals(3, iter.next()); Assert.assertFalse(iter.hasNext()); iter.remove(); assertListEquals(sut, 2); Assert.assertFalse(iter.hasNext()); iter = sut.iterator(); Assert.assertTrue(iter.hasNext()); Assert.assertEquals(2, iter.next()); } } ================================================ FILE: src/test/java/io/mycat/util/SplitUtilTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import org.junit.Assert; import org.junit.Test; import io.mycat.util.SplitUtil; /** * @author mycat */ public class SplitUtilTest { @Test public void test() { String str = "mysql$1-3,mysql7,mysql9"; String[] destStr = SplitUtil.split(str, ',', '$', '-'); Assert.assertEquals(5, destStr.length); Assert.assertEquals("mysql1", destStr[0]); Assert.assertEquals("mysql2", destStr[1]); Assert.assertEquals("mysql3", destStr[2]); Assert.assertEquals("mysql7", destStr[3]); Assert.assertEquals("mysql9", destStr[4]); } @Test public void test1() { String src = "offer$0-3"; String[] dest = SplitUtil.split(src, '$', true); Assert.assertEquals(2, dest.length); Assert.assertEquals("offer", dest[0]); Assert.assertEquals("0-3", dest[1]); } @Test public void test2() { String src = "OFFER_group"; String[] dest = SplitUtil.split2(src, '$', '-'); Assert.assertEquals(1, dest.length); Assert.assertEquals("OFFER_group", dest[0]); } @Test public void test3() { String src = "OFFER_group$2"; String[] dest = SplitUtil.split2(src, '$', '-'); Assert.assertEquals(1, dest.length); Assert.assertEquals("OFFER_group[2]", dest[0]); } @Test public void test4() { String src = "offer$0-3"; String[] dest = SplitUtil.split2(src, '$', '-'); Assert.assertEquals(4, dest.length); Assert.assertEquals("offer[0]", dest[0]); Assert.assertEquals("offer[1]", dest[1]); Assert.assertEquals("offer[2]", dest[2]); Assert.assertEquals("offer[3]", dest[3]); } @Test public void test5() { Assert.assertNull(SplitUtil.split(null, '\u0000')); Assert.assertArrayEquals(new String[] {}, SplitUtil.split("", '/')); Assert.assertArrayEquals(new String[] {}, SplitUtil.split("/", '/')); Assert.assertArrayEquals(new String[] {"/"}, SplitUtil.split("/", '\"')); Assert.assertArrayEquals(new String[] {"/"}, SplitUtil.split("/\"", '\"')); } @Test public void test6() { Assert.assertNull(SplitUtil.split(null, "1", 1)); Assert.assertArrayEquals(new String[] {}, SplitUtil.split("", "foo", 1)); Assert.assertArrayEquals(new String[] {}, SplitUtil.split(" ", null, 1)); Assert.assertArrayEquals(new String[] {"foo bar"}, SplitUtil.split("foo bar", null, 1)); Assert.assertArrayEquals(new String[] {"foo", "bar"}, SplitUtil.split("foo bar", null, 2)); Assert.assertArrayEquals(new String[] {}, SplitUtil.split("1", "1", 1)); Assert.assertArrayEquals(new String[] {"foo1bar"}, SplitUtil.split("foo1bar", "1", 1)); Assert.assertArrayEquals(new String[] {"foo", "bar"}, SplitUtil.split("foo1bar", "1", 2)); Assert.assertArrayEquals(new String[] {"foo11bar"}, SplitUtil.split("foo11bar", "11", 1)); Assert.assertArrayEquals(new String[] {"foo", "bar"}, SplitUtil.split("foo11bar", "11", 2)); } @Test public void test7() { Assert.assertNull(SplitUtil.split(null, '0', '0', '0', '0')); Assert.assertArrayEquals(new String[] {}, SplitUtil.split("", '0', '0', '0', '0')); Assert.assertArrayEquals(new String[] {"0-1-2"}, SplitUtil.split("0-1-2", '3', ' ', '0', '0')); Assert.assertArrayEquals(new String[] {"011"}, SplitUtil.split("0-1-2", '-', ' ', '1', '0')); Assert.assertArrayEquals(new String[] {"0111"}, SplitUtil.split("0-1-2", '-', ' ', '1', '1')); } @Test public void test8() { Assert.assertArrayEquals(new String[] {"foo"}, SplitUtil.splitByByteSize("foo", 1)); Assert.assertArrayEquals(new String[] {"f"}, SplitUtil.splitByByteSize("f", 2)); Assert.assertArrayEquals(new String[] {"fo", "o"}, SplitUtil.splitByByteSize("foo", 2)); Assert.assertArrayEquals(new String[] {"fo", "ob", "ar"}, SplitUtil.splitByByteSize("foobar", 2)); } } ================================================ FILE: src/test/java/io/mycat/util/StringHashPerfMain.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import io.mycat.util.StringUtil; /** * @author mycat */ public class StringHashPerfMain { public static void main(String[] args) { String s = "abcdejdsalfp"; int end = s.length(); for (int i = 0; i < 10; i++) { StringUtil.hash(s, 0, end); } long loop = 10000 * 10000; long t1 = System.currentTimeMillis(); t1 = System.currentTimeMillis(); for (long i = 0; i < loop; ++i) { StringUtil.hash(s, 0, end); } long t2 = System.currentTimeMillis(); System.out.println((((t2 - t1) * 1000 * 1000) / loop) + " ns."); } } ================================================ FILE: src/test/java/io/mycat/util/StringUtilTest.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import java.io.PrintWriter; import java.io.StringWriter; import junit.framework.Assert; import org.junit.Test; import io.mycat.util.StringUtil; /** * @author mycat */ public class StringUtilTest { @Test public void test() { String oriSql = "insert into ssd (id) values (s)"; String tableName = StringUtil.getTableName(oriSql); Assert.assertEquals("ssd", tableName); } @Test public void test1() { String oriSql = "insert into ssd(id) values (s)"; String tableName = StringUtil.getTableName(oriSql); Assert.assertEquals("ssd", tableName); } @Test public void test2() { String oriSql = " insert into ssd(id) values (s)"; String tableName = StringUtil.getTableName(oriSql); Assert.assertEquals("ssd", tableName); } @Test public void test3() { String oriSql = " insert into isd(id) values (s)"; String tableName = StringUtil.getTableName(oriSql); Assert.assertEquals("isd", tableName); } @Test public void test4() { String oriSql = "INSERT INTO test_activity_input (id,vip_no"; String tableName = StringUtil.getTableName(oriSql); Assert.assertEquals("test_activity_input", tableName); } @Test public void test5() { String oriSql = " /* ApplicationName=DBeaver 3.3.1 - Main connection */ insert into employee(id,name,sharding_id) values(4, 'myhome', 10011)"; String tableName = StringUtil.getTableName(oriSql); Assert.assertEquals("employee", tableName); } @Test public void test6() { String oriSql = " /* insert int a (id, name) value(1, 'ben') */ insert into employee(id,name,sharding_id) values(4, 'myhome', 10011)"; String tableName = StringUtil.getTableName(oriSql); Assert.assertEquals("employee", tableName); } @Test public void test7() { String oriSql = " /**/ insert into employee(id,name,sharding_id) values(4, 'myhome', 10011)"; String tableName = StringUtil.getTableName(oriSql); Assert.assertEquals("employee", tableName); } @Test public void test8() { String oriSql = " /* */ insert into employee(id,name,sharding_id) values(4, 'myhome', 10011) /**/"; String tableName = StringUtil.getTableName(oriSql); Assert.assertEquals("employee", tableName); } @Test public void test9() { String oriSql = " /* hint1 insert */ /**/ /* hint3 insert */ insert into employee(id,name,sharding_id) values(4, 'myhome', 10011) /**/"; String tableName = StringUtil.getTableName(oriSql); Assert.assertEquals("employee", tableName); } @Test public void test10() { String oriSql = " /* hint1 insert */ /* // */ /* hint3 insert */ insert into employee(id,name,sharding_id) values(4, 'myhome', 10011) /**/"; String tableName = StringUtil.getTableName(oriSql); Assert.assertEquals("employee", tableName); } @Test public void test11() { String oriSql = " /* hint1 insert */ /* // */ /* hint3 insert */ insert /* */ into employee(id,name,sharding_id) values(4, 'myhome', 10011) /**/"; String tableName = StringUtil.getTableName(oriSql); Assert.assertEquals("employee", tableName); } @Test public void test12() { StringWriter sw=new StringWriter(); PrintWriter pw=new PrintWriter(sw); pw.println("insert into"); pw.println(" employee(id,name,sharding_id) values(4, 'myhome', 10011)"); pw.flush(); String oriSql = sw.toString(); String tableName = StringUtil.getTableName(oriSql); Assert.assertEquals("employee", tableName); } @Test public void test13() { StringWriter sw=new StringWriter(); PrintWriter pw=new PrintWriter(sw); pw.println("insert into"); pw.println("employee(id,name,sharding_id) values(4, 'myhome', 10011)"); pw.flush(); String oriSql = sw.toString(); String tableName = StringUtil.getTableName(oriSql); Assert.assertEquals("employee", tableName); } } ================================================ FILE: src/test/java/io/mycat/util/SyncPerfMain.java ================================================ /* * Copyright (c) 2020, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software;Designed and Developed mainly by many Chinese * opensource volunteers. you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 only, as published by the * Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Any questions about this component can be directed to it's project Web address * https://code.google.com/p/opencloudb/. * */ package io.mycat.util; import java.util.concurrent.locks.ReentrantLock; /** * @author mycat */ public class SyncPerfMain { long i = 0L; private final Object lockA = new Object(); private final ReentrantLock lockB = new ReentrantLock(); final void tLockA() { final Object lock = this.lockA; synchronized (lock) { i++; } } final void tLockB() { final ReentrantLock lock = this.lockB; lock.lock(); try { i++; } finally { lock.unlock(); } } public static void main(String[] args) { int count = 10000000; SyncPerfMain test = new SyncPerfMain(); System.currentTimeMillis(); long t1 = System.currentTimeMillis(); for (int i = 0; i < count; i++) { test.tLockA(); // test.testLockB(); } long t2 = System.currentTimeMillis(); System.out.println("take:" + (t2 - t1) + " ms."); } } ================================================ FILE: src/test/resources/autopartition-long-dupl.txt ================================================ # range start-end ,data node index 0-1000=0 1001-2000=1 2001-3000=0 3001-4000=1 ================================================ FILE: src/test/resources/autopartition-long.txt ================================================ # range start-end ,data node index 0-200M=0 200M1-400M=1 400M1-600M=2 #600M1-800M=3 #800M1-1000M=4 ================================================ FILE: src/test/resources/autopartition-long2.txt ================================================ # range start-end ,data node index 0-200M=0 200M1-400M=1 #400M1-600M=2 #600M1-800M=3 #800M1-1000M=4 ================================================ FILE: src/test/resources/config/rule.xml ================================================ id func1 2 512 ================================================ FILE: src/test/resources/config/schema.xml ================================================
select user() select user() ================================================ FILE: src/test/resources/ehcache.xml ================================================ ================================================ FILE: src/test/resources/log4j2.xml ================================================ %d{yyyy-MM-dd HH:mm:ss.SSS} %5p [%t] (%l) - %m%n ================================================ FILE: src/test/resources/partition-pattern.txt ================================================ # id partition range start-end ,data node index ###### first host configuration 1-32=0 33-64=1 65-96=2 97-128=3 ######## second host configuration 129-160=4 161-192=5 193-224=6 225-256=7 0-0=7 ================================================ FILE: src/test/resources/partition-range-mod.txt ================================================ # range start-end ,data node group size 0-200M=5 200M1-400M=1 400M1-600M=4 600M1-800M=4 800M1-1000M=6 ================================================ FILE: src/test/resources/partition_prefix_pattern.txt ================================================ # range start-end ,data node index # ASCII编码：主要划分出10个数字，小字母26，一共36个字母进行分片 # 48-57=0-9阿拉伯数字 # 64、65-90=@、A-Z # 97-122=a-z ###### first host configuration 1-4=0 5-8=1 9-12=2 13-16=3 ###### second host configuration 17-20=4 21-24=5 25-28=6 29-32=7 0-0=7 ================================================ FILE: src/test/resources/route/rule.xml ================================================ member_id func col_date by-date offer_id func2 sharding_id func1 id rang-long id rang-long2 id rang-mod id partitionByMod col_date range-date-hash id crc32slot id mod-long 2 0 sharding.txt 128 8 :8 128 8 autopartition-long.txt autopartition-long2.txt 2014-01-01 10 yyyy-MM-dd partition-range-mod.txt 2014-01-01 00:00:00 3 yyyy-MM-dd HH:mm:ss 6 2 ================================================ FILE: src/test/resources/route/schema.xml ================================================
select user() select user() select user() select 1 from dual alter session set nls_date_format='yyyy-mm-dd hh24:mi:ss' select 1 from dual alter session set nls_date_format='yyyy-mm-dd hh24:mi:ss' select 1 from SYSIBM.SYSDUMMY1 select 1 from SYSIBM.SYSDUMMY1 select 1 select 1 select 1 select 1 ================================================ FILE: src/test/resources/rule.xml ================================================ id func1 2 512 ================================================ FILE: src/test/resources/schema.xml ================================================
select user() ================================================ FILE: src/test/resources/sequence_conf.properties ================================================ GLOBAL.HISIDS= GLOBAL.MINID=1 GLOBAL.MAXID=10 GLOBAL.CURID=1 MY1.HISIDS= MY1.MINID=1001 MY1.MAXID=2000 MY1.CURID=1000 ================================================ FILE: src/test/resources/server.xml ================================================ test dbtest 11111 127.0.0.1 1 ================================================ FILE: src/test/resources/sharding.txt ================================================ 10000=0 10010=1 ================================================ FILE: src/test/resources/zk-create-test.yaml ================================================ zkURL : 127.0.0.1:2181 mycat-cluster: mycat-cluster-1: blockSQLs: sql1 : name : sql1 sql2 : name : sql2 sql3 : name : sql3 user : test : name : test password : admin readOnly : true schemas : - testdb - test mycat : name: mycat password: admin readOnly : false schemas: - testdb rule : sharding-by-enum : name : sharding-by-enum functionName : io.mycat.route.function.PartitionByFileMap column : create_time defaultnode : 0 type : 0 config : 10000 : 0 10010 : 1 sharding-by-hour : name : sharding-by-hour functionName : io.mycat.route.function.LatestMonthPartion column : createTime splitOneDay : 24 auto-sharding-long : name : auto-sharding-long column : id functionName : io.mycat.route.function.AutoPartitionByLong defaultNode : 0 config : 0-2000000 : 0 2000001-4000000 : 1 4000001-8000000 : 2 sharding-by-mod : name : sharding-by-mod column : id functionName : io.mycat.route.function.PartitionByMod count : 3 auto-sharding-rang-mod : name : auto-sharding-rang-mod column : id functionName : io.mycat.route.function.PartitionByRangeMod defaultNode : 21 config : 0-200M : 5 200M1-400M : 1 400M1-600M : 4 600M1-800M : 4 800M1-1000M : 6 auto-sharding-rang-mod : name : sharding-by-RangeDateHash column : create_time functionName : io.mycat.route.function.PartitionByRangeDateHash sBeginDate : "2014-01-01 00:00:00" sPartionDay : 3 dateFormat : yyyy-MM-dd HH:mm:ss groupPartionSize : 6 sequence: sequence-3 : current_value : 100000 increament : 100 sequence-2 : workid: 1 centerid : 2 sequence-0 : type : file sequence-1 : type : 1 config : current_value : 100000 increament : 100 sequence-mapping : T_NODE : 0 schema : TESTDB : name : TESTDB checkSQLSchema : false defaultMaxLimit : 100 travelrecord : name : travelrecord datanode : dn1,dn2,dn3 ruleName : auto-sharding-long company : name : company datanode : dn1,dn2,dn3 primaryKey : ID type : 1 #全局表为 1 goods : name : goods datanode : dn1,dn2 primaryKey : ID type : 1 #全局表为 1 hotnews : name : hotnews datanode : dn1,dn2,dn3 primaryKey : ID ruleName : sharding-by-mod employee : name : employee datanode : dn1,dn2 primaryKey : ID ruleName : sharding-by-enum customer : name : customer datanode : dn1,dn2 primaryKey : ID ruleName : sharding-by-enum orders : name : orders primaryKey : ID joinKey : customer_id parentKey : ID order_items : name : order_items joinKey : order_id parentKey : ID customer_addr : name : customer_addr joinKey : customer_id parentKey : ID offer : name : offer datanode : offer_dn$1-20 primaryKey : id ruleName : auto-sharding-rang-mod offer1 : name : offer1 datanode : offer_dn$1-36 primaryKey : id ruleName : sharding-by-RangeDateHash datanode : dn1: name : dn1 dataHost : localhost1 database : db1 dn2: name : dn2 dataHost : localhost1 database : db2 dn3: name : dn3 dataHost : localhost1 database : db3 offer_dn$0-127: name : offer_dn$0-127 dataHost : localhost1 database : db1$0-127 datahost : localhost1 : name : localhost1 maxcon : 1000 mincon : 10 balance : 0 writetype : 0 dbtype : mysql dbDriver : native switchType : 1 slaveThreshold : 100 heartbeatSQL : select user() mysqlGroup : mysql_rep_1 #集群中所有的主机信息 mycat-hosts: fz_vm1: hostname: fz_vm1 ip: 192.168.10.2 root: root password: admin mycat-zones: wh: name: 武汉中心 fz: 福州中心 #zone内mycat 实例配置,名字为mycat实例的,myid. mycat-nodes: mycat_fz_01: name: mycat_fz_01 hostname: fz_vm1 zone: fz cluster: mycat-cluster-1 weigth: 1 leader: 1 state: red systemParams: defaultsqlparser : druidparser serverport : 8066 sequncehandlertype : 1 mycat-mysqls: mysql_1: ip: 192.168.8.2 port: 3306 user: mysql password: mysql hostId: host zone: bj mysql_2: ip: 192.168.8.3 port: 3307 user: mysql password: mysql hostId: host zone: bj mysql_3: ip: 192.168.8.4 port: 3308 user: mysql password: mysql hostId: host zone: bj mycat-mysqlgroup : mysql_rep_1: name: mysql_rep_1 repType: 0 zone: bj servers: - mysql_1 - mysql_2 - mysql_3 cur-write-server: mysql_1 auto-write-switch: true heartbeatSQL : select user() ================================================ FILE: test-output/Default suite/Default test.html ================================================ TestNG: Default test

Default test

Tests passed/Failed/Skipped:0/0/0
Started on:Mon Mar 17 10:49:43 CST 2014
Total time:0 seconds (12 ms)
Included groups:
Excluded groups:

(Hover the method name to see the test class name)

================================================ FILE: test-output/Default suite/Default test.xml ================================================ ================================================ FILE: test-output/emailable-report.html ================================================ TestNG Report
Test# Passed# Skipped# FailedTime (ms)Included GroupsExcluded Groups
Default suite
Default test00012
ClassMethodStartTime (ms)
Default suite

Default test

================================================ FILE: test-output/index.html ================================================ TestNG reports
Test results
1 suite
All suites
Default suite
Info
Results
  • 0 methods,
C:\Users\wuzhih\AppData\Local\Temp\testng-eclipse--90647589\testng-customsuite.xml 
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd">
<suite name="Default suite">
  <test verbose="2" name="Default test">
    <classes>
      <class name="io.mycat.route.function.PartitionByStringTest"/>
    </classes>
  </test> <!-- Default test -->
</suite> <!-- Default suite -->
Tests for Default suite
  • Default test (1 class)
Groups for Default suite
Times for Default suite
Reporter output for Default suite
0 ignored methods
Methods in chronological order
================================================ FILE: test-output/old/Default suite/Default test.properties ================================================ [SuiteResult context=Default test] ================================================ FILE: test-output/old/Default suite/classes.html ================================================
Class name Method name Groups
================================================ FILE: test-output/old/Default suite/groups.html ================================================

Groups used for this test run

================================================ FILE: test-output/old/Default suite/index.html ================================================ Results for Default suite ================================================ FILE: test-output/old/Default suite/main.html ================================================ Results for Default suite Select a result on the left-hand pane. ================================================ FILE: test-output/old/Default suite/methods-alphabetical.html ================================================

Methods run, sorted chronologically

>> means before, << means after


Default suite

(Hover the method name to see the test class name)

================================================ FILE: test-output/old/Default suite/methods-not-run.html ================================================

Methods that were not run

================================================ FILE: test-output/old/Default suite/methods.html ================================================

Methods run, sorted chronologically

>> means before, << means after


Default suite

(Hover the method name to see the test class name)

================================================ FILE: test-output/old/Default suite/reporter-output.html ================================================

Reporter output

================================================ FILE: test-output/old/Default suite/testng.xml.html ================================================ testng.xml for Default suite<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd">
<suite name="Default suite">
  <test verbose="2" name="Default test">
    <classes>
      <class name="io.mycat.route.function.PartitionByStringTest"/>
    </classes>
  </test> <!-- Default test -->
</suite> <!-- Default suite -->
================================================ FILE: test-output/old/Default suite/toc.html ================================================ Results for Default suite

Results for
Default suite

1 test 0 class 0 method:
  chronological
  alphabetical
  not run (0)
0 group reporter output testng.xml

Default test (0/0/0) Results
================================================ FILE: test-output/old/index.html ================================================ Test results

Test results

SuitePassedFailedSkippedtestng.xml
Total000 
Default suite 000Link
================================================ FILE: test-output/testng-reports.css ================================================ body { margin: 0px 0px 5px 5px; } ul { margin: 0px; } li { list-style-type: none; } a { text-decoration: none; } a:hover { text-decoration: underline; } .navigator-selected { background: #ffa500; } .wrapper { position: absolute; top: 60px; bottom: 0; left: 400px; right: 0; overflow: auto; } .navigator-root { position: absolute; top: 60px; bottom: 0; left: 0; width: 400px; overflow-y: auto; } .suite { margin: 0px 10px 10px 0px; background-color: #fff8dc; } .suite-name { padding-left: 10px; font-size: 25px; font-family: Times; } .main-panel-header { padding: 5px; background-color: #9FB4D9; //afeeee; font-family: monospace; font-size: 18px; } .main-panel-content { padding: 5px; margin-bottom: 10px; background-color: #DEE8FC; //d0ffff; } .rounded-window { border-radius: 10px; border-style: solid; border-width: 1px; } .rounded-window-top { border-top-right-radius: 10px 10px; border-top-left-radius: 10px 10px; border-style: solid; border-width: 1px; overflow: auto; } .light-rounded-window-top { border-top-right-radius: 10px 10px; border-top-left-radius: 10px 10px; } .rounded-window-bottom { border-style: solid; border-width: 0px 1px 1px 1px; border-bottom-right-radius: 10px 10px; border-bottom-left-radius: 10px 10px; overflow: auto; } .method-name { font-size: 12px; font-family: monospace; } .method-content { border-style: solid; border-width: 0px 0px 1px 0px; margin-bottom: 10; padding-bottom: 5px; width: 80%; } .parameters { font-size: 14px; font-family: monospace; } .stack-trace { white-space: pre; font-family: monospace; font-size: 12px; font-weight: bold; margin-top: 0px; margin-left: 20px; } .testng-xml { font-family: monospace; } .method-list-content { margin-left: 10px; } .navigator-suite-content { margin-left: 10px; font: 12px 'Lucida Grande'; } .suite-section-title { margin-top: 10px; width: 80%; border-style: solid; border-width: 1px 0px 0px 0px; font-family: Times; font-size: 18px; font-weight: bold; } .suite-section-content { list-style-image: url(bullet_point.png); } .top-banner-root { position: absolute; top: 0; height: 45px; left: 0; right: 0; padding: 5px; margin: 0px 0px 5px 0px; background-color: #0066ff; font-family: Times; color: #fff; text-align: center; } .top-banner-title-font { font-size: 25px; } .test-name { font-family: 'Lucida Grande'; font-size: 16px; } .suite-icon { padding: 5px; float: right; height: 20; } .test-group { font: 20px 'Lucida Grande'; margin: 5px 5px 10px 5px; border-width: 0px 0px 1px 0px; border-style: solid; padding: 5px; } .test-group-name { font-weight: bold; } .method-in-group { font-size: 16px; margin-left: 80px; } table.google-visualization-table-table { width: 100%; } .reporter-method-name { font-size: 14px; font-family: monospace; } .reporter-method-output-div { padding: 5px; margin: 0px 0px 5px 20px; font-size: 12px; font-family: monospace; border-width: 0px 0px 0px 1px; border-style: solid; } .ignored-class-div { font-size: 14px; font-family: monospace; } .ignored-methods-div { padding: 5px; margin: 0px 0px 5px 20px; font-size: 12px; font-family: monospace; border-width: 0px 0px 0px 1px; border-style: solid; } .border-failed { border-top-left-radius: 10px 10px; border-bottom-left-radius: 10px 10px; border-style: solid; border-width: 0px 0px 0px 10px; border-color: #f00; } .border-skipped { border-top-left-radius: 10px 10px; border-bottom-left-radius: 10px 10px; border-style: solid; border-width: 0px 0px 0px 10px; border-color: #edc600; } .border-passed { border-top-left-radius: 10px 10px; border-bottom-left-radius: 10px 10px; border-style: solid; border-width: 0px 0px 0px 10px; border-color: #19f52d; } .times-div { text-align: center; padding: 5px; } .suite-total-time { font: 16px 'Lucida Grande'; } .configuration-suite { margin-left: 20px; } .configuration-test { margin-left: 40px; } .configuration-class { margin-left: 60px; } .configuration-method { margin-left: 80px; } .test-method { margin-left: 100px; } .chronological-class { background-color: #0ccff; border-style: solid; border-width: 0px 0px 1px 1px; } .method-start { float: right; } .chronological-class-name { padding: 0px 0px 0px 5px; color: #008; } .after, .before, .test-method { font-family: monospace; font-size: 14px; } .navigator-suite-header { font-size: 22px; margin: 0px 10px 5px 0px; background-color: #deb887; text-align: center; } .collapse-all-icon { padding: 5px; float: right; } ================================================ FILE: test-output/testng-reports.js ================================================ $(document).ready(function() { $('a.navigator-link').click(function() { // Extract the panel for this link var panel = getPanelName($(this)); // Mark this link as currently selected $('.navigator-link').parent().removeClass('navigator-selected'); $(this).parent().addClass('navigator-selected'); showPanel(panel); }); installMethodHandlers('failed'); installMethodHandlers('skipped'); installMethodHandlers('passed', true); // hide passed methods by default $('a.method').click(function() { showMethod($(this)); return false; }); // Hide all the panels and display the first one (do this last // to make sure the click() will invoke the listeners) $('.panel').hide(); $('.navigator-link').first().click(); // Collapse/expand the suites $('a.collapse-all-link').click(function() { var contents = $('.navigator-suite-content'); if (contents.css('display') == 'none') { contents.show(); } else { contents.hide(); } }); }); // The handlers that take care of showing/hiding the methods function installMethodHandlers(name, hide) { function getContent(t) { return $('.method-list-content.' + name + "." + t.attr('panel-name')); } function getHideLink(t, name) { var s = 'a.hide-methods.' + name + "." + t.attr('panel-name'); return $(s); } function getShowLink(t, name) { return $('a.show-methods.' + name + "." + t.attr('panel-name')); } function getMethodPanelClassSel(element, name) { var panelName = getPanelName(element); var sel = '.' + panelName + "-class-" + name; return $(sel); } $('a.hide-methods.' + name).click(function() { var w = getContent($(this)); w.hide(); getHideLink($(this), name).hide(); getShowLink($(this), name).show(); getMethodPanelClassSel($(this), name).hide(); }); $('a.show-methods.' + name).click(function() { var w = getContent($(this)); w.show(); getHideLink($(this), name).show(); getShowLink($(this), name).hide(); showPanel(getPanelName($(this))); getMethodPanelClassSel($(this), name).show(); }); if (hide) { $('a.hide-methods.' + name).click(); } else { $('a.show-methods.' + name).click(); } } function getHashForMethod(element) { return element.attr('hash-for-method'); } function getPanelName(element) { return element.attr('panel-name'); } function showPanel(panelName) { $('.panel').hide(); var panel = $('.panel[panel-name="' + panelName + '"]'); panel.show(); } function showMethod(element) { var hashTag = getHashForMethod(element); var panelName = getPanelName(element); showPanel(panelName); var current = document.location.href; var base = current.substring(0, current.indexOf('#')) document.location.href = base + '#' + hashTag; var newPosition = $(document).scrollTop() - 65; $(document).scrollTop(newPosition); } function drawTable() { for (var i = 0; i < suiteTableInitFunctions.length; i++) { window[suiteTableInitFunctions[i]](); } for (var k in window.suiteTableData) { var v = window.suiteTableData[k]; var div = v.tableDiv; var data = v.tableData var table = new google.visualization.Table(document.getElementById(div)); table.draw(data, { showRowNumber : false }); } } ================================================ FILE: test-output/testng-results.xml ================================================ ================================================ FILE: test-output/testng.css ================================================ .invocation-failed, .test-failed { background-color: #DD0000; } .invocation-percent, .test-percent { background-color: #006600; } .invocation-passed, .test-passed { background-color: #00AA00; } .invocation-skipped, .test-skipped { background-color: #CCCC00; } .main-page { font-size: x-large; } ================================================ FILE: tmlogs/tmlog-1.log ================================================ ================================================ FILE: version.txt ================================================ BuildTime 2020-11-04 09:46:07 GitVersion 277758d6dd4193529ebab066571512f8e8383d1d MavenVersion 1.6.7.6-release GitUrl https://github.com/MyCATApache/Mycat-Server.git MyCatSite http://www.mycat.org.cn QQGroup 106088787 ================================================ FILE: version.txt.template ================================================ BuildTime @buildtime@ GitVersion @buildnumber@ MavenVersion @pomversion@ GitUrl @giturl@ MyCatSite @mycatsite@ QQGroup @qqgroup@