diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..814913e --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,282 @@ +# $%BEGINLICENSE%$ +# Copyright (c) 2009, 2013, Oracle and/or its affiliates. All rights reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2 of the +# License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA +# 02110-1301 USA +# +# $%ENDLICENSE%$ + +PROJECT(cetus C) + +SET(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake) + +CMAKE_MINIMUM_REQUIRED(VERSION 2.6) + +INCLUDE(CheckIncludeFiles) +INCLUDE(CheckFunctionExists) +INCLUDE(CheckLibraryExists) +INCLUDE(FindPkgConfig) +INCLUDE(CheckTypeSize) +INCLUDE(ChassisPlugin) +INCLUDE(ChassisInstall) +INCLUDE(CTest) +INCLUDE(Tar) + +ENABLE_TESTING() + +set(CETUS_TOOLS_DIR ${PROJECT_SOURCE_DIR}/tools) + +OPTION(ENABLE_GCOV "Enable gcov (debug, Linux builds only)" OFF) + +IF (ENABLE_GCOV AND NOT WIN32 AND NOT APPLE) + MESSAGE(STATUS "***************building gcov version*************") + SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fprofile-arcs -ftest-coverage") + SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -fprofile-arcs -ftest-coverage") + SET(CMAKE_EXE_LINKER_FLAGS_DEBUG + "${CMAKE_EXE_LINKER_FLAGS_DEBUG} -fprofile-arcs -ftest-coverage -lgcov") +ENDIF() + +option(NETWORK_DEBUG_TRACE_IO "if NETWORK_DEBUG_TRACE_IO is defined, the network layer will log the raw MySQL packets as log-level debug") + +option(NETWORK_DEBUG_TRACE_EVENT "if NETWORK_DEBUG_TRACE_EVENT is defined, cetus will log the abnormal event") + +option(NETWORK_DEBUG_TRACE_STATE_CHANGES "if NETWORK_DEBUG_TRACE_STATE_CHANGES is defined the state engine for the mysql protocol will log all state changes") + +option(USE_GLIB_DEBUG_LOG "g_debug() is default defined to nothing, change to ON if you want to use g_debug()") + +EXECUTE_PROCESS(COMMAND git describe --tags + TIMEOUT 5 + OUTPUT_VARIABLE GIT_REVISION + OUTPUT_STRIP_TRAILING_WHITESPACE + ) +MESSAGE(STATUS "building from ${GIT_REVISION}") +SET(CPACK_PACKAGE_VERSION_MAJOR "0") +SET(CPACK_PACKAGE_VERSION_MINOR "8") +SET(CPACK_PACKAGE_VERSION_PATCH "8") +SET(CPACK_PACKAGE_VENDOR "MySQL") +SET(PACKAGE_NAME cetus) + +## SET(CPACK_*) before the INCLUDE(CPack) +INCLUDE(CPack) + +MESSAGE(STATUS "libevent at libs:${EVENT_LIBRARY_DIRS} incl:${EVENT_INCLUDE_DIRS}") +MESSAGE(STATUS "gthread at libs:${GTHREAD_LIBRARY_DIRS} incl:${GTHREAD_INCLUDE_DIRS}") +MESSAGE(STATUS "gmodule at libs:${GMODULE_LIBRARY_DIRS} incl:${GMODULE_INCLUDE_DIRS}") + +## if the paths are not set, use pkg-config to fetch them +IF(NOT GLIB_DEV_BASE_DIR) + SET(GMODULE_INCLUDE_DIRS CACHE PATH "gmodule-2.0 include dir [see pkg-config gmodule-2.0 --cflags]") + SET(GMODULE_LIBRARY_DIRS CACHE PATH "gmodule-2.0 library dir [see pkg-config gmodule-2.0 --libs]") + SET(GTHREAD_INCLUDE_DIRS CACHE PATH "gthread-2.0 include dir [see pkg-config gthread-2.0 --cflags]") + SET(GTHREAD_LIBRARY_DIRS CACHE PATH "gthread-2.0 library dir [see pkg-config gthread-2.0 --libs]") +ENDIF(NOT GLIB_DEV_BASE_DIR) +SET(MYSQL_INCLUDE_DIRS CACHE PATH "MySQL include dir") +SET(MYSQL_LIBRARY_DIRS CACHE PATH "MySQL library dir") + +IF (NOT EVENT_BASE_DIR) + SET(EVENT_INCLUDE_DIRS CACHE PATH "libevent include dir") + SET(EVENT_LIBRARY_DIRS CACHE PATH "libevent library dir") +ENDIF(NOT EVENT_BASE_DIR) + +MESSAGE(STATUS "libevent at libs:${EVENT_LIBRARY_DIRS} incl:${EVENT_INCLUDE_DIRS}") +MESSAGE(STATUS "glib2.0 at libs:${GLIB_LIBRARY_DIRS} incl:${GLIB_INCLUDE_DIRS}") +MESSAGE(STATUS "gthread at libs:${GTHREAD_LIBRARY_DIRS} incl:${GTHREAD_INCLUDE_DIRS}") +MESSAGE(STATUS "gmodule at libs:${GMODULE_LIBRARY_DIRS} incl:${GMODULE_INCLUDE_DIRS}") + +IF(NOT GLIB_INCLUDE_DIRS) + SET(__pkg_config_checked_GLIB 0) + PKG_CHECK_MODULES(GLIB REQUIRED glib-2.0>=2.16) + ADD_DEFINITIONS(-DHAVE_GLIB) +ENDIF(NOT GLIB_INCLUDE_DIRS) + +IF(NOT GMODULE_INCLUDE_DIRS) + PKG_CHECK_MODULES(GMODULE REQUIRED gmodule-2.0>=2.16) + ADD_DEFINITIONS(-DHAVE_GMODULE) +ENDIF(NOT GMODULE_INCLUDE_DIRS) + +IF(NOT GTHREAD_INCLUDE_DIRS) + PKG_CHECK_MODULES(GTHREAD REQUIRED gthread-2.0>=2.16) +ENDIF(NOT GTHREAD_INCLUDE_DIRS) + +MACRO(_mysql_config VAR _regex _opt) + EXECUTE_PROCESS(COMMAND ${MYSQL_CONFIG_EXECUTABLE} ${_opt} + OUTPUT_VARIABLE _mysql_config_output + ) + + SET(_var ${_mysql_config_output}) + STRING(REGEX MATCHALL "${_regex}([^ ]+)" _mysql_config_output "${_mysql_config_output}") + STRING(REGEX REPLACE "^[ \t]+" "" _mysql_config_output "${_mysql_config_output}") + STRING(REGEX REPLACE "[\r\n]$" "" _mysql_config_output "${_mysql_config_output}") + STRING(REGEX REPLACE "${_regex}" "" _mysql_config_output "${_mysql_config_output}") + SEPARATE_ARGUMENTS(_mysql_config_output) + SET(${VAR} ${_mysql_config_output}) +ENDMACRO(_mysql_config _regex _opt) + +IF(NOT MYSQL_INCLUDE_DIRS) + FIND_PROGRAM(MYSQL_CONFIG_EXECUTABLE NAMES mysql_config DOC "full path of mysql_config") + IF(NOT MYSQL_CONFIG_EXECUTABLE) + MESSAGE(SEND_ERROR "mysql_config wasn't found, -DMYSQL_CONFIG_EXECUTABLE=...") + ENDIF(NOT MYSQL_CONFIG_EXECUTABLE) + + _MYSQL_CONFIG(MYSQL_INCLUDE_DIRS "(^| )-I" "--include") + _MYSQL_CONFIG(MYSQL_LIBRARIES "(^| )-l" "--libs") + _MYSQL_CONFIG(MYSQL_LIBRARY_DIRS "(^| )-L" "--libs") +ELSE(NOT MYSQL_INCLUDE_DIRS) + SET(MYSQL_LIBRARIES libmysql) +ENDIF(NOT MYSQL_INCLUDE_DIRS) + +SET(CMAKE_REQUIRED_INCLUDES + ${EVENT_INCLUDE_DIRS} + ${MYSQL_INCLUDE_DIRS} + ${GLIB_INCLUDE_DIRS} + ${GTHREAD_INCLUDE_DIRS} +) + +MESSAGE(STATUS "checking in dirs: ${CMAKE_REQUIRED_INCLUDES}") + +CHECK_INCLUDE_FILES("sys/types.h;event.h" HAVE_EVENT_H) +CHECK_INCLUDE_FILES(inttypes.h HAVE_INTTYPES_H) +CHECK_INCLUDE_FILES(netinet/in.h HAVE_NETINET_IN_H) +CHECK_INCLUDE_FILES(net/if.h HAVE_NET_IF_H) +CHECK_INCLUDE_FILES(net/if_dl.h HAVE_NET_IF_DL_H) +CHECK_INCLUDE_FILES(stddef.h HAVE_STDDEF_H) +CHECK_INCLUDE_FILES(stdint.h HAVE_STDINT_H) +CHECK_INCLUDE_FILES(stdlib.h HAVE_STDLIB_H) +CHECK_INCLUDE_FILES(signal.h HAVE_SIGNAL_H) +CHECK_INCLUDE_FILES(syslog.h HAVE_SYSLOG_H) +CHECK_INCLUDE_FILES(sys/filio.h HAVE_SYS_FILIO_H) +CHECK_INCLUDE_FILES(sys/ioctl.h HAVE_SYS_IOCTL_H) +CHECK_INCLUDE_FILES(sys/param.h HAVE_SYS_PARAM_H) +CHECK_INCLUDE_FILES(sys/resource.h HAVE_SYS_RESOURCE_H) +CHECK_INCLUDE_FILES(sys/socket.h HAVE_SYS_SOCKET_H) +CHECK_INCLUDE_FILES(sys/sockio.h HAVE_SYS_SOCKIO_H) +CHECK_INCLUDE_FILES(sys/time.h HAVE_SYS_TIME_H) +CHECK_INCLUDE_FILES(sys/types.h HAVE_SYS_TYPES_H) +CHECK_INCLUDE_FILES(sys/uio.h HAVE_SYS_UIO_H) +CHECK_INCLUDE_FILES(sys/un.h HAVE_SYS_UN_H) +CHECK_INCLUDE_FILES(time.h HAVE_TIME_H) +CHECK_INCLUDE_FILES(unistd.h HAVE_UNISTD_H) +CHECK_INCLUDE_FILES(mysql.h HAVE_MYSQL_H) +CHECK_INCLUDE_FILES(glib.h HAVE_GLIB_H) +CHECK_INCLUDE_FILES(glib/gthread.h HAVE_GTHREAD_H) +CHECK_INCLUDE_FILES(pwd.h HAVE_PWD_H) + +CHECK_FUNCTION_EXISTS(inet_ntop HAVE_INET_NTOP) +CHECK_FUNCTION_EXISTS(getcwd HAVE_GETCWD) +CHECK_FUNCTION_EXISTS(signal HAVE_SIGNAL) +CHECK_FUNCTION_EXISTS(strerror HAVE_STRERROR) +CHECK_FUNCTION_EXISTS(srandom HAVE_SRANDOM) +CHECK_FUNCTION_EXISTS(writev HAVE_WRITEV) +CHECK_FUNCTION_EXISTS(sigaction HAVE_SIGACTION) +CHECK_FUNCTION_EXISTS(getaddrinfo HAVE_GETADDRINFO) +# check for gthread actually being present +CHECK_LIBRARY_EXISTS(gthread-2.0 g_thread_init "${GTHREAD_LIBRARY_DIRS}" HAVE_GTHREAD) +#SET(OLD_CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES}) +#SET(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${GTHREAD_LIBRARIES}) +# fails for some reason use check_library_exists instead and hope it's better +#CHECK_FUNCTION_EXISTS(g_thread_init HAVE_GTHREAD) +#SET(CMAKE_REQUIRED_LIBRARIES ${OLD_CMAKE_REQUIRED_LIBRARIES}) +find_package(OpenSSL QUIET) +if (OPENSSL_FOUND) + message("-- OpenSSL version: ${OPENSSL_VERSION}") + set(HAVE_OPENSSL 1) +else(OPENSSL_FOUND) + message("-- OpenSSL not found") + set(OPENSSL_LIBRARIES "") +endif(OPENSSL_FOUND) + +IF(${HAVE_SYS_TYPES_H}) + SET(CMAKE_EXTRA_INCLUDE_FILES sys/types.h) + CHECK_TYPE_SIZE(ulong HAVE_ULONG) + SET(CMAKE_EXTRA_INCLUDE_FILES ) +ENDIF(${HAVE_SYS_TYPES_H}) + +IF(${HAVE_SYS_SOCKET_H}) + SET(CMAKE_EXTRA_INCLUDE_FILES sys/socket.h) + CHECK_TYPE_SIZE(socklen_t HAVE_SOCKLEN_T) + SET(CMAKE_EXTRA_INCLUDE_FILES ) +ENDIF(${HAVE_SYS_SOCKET_H}) + + +IF(${HAVE_SYS_RESOURCE_H}) + SET(CMAKE_EXTRA_INCLUDE_FILES sys/resource.h) + CHECK_TYPE_SIZE(rlim_t SIZEOF_RLIM_T) + SET(CMAKE_EXTRA_INCLUDE_FILES ) +ENDIF(${HAVE_SYS_RESOURCE_H}) + +IF(EVENT_LIBRARY_DIRS) + FIND_LIBRARY(EVENT_LIBRARIES + NAMES event + PATHS ${EVENT_LIBRARY_DIRS} + NO_DEFAULT_PATH + ) +ELSE(EVENT_LIBRARY_DIRS) + FIND_LIBRARY(EVENT_LIBRARIES event) +ENDIF(EVENT_LIBRARY_DIRS) + +SET(OLD_CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES}) +SET(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${EVENT_LIBRARIES}) +MESSAGE(STATUS "CMAKE_REQUIRED_LIBRARIES is ${CMAKE_REQUIRED_LIBRARIES}") +CHECK_FUNCTION_EXISTS(event_base_new HAVE_EVENT_BASE_NEW) +CHECK_FUNCTION_EXISTS(event_base_free HAVE_EVENT_BASE_FREE) +SET(CMAKE_REQUIRED_LIBRARIES ${OLD_CMAKE_REQUIRED_LIBRARIES}) + +IF(GIT_REVISION) + ADD_DEFINITIONS(-DCHASSIS_BUILD_TAG="${GIT_REVISION}") +ENDIF(GIT_REVISION) + +CONFIGURE_FILE(config.h.cmake config.h) +ADD_DEFINITIONS(-DHAVE_CONFIG_H) + +SET(PACKAGE_VERSION_ID "(${CPACK_PACKAGE_VERSION_MAJOR} << 16 | ${CPACK_PACKAGE_VERSION_MINOR} << 8 | ${CPACK_PACKAGE_VERSION_PATCH})") +SET(PACKAGE_VERSION_STRING "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}") +SET(PACKAGE_VERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}") +SET(PACKAGE_STRING "${CMAKE_PROJECT_NAME} ${CPACK_PACKAGE_VERSION}") +ADD_DEFINITIONS( + -DPACKAGE_VERSION_ID=${PACKAGE_VERSION_ID} + -DPACKAGE_VERSION="${PACKAGE_VERSION}" + -DPACKAGE_STRING="${PACKAGE_STRING}" + -DPACKAGE="${CMAKE_PROJECT_NAME}" +) + +ADD_SUBDIRECTORY(src) +ADD_SUBDIRECTORY(plugins) +IF(EXISTS tests) + ADD_SUBDIRECTORY(tests) +ENDIF(EXISTS tests) +IF(EXISTS examples) + ADD_SUBDIRECTORY(examples) +ENDIF(EXISTS examples) +ADD_SUBDIRECTORY(lib) + +CONFIGURE_FILE(mysql-chassis.pc.cmake mysql-chassis.pc @ONLY) +CONFIGURE_FILE(cetus.pc.cmake cetus.pc @ONLY) +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/mysql-chassis.pc + DESTINATION lib/pkgconfig/ + ) +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/cetus.pc + DESTINATION lib/pkgconfig/ + ) +INSTALL(FILES README_XA + DESTINATION logs + ) +install(FILES + doc/users.json.example + doc/variables.json.example + doc/sharding.json.example + doc/shard.conf.example + doc/proxy.conf.example + DESTINATION conf + ) diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..d159169 --- /dev/null +++ b/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/COPYING.rtf b/COPYING.rtf new file mode 100644 index 0000000..e58023b --- /dev/null +++ b/COPYING.rtf @@ -0,0 +1,343 @@ +{\rtf\ansi\deff0 +{\fonttbl{\f0\fswiss Courier New;}} +\viewkind4\uc1\pard\lang1031\f0\fs15 + GNU GENERAL PUBLIC LICENSE\line + Version 2, June 1991\line +\par + Copyright (C) 1989, 1991 Free Software Foundation, Inc.,\line + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA\line + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed.\line +\par + Preamble\line +\par + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too.\line +\par + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things.\line +\par + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it.\line +\par + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights.\line +\par + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software.\line +\par + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations.\line +\par + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all.\line +\par + The precise terms and conditions for copying, distribution and +modification follow.\line +\par + GNU GENERAL PUBLIC LICENSE\line + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION\line +\par + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you".\line +\par +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does.\line +\par + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program.\line +\par +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee.\line +\par + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions:\line +\par + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change.\line +\par + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License.\line +\par + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.)\line +\par +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it.\line +\par +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program.\line +\par +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License.\line +\par + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following:\line +\par + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or,\line +\par + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or,\line +\par + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.)\line +\par +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable.\line +\par +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code.\line +\par + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance.\line +\par + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it.\line +\par + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License.\line +\par + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program.\line +\par +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances.\line +\par +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice.\line +\par +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License.\line +\par + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License.\line +\par + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns.\line +\par +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation.\line +\par + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. +\par + NO WARRANTY\line +\par + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION.\line +\par + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES.\line +\par + END OF TERMS AND CONDITIONS\line +\par + How to Apply These Terms to Your New Programs\line +\par + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms.\line +\par + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found.\line +\par + + Copyright (C) \line +\par + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version.\line +\par + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details.\line +\par + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\line +\par +Also add information on how to contact you by electronic and paper mail.\line +\par +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode:\line +\par + Gnomovision version 69, Copyright (C) year name of author\line + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.\line + This is free software, and you are welcome to redistribute it\line + under certain conditions; type `show c' for details.\line +\par +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program.\line +\par +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names:\line +\par + Yoyodyne, Inc., hereby disclaims all copyright interest in the program\line + `Gnomovision' (which makes passes at compilers) written by James Hacker.\line +\par + , 1 April 1989\line + Ty Coon, President of Vice\line +\par +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License.\line +} diff --git a/README.md b/README.md index a338a80..7376d14 100644 --- a/README.md +++ b/README.md @@ -1,2 +1 @@ # cetus -A powerful Proxy for MySQL diff --git a/cetus.pc.cmake b/cetus.pc.cmake new file mode 100644 index 0000000..9709476 --- /dev/null +++ b/cetus.pc.cmake @@ -0,0 +1,12 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=@CMAKE_INSTALL_PREFIX@ +libdir=@CMAKE_INSTALL_PREFIX@/lib +pkglibdir=${libdir}/cetus +plugindir=${pkglibdir}/plugins + +Name: MySQL Proxy +Version: @PACKAGE_VERSION_STRING@ +Description: MySQL Proxy +URL: http://forge.mysql.com/wiki/MySQL_Proxy +Requires: glib-2.0 >= 2.16, mysql-chassis >= @PACKAGE_VERSION_STRING@ +Libs: -L${libdir} -lmysql-chassis-proxy diff --git a/cmake/ChassisInstall.cmake b/cmake/ChassisInstall.cmake new file mode 100644 index 0000000..479c74a --- /dev/null +++ b/cmake/ChassisInstall.cmake @@ -0,0 +1,79 @@ +# $%BEGINLICENSE%$ +# Copyright (c) 2009, 2013, Oracle and/or its affiliates. All rights reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2 of the +# License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA +# 02110-1301 USA +# +# $%ENDLICENSE%$ + +## print a few properties of a target +MACRO(PRINT_TARGET_PROPERTIES target) + MESSAGE(STATUS "properties of target ${target}") + FOREACH(prop "COMPILE_FLAGS" "IMPORT_PREFIX" "IMPORT_SUFFIX" "TYPE" "LOCATION") + GET_TARGET_PROPERTY(value ${target} ${prop}) + MESSAGE(STATUS " ${prop} = ${value}") + ENDFOREACH() + +ENDMACRO(PRINT_TARGET_PROPERTIES target) + +## install a shared-lib or executable target incl. .pdb files on win32 +## +## works around a bug in cmake that doesn't include TARGETS in cpack +MACRO(CHASSIS_INSTALL_TARGET target) + IF(WIN32) + GET_TARGET_PROPERTY(built_location ${target} LOCATION) + GET_TARGET_PROPERTY(type ${target} TYPE) + STRING(REPLACE "$(OutDir)" "${CMAKE_BUILD_TYPE}" built_location ${built_location}) + IF(type MATCHES "SHARED_LIBRARY") + STRING(REPLACE ".dll" ".lib" lib_location ${built_location}) + INSTALL(FILES + ${built_location} + DESTINATION bin + ) + INSTALL(FILES + ${lib_location} + DESTINATION lib + ) + IF(CMAKE_BUILD_TYPE MATCHES "RelWithDebInfo") + STRING(REPLACE ".dll" ".pdb" pdb_location ${built_location}) + INSTALL(FILES + ${pdb_location} + DESTINATION bin + ) + ENDIF() + ENDIF() + IF(type MATCHES "EXECUTABLE") + INSTALL(FILES + ${built_location} + DESTINATION bin + ) + IF(CMAKE_BUILD_TYPE MATCHES "RelWithDebInfo") + STRING(REPLACE ".exe" ".pdb" pdb_location ${built_location}) + INSTALL(FILES + ${pdb_location} + DESTINATION bin + ) + ENDIF() + ENDIF() + ELSE(WIN32) + INSTALL(TARGETS ${target} + RUNTIME DESTINATION bin + ARCHIVE DESTINATION lib + LIBRARY DESTINATION lib + ) + ENDIF(WIN32) +ENDMACRO(CHASSIS_INSTALL_TARGET target) + + diff --git a/cmake/ChassisPlugin.cmake b/cmake/ChassisPlugin.cmake new file mode 100644 index 0000000..db3343c --- /dev/null +++ b/cmake/ChassisPlugin.cmake @@ -0,0 +1,43 @@ +# $%BEGINLICENSE%$ +# Copyright (c) 2008, 2013, Oracle and/or its affiliates. All rights reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2 of the +# License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA +# 02110-1301 USA +# +# $%ENDLICENSE%$ + +MACRO(CHASSIS_PLUGIN_INSTALL _plugin_name) + IF(NOT WIN32) + INSTALL(TARGETS ${_plugin_name} + DESTINATION lib/cetus/plugins) + ELSE(NOT WIN32) + ## on win32 the chassis plugins gets prefixed with plugin- and end up in bin/ + GET_TARGET_PROPERTY(built_location ${_plugin_name} LOCATION) + STRING(REPLACE "$(OutDir)" "${CMAKE_BUILD_TYPE}" built_location ${built_location}) + INSTALL(FILES ${built_location} + DESTINATION bin/ + RENAME plugin-${_plugin_name}${CMAKE_SHARED_LIBRARY_SUFFIX} + ) + ## install the .pdb too + IF(CMAKE_BUILD_TYPE MATCHES "RelWithDebInfo") + STRING(REPLACE ${CMAKE_SHARED_LIBRARY_SUFFIX} ".pdb" pdb_location ${built_location}) + INSTALL(FILES + ${pdb_location} + DESTINATION bin + ) + ENDIF() + ENDIF(NOT WIN32) +ENDMACRO(CHASSIS_PLUGIN_INSTALL _plugin_name) + diff --git a/cmake/Tar.cmake b/cmake/Tar.cmake new file mode 100644 index 0000000..a1c9b87 --- /dev/null +++ b/cmake/Tar.cmake @@ -0,0 +1,73 @@ +# $%BEGINLICENSE%$ +# Copyright (c) 2008, 2013, Oracle and/or its affiliates. All rights reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2 of the +# License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA +# 02110-1301 USA +# +# $%ENDLICENSE%$ + +MACRO(TAR_UNPACK _file _wd) + FIND_PROGRAM(GTAR_EXECUTABLE NAMES gtar tar) + FIND_PROGRAM(GZIP_EXECUTABLE NAMES gzip) + + MESSAGE(STATUS "gtar: ${GTAR_EXECUTABLE}") + MESSAGE(STATUS "gzip: ${GZIP_EXECUTABLE}") + + IF(GTAR_EXECUTABLE AND GZIP_EXECUTABLE) + # On Windows gzip -cd $file | tar xvf - sometimes reports + # "Broken pipe" which causes cmake to assume an error has occured + # In fact, this is harmless, but because of it, this is done in + # two steps + MESSAGE(STATUS "unzipping ${_file} with ${GZIP_EXECUTABLE}") + EXECUTE_PROCESS(COMMAND ${GZIP_EXECUTABLE} "-cd" ${_file} + OUTPUT_FILE "${_file}.tar" + WORKING_DIRECTORY ${_wd} + ERROR_VARIABLE _err) + IF(_err) + MESSAGE(SEND_ERROR "GZIP_UNPACK()-err: ${_err}") + ENDIF(_err) + # untar after successful unzip + MESSAGE(STATUS "untarring ${_file}.tar with ${GTAR_EXECUTABLE}") + EXECUTE_PROCESS(COMMAND ${GTAR_EXECUTABLE} "xvf" "-" + INPUT_FILE "${_file}.tar" + WORKING_DIRECTORY ${_wd} + OUTPUT_VARIABLE _out + ERROR_VARIABLE _err) + IF(_err) + MESSAGE(SEND_ERROR "TAR_UNPACK()-err: ${_err}") + ENDIF(_err) + IF(_out) + MESSAGE(DEBUG "TAR_UNPACK()-out: ${_out}") + ENDIF(_out) + FILE(REMOVE "${_file}.tar") + ELSE(GTAR_EXECUTABLE AND GZIP_EXECUTABLE) + MESSAGE(STATUS "gnutar not found") + ENDIF(GTAR_EXECUTABLE AND GZIP_EXECUTABLE) +ENDMACRO(TAR_UNPACK _file _wd) + + +MACRO(ZIP_UNPACK _file _wd) + FIND_PROGRAM(7Z_EXECUTABLE NAMES 7z 7za) + + MESSAGE(STATUS "7zip: ${7Z_EXECUTABLE}") + + IF(7Z_EXECUTABLE) + EXECUTE_PROCESS(COMMAND ${7Z_EXECUTABLE} x "-o${_wd}" "${_file}") + ELSE(7Z_EXECUTABLE) + MESSAGE(STATUS "7zip not found") + ENDIF(7Z_EXECUTABLE) +ENDMACRO(ZIP_UNPACK _file _wd) + + diff --git a/config.h.cmake b/config.h.cmake new file mode 100644 index 0000000..ec1a008 --- /dev/null +++ b/config.h.cmake @@ -0,0 +1,48 @@ +#cmakedefine HAVE_EVENT_BASE_NEW +#cmakedefine HAVE_EVENT_BASE_FREE +#cmakedefine HAVE_EVENT_H +#cmakedefine HAVE_INTTYPES_H +#cmakedefine HAVE_MGMAPI_H +#cmakedefine HAVE_NETINET_IN_H +#cmakedefine HAVE_NET_IF_H +#cmakedefine HAVE_NET_IF_DL_H +#cmakedefine HAVE_PWD_H +#cmakedefine HAVE_SIGNAL_H +#cmakedefine HAVE_STDDEF_H +#cmakedefine HAVE_STDINT_H +#cmakedefine HAVE_STDLIB_H +#cmakedefine HAVE_SYSLOG_H +#cmakedefine HAVE_SYS_IOCTL_H +#cmakedefine HAVE_SYS_FILIO_H +#cmakedefine HAVE_SYS_PARAM_H +#cmakedefine HAVE_SYS_RESOURCE_H +#cmakedefine HAVE_SYS_SOCKET_H +#cmakedefine HAVE_SYS_SOCKIO_H +#cmakedefine HAVE_SYS_TIME_H +#cmakedefine HAVE_SYS_TYPES_H +#cmakedefine HAVE_SYS_UIO_H +#cmakedefine HAVE_SYS_UN_H +#cmakedefine HAVE_TIME_H +#cmakedefine HAVE_UNISTD_H +#cmakedefine HAVE_SYSLOG_H + +#cmakedefine HAVE_INET_NTOP +#cmakedefine HAVE_GETCWD +#cmakedefine HAVE_SIGNAL +#cmakedefine HAVE_SRANDOM +#cmakedefine HAVE_STRERROR +#cmakedefine HAVE_WRITEV +#cmakedefine HAVE_SIGACTION + +#cmakedefine HAVE_SOCKLEN_T +#cmakedefine HAVE_ULONG + +#cmakedefine HAVE_GTHREAD +#cmakedefine HAVE_GTHREAD_H +#cmakedefine HAVE_OPENSSL +#define SIZEOF_RLIM_T @SIZEOF_RLIM_T@ + +#cmakedefine NETWORK_DEBUG_TRACE_IO 1 +#cmakedefine NETWORK_DEBUG_TRACE_STATE_CHANGES 1 +#cmakedefine USE_GLIB_DEBUG_LOG 1 +#cmakedefine NETWORK_DEBUG_TRACE_EVENT 1 diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt new file mode 100644 index 0000000..25c93b2 --- /dev/null +++ b/deps/CMakeLists.txt @@ -0,0 +1 @@ +# INCLUDE(deps/libevent.cmake) diff --git a/deps/libevent.cmake b/deps/libevent.cmake new file mode 100644 index 0000000..61f5428 --- /dev/null +++ b/deps/libevent.cmake @@ -0,0 +1,25 @@ +IF(WIN32) + SET(LIBEVENT_SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/libevent-1.4.11-stable") + IF(EXISTS ${LIBEVENT_SOURCE_DIR}) + ## write CMake file for libevent + + CONFIGURE_FILE(deps/libevent.config.h.cmake ${LIBEVENT_SOURCE_DIR}/config.h.cmake COPYONLY) + CONFIGURE_FILE(deps/libevent.event-config.h.cmake ${LIBEVENT_SOURCE_DIR}/event-config.h.cmake COPYONLY) + CONFIGURE_FILE(deps/libevent.CMakeLists.txt ${LIBEVENT_SOURCE_DIR}/CMakeLists.txt COPYONLY) + CONFIGURE_FILE(deps/libevent.def ${LIBEVENT_SOURCE_DIR}/libevent.def COPYONLY) + ## CONFIGURE_FILE(deps/libevent.event.h.cmake ${LIBEVENT_SOURCE_DIR}/event.h COPYONLY) + ## CONFIGURE_FILE(deps/libevent.evutil.h.cmake ${LIBEVENT_SOURCE_DIR}/evutil.h COPYONLY) + + ADD_SUBDIRECTORY(${LIBEVENT_SOURCE_DIR} build-libevent) + + SET(EVENT_INCLUDE_DIRS ${LIBEVENT_SOURCE_DIR} CACHE INTERNAL "") + IF(EXISTS ${CMAKE_BINARY_DIR}/build-libevent/${CMAKE_BUILD_TYPE}/) + SET(EVENT_LIBRARY_DIRS ${CMAKE_BINARY_DIR}/build-libevent/${CMAKE_BUILD_TYPE} CACHE INTERNAL "") + ELSE(EXISTS ${CMAKE_BINARY_DIR}/build-libevent/${CMAKE_BUILD_TYPE}/) + SET(EVENT_LIBRARY_DIRS ${CMAKE_BINARY_DIR}/build-libevent CACHE INTERNAL "") + ENDIF(EXISTS ${CMAKE_BINARY_DIR}/build-libevent/${CMAKE_BUILD_TYPE}/) + SET(EVENT_LIBRARIES event CACHE INTERNAL "") + ELSE(EXISTS ${LIBEVENT_SOURCE_DIR}) + MESSAGE(FATAL_ERROR "Could not find dependency libevent-1.4.11-stable in ${LIBEVENT_SOURCE_DIR}") + ENDIF(EXISTS ${LIBEVENT_SOURCE_DIR}) +ENDIF(WIN32) diff --git a/deps/libevent.config.h.cmake b/deps/libevent.config.h.cmake new file mode 100644 index 0000000..d434e40 --- /dev/null +++ b/deps/libevent.config.h.cmake @@ -0,0 +1,35 @@ +#cmakedefine HAVE_STDINT_H +#cmakedefine HAVE_SYS_TIME_H +#cmakedefine HAVE_SYS_TREE_H +#cmakedefine HAVE_FCNTL_H +#cmakedefine HAVE_STDARG_H +#cmakedefine HAVE_INTTYPES_H +#cmakedefine HAVE_STDLIB_H +#cmakedefine HAVE_POLL_H +#cmakedefine HAVE_SIGNAL_H +#cmakedefine HAVE_UNISTD_H +#cmakedefine HAVE_SYS_DEVPOLL_H +#cmakedefine HAVE_PORT_H +#cmakedefine HAVE_KQUEUE_H + +#cmakedefine HAVE_POLL +#cmakedefine HAVE_SELECT +#cmakedefine HAVE_GETTIMEOFDAY +#cmakedefine HAVE_VASPRINTF +#cmakedefine HAVE_FCNTL +#cmakedefine HAVE_CLOCK_GETTIME +#cmakedefine HAVE_STRTOK_R +#cmakedefine HAVE_STRSEP +#cmakedefine HAVE_GETADDRINFO +#cmakedefine HAVE_GETNAMEINFO +#cmakedefine HAVE_STRLCPY +#cmakedefine HAVE_INET_NTOP +#cmakedefine HAVE_SIGTIMEDWAIT +#cmakedefine HAVE_EPOLL +#cmakedefine HAVE_KQUEUE +#cmakedefine HAVE_TIMERADD +#cmakedefine HAVE_TIMERSUB +#cmakedefine HAVE_TIMERISSET +#cmakedefine HAVE_TIMERCLEAR + +#define VERSION "libevent-1.4.11-stable" diff --git a/deps/libevent.event-config.h.cmake b/deps/libevent.event-config.h.cmake new file mode 100644 index 0000000..9c4a8ca --- /dev/null +++ b/deps/libevent.event-config.h.cmake @@ -0,0 +1,35 @@ +#cmakedefine _EVENT_HAVE_STDINT_H +#cmakedefine _EVENT_HAVE_SYS_TIME_H +#cmakedefine _EVENT_HAVE_SYS_TREE_H +#cmakedefine _EVENT_HAVE_FCNTL_H +#cmakedefine _EVENT_HAVE_STDARG_H +#cmakedefine _EVENT_HAVE_INTTYPES_H +#cmakedefine _EVENT_HAVE_STDLIB_H +#cmakedefine _EVENT_HAVE_POLL_H +#cmakedefine _EVENT_HAVE_SIGNAL_H +#cmakedefine _EVENT_HAVE_UNISTD_H +#cmakedefine _EVENT_HAVE_SYS_DEVPOLL_H +#cmakedefine _EVENT_HAVE_PORT_H +#cmakedefine _EVENT_HAVE_KQUEUE_H + +#cmakedefine _EVENT_HAVE_POLL +#cmakedefine _EVENT_HAVE_SELECT +#cmakedefine _EVENT_HAVE_GETTIMEOFDAY +#cmakedefine _EVENT_HAVE_VASPRINTF +#cmakedefine _EVENT_HAVE_FCNTL +#cmakedefine _EVENT_HAVE_CLOCK_GETTIME +#cmakedefine _EVENT_HAVE_STRTOK_R +#cmakedefine _EVENT_HAVE_STRSEP +#cmakedefine _EVENT_HAVE_GETADDRINFO +#cmakedefine _EVENT_HAVE_GETNAMEINFO +#cmakedefine _EVENT_HAVE_STRLCPY +#cmakedefine _EVENT_HAVE_INET_NTOP +#cmakedefine _EVENT_HAVE_SIGTIMEDWAIT +#cmakedefine _EVENT_HAVE_EPOLL +#cmakedefine _EVENT_HAVE_KQUEUE +#cmakedefine _EVENT_HAVE_TIMERADD +#cmakedefine _EVENT_HAVE_TIMERSUB +#cmakedefine _EVENT_HAVE_TIMERISSET +#cmakedefine _EVENT_HAVE_TIMERCLEAR + +#define _EVENT_VERSION "libevent-1.4.11-stable" diff --git a/deps/libevent.event.h.cmake b/deps/libevent.event.h.cmake new file mode 100644 index 0000000..307100c --- /dev/null +++ b/deps/libevent.event.h.cmake @@ -0,0 +1,1182 @@ +/* + * Copyright (c) 2000-2007 Niels Provos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _EVENT_H_ +#define _EVENT_H_ + +/** @mainpage + + @section intro Introduction + + libevent is an event notification library for developing scalable network + servers. The libevent API provides a mechanism to execute a callback + function when a specific event occurs on a file descriptor or after a + timeout has been reached. Furthermore, libevent also support callbacks due + to signals or regular timeouts. + + libevent is meant to replace the event loop found in event driven network + servers. An application just needs to call event_dispatch() and then add or + remove events dynamically without having to change the event loop. + + Currently, libevent supports /dev/poll, kqueue(2), select(2), poll(2) and + epoll(4). It also has experimental support for real-time signals. The + internal event mechanism is completely independent of the exposed event API, + and a simple update of libevent can provide new functionality without having + to redesign the applications. As a result, Libevent allows for portable + application development and provides the most scalable event notification + mechanism available on an operating system. Libevent can also be used for + multi-threaded aplications; see Steven Grimm's explanation. Libevent should + compile on Linux, *BSD, Mac OS X, Solaris and Windows. + + @section usage Standard usage + + Every program that uses libevent must include the header, and pass + the -levent flag to the linker. Before using any of the functions in the + library, you must call event_init() or event_base_new() to perform one-time + initialization of the libevent library. + + @section event Event notification + + For each file descriptor that you wish to monitor, you must declare an event + structure and call event_set() to initialize the members of the structure. + To enable notification, you add the structure to the list of monitored + events by calling event_add(). The event structure must remain allocated as + long as it is active, so it should be allocated on the heap. Finally, you + call event_dispatch() to loop and dispatch events. + + @section bufferevent I/O Buffers + + libevent provides an abstraction on top of the regular event callbacks. This + abstraction is called a buffered event. A buffered event provides input and + output buffers that get filled and drained automatically. The user of a + buffered event no longer deals directly with the I/O, but instead is reading + from input and writing to output buffers. + + Once initialized via bufferevent_new(), the bufferevent structure can be + used repeatedly with bufferevent_enable() and bufferevent_disable(). + Instead of reading and writing directly to a socket, you would call + bufferevent_read() and bufferevent_write(). + + When read enabled the bufferevent will try to read from the file descriptor + and call the read callback. The write callback is executed whenever the + output buffer is drained below the write low watermark, which is 0 by + default. + + @section timers Timers + + libevent can also be used to create timers that invoke a callback after a + certain amount of time has expired. The evtimer_set() function prepares an + event struct to be used as a timer. To activate the timer, call + evtimer_add(). Timers can be deactivated by calling evtimer_del(). + + @section timeouts Timeouts + + In addition to simple timers, libevent can assign timeout events to file + descriptors that are triggered whenever a certain amount of time has passed + with no activity on a file descriptor. The timeout_set() function + initializes an event struct for use as a timeout. Once initialized, the + event must be activated by using timeout_add(). To cancel the timeout, call + timeout_del(). + + @section evdns Asynchronous DNS resolution + + libevent provides an asynchronous DNS resolver that should be used instead + of the standard DNS resolver functions. These functions can be imported by + including the header in your program. Before using any of the + resolver functions, you must call evdns_init() to initialize the library. To + convert a hostname to an IP address, you call the evdns_resolve_ipv4() + function. To perform a reverse lookup, you would call the + evdns_resolve_reverse() function. All of these functions use callbacks to + avoid blocking while the lookup is performed. + + @section evhttp Event-driven HTTP servers + + libevent provides a very simple event-driven HTTP server that can be + embedded in your program and used to service HTTP requests. + + To use this capability, you need to include the header in your + program. You create the server by calling evhttp_new(). Add addresses and + ports to listen on with evhttp_bind_socket(). You then register one or more + callbacks to handle incoming requests. Each URI can be assigned a callback + via the evhttp_set_cb() function. A generic callback function can also be + registered via evhttp_set_gencb(); this callback will be invoked if no other + callbacks have been registered for a given URI. + + @section evrpc A framework for RPC servers and clients + + libevents provides a framework for creating RPC servers and clients. It + takes care of marshaling and unmarshaling all data structures. + + @section api API Reference + + To browse the complete documentation of the libevent API, click on any of + the following links. + + event.h + The primary libevent header + + evdns.h + Asynchronous DNS resolution + + evhttp.h + An embedded libevent-based HTTP server + + evrpc.h + A framework for creating RPC servers and clients + + */ + +/** @file event.h + + A library for writing event-driven network servers + + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#ifdef _EVENT_HAVE_SYS_TYPES_H +#include +#endif +#ifdef _EVENT_HAVE_SYS_TIME_H +#include +#endif +#ifdef _EVENT_HAVE_STDINT_H +#include +#endif +#include + +/* For int types. */ +#include + +#ifdef WIN32 +#define WIN32_LEAN_AND_MEAN +#include +#undef WIN32_LEAN_AND_MEAN +typedef unsigned char u_char; +typedef unsigned short u_short; +#ifdef BUILD_EVENT_DLL +#define EVEXPORT __declspec(dllexport) +#else +#define EVEXPORT __declspec(dllimport) +#endif +#else +#define EVEXPORT +#endif + +#define EVLIST_TIMEOUT 0x01 +#define EVLIST_INSERTED 0x02 +#define EVLIST_SIGNAL 0x04 +#define EVLIST_ACTIVE 0x08 +#define EVLIST_INTERNAL 0x10 +#define EVLIST_INIT 0x80 + +/* EVLIST_X_ Private space: 0x1000-0xf000 */ +#define EVLIST_ALL (0xf000 | 0x9f) + +#define EV_TIMEOUT 0x01 +#define EV_READ 0x02 +#define EV_WRITE 0x04 +#define EV_SIGNAL 0x08 +#define EV_PERSIST 0x10 /* Persistant event */ + +/* Fix so that ppl dont have to run with */ +#ifndef TAILQ_ENTRY +#define _EVENT_DEFINED_TQENTRY +#define TAILQ_ENTRY(type) \ +struct { \ + struct type *tqe_next; /* next element */ \ + struct type **tqe_prev; /* address of previous next element */ \ +} +#endif /* !TAILQ_ENTRY */ + +struct event_base; +struct event { + TAILQ_ENTRY (event) ev_next; + TAILQ_ENTRY (event) ev_active_next; + TAILQ_ENTRY (event) ev_signal_next; + unsigned int min_heap_idx; /* for managing timeouts */ + + struct event_base *ev_base; + + int ev_fd; + short ev_events; + short ev_ncalls; + short *ev_pncalls; /* Allows deletes in callback */ + + struct timeval ev_timeout; + + int ev_pri; /* smaller numbers are higher priority */ + + void (*ev_callback)(int, short, void *arg); + void *ev_arg; + + int ev_res; /* result passed to event callback */ + int ev_flags; +}; + +#define EVENT_SIGNAL(ev) (int)(ev)->ev_fd +#define EVENT_FD(ev) (int)(ev)->ev_fd + +/* + * Key-Value pairs. Can be used for HTTP headers but also for + * query argument parsing. + */ +struct evkeyval { + TAILQ_ENTRY(evkeyval) next; + + char *key; + char *value; +}; + +#ifdef _EVENT_DEFINED_TQENTRY +#undef TAILQ_ENTRY +struct event_list; +struct evkeyvalq; +#undef _EVENT_DEFINED_TQENTRY +#else +TAILQ_HEAD (event_list, event); +TAILQ_HEAD (evkeyvalq, evkeyval); +#endif /* _EVENT_DEFINED_TQENTRY */ + +/** + Initialize the event API. + + Use event_base_new() to initialize a new event base, but does not set + the current_base global. If using only event_base_new(), each event + added must have an event base set with event_base_set() + + @see event_base_set(), event_base_free(), event_init() + */ +EVEXPORT struct event_base *event_base_new(void); + +/** + Initialize the event API. + + The event API needs to be initialized with event_init() before it can be + used. Sets the current_base global representing the default base for + events that have no base associated with them. + + @see event_base_set(), event_base_new() + */ +EVEXPORT struct event_base *event_init(void); + +/** + Reinitialized the event base after a fork + + Some event mechanisms do not survive across fork. The event base needs + to be reinitialized with the event_reinit() function. + + @param base the event base that needs to be re-initialized + @return 0 if successful, or -1 if some events could not be re-added. + @see event_base_new(), event_init() +*/ +EVEXPORT int event_reinit(struct event_base *base); + +/** + Loop to process events. + + In order to process events, an application needs to call + event_dispatch(). This function only returns on error, and should + replace the event core of the application program. + + @see event_base_dispatch() + */ +EVEXPORT int event_dispatch(void); + + +/** + Threadsafe event dispatching loop. + + @param eb the event_base structure returned by event_init() + @see event_init(), event_dispatch() + */ +EVEXPORT int event_base_dispatch(struct event_base *); + + +/** + Get the kernel event notification mechanism used by libevent. + + @param eb the event_base structure returned by event_base_new() + @return a string identifying the kernel event mechanism (kqueue, epoll, etc.) + */ +EVEXPORT const char *event_base_get_method(struct event_base *); + + +/** + Deallocate all memory associated with an event_base, and free the base. + + Note that this function will not close any fds or free any memory passed + to event_set as the argument to callback. + + @param eb an event_base to be freed + */ +EVEXPORT void event_base_free(struct event_base *); + + +#define _EVENT_LOG_DEBUG 0 +#define _EVENT_LOG_MSG 1 +#define _EVENT_LOG_WARN 2 +#define _EVENT_LOG_ERR 3 +typedef void (*event_log_cb)(int severity, const char *msg); +/** + Redirect libevent's log messages. + + @param cb a function taking two arguments: an integer severity between + _EVENT_LOG_DEBUG and _EVENT_LOG_ERR, and a string. If cb is NULL, + then the default log is used. + */ +EVEXPORT void event_set_log_callback(event_log_cb cb); + +/** + Associate a different event base with an event. + + @param eb the event base + @param ev the event + */ +EVEXPORT int event_base_set(struct event_base *, struct event *); + +/** + event_loop() flags + */ +/*@{*/ +#define EVLOOP_ONCE 0x01 /**< Block at most once. */ +#define EVLOOP_NONBLOCK 0x02 /**< Do not block. */ +/*@}*/ + +/** + Handle events. + + This is a more flexible version of event_dispatch(). + + @param flags any combination of EVLOOP_ONCE | EVLOOP_NONBLOCK + @return 0 if successful, -1 if an error occurred, or 1 if no events were + registered. + @see event_loopexit(), event_base_loop() +*/ +EVEXPORT int event_loop(int); + +/** + Handle events (threadsafe version). + + This is a more flexible version of event_base_dispatch(). + + @param eb the event_base structure returned by event_init() + @param flags any combination of EVLOOP_ONCE | EVLOOP_NONBLOCK + @return 0 if successful, -1 if an error occurred, or 1 if no events were + registered. + @see event_loopexit(), event_base_loop() + */ +EVEXPORT int event_base_loop(struct event_base *, int); + +/** + Exit the event loop after the specified time. + + The next event_loop() iteration after the given timer expires will + complete normally (handling all queued events) then exit without + blocking for events again. + + Subsequent invocations of event_loop() will proceed normally. + + @param tv the amount of time after which the loop should terminate. + @return 0 if successful, or -1 if an error occurred + @see event_loop(), event_base_loop(), event_base_loopexit() + */ +EVEXPORT int event_loopexit(const struct timeval *); + + +/** + Exit the event loop after the specified time (threadsafe variant). + + The next event_base_loop() iteration after the given timer expires will + complete normally (handling all queued events) then exit without + blocking for events again. + + Subsequent invocations of event_base_loop() will proceed normally. + + @param eb the event_base structure returned by event_init() + @param tv the amount of time after which the loop should terminate. + @return 0 if successful, or -1 if an error occurred + @see event_loopexit() + */ +EVEXPORT int event_base_loopexit(struct event_base *, const struct timeval *); + +/** + Abort the active event_loop() immediately. + + event_loop() will abort the loop after the next event is completed; + event_loopbreak() is typically invoked from this event's callback. + This behavior is analogous to the "break;" statement. + + Subsequent invocations of event_loop() will proceed normally. + + @return 0 if successful, or -1 if an error occurred + @see event_base_loopbreak(), event_loopexit() + */ +EVEXPORT int event_loopbreak(void); + +/** + Abort the active event_base_loop() immediately. + + event_base_loop() will abort the loop after the next event is completed; + event_base_loopbreak() is typically invoked from this event's callback. + This behavior is analogous to the "break;" statement. + + Subsequent invocations of event_loop() will proceed normally. + + @param eb the event_base structure returned by event_init() + @return 0 if successful, or -1 if an error occurred + @see event_base_loopexit + */ +EVEXPORT int event_base_loopbreak(struct event_base *); + + +/** + Add a timer event. + + @param ev the event struct + @param tv timeval struct + */ +#define evtimer_add(ev, tv) event_add(ev, tv) + + +/** + Define a timer event. + + @param ev event struct to be modified + @param cb callback function + @param arg argument that will be passed to the callback function + */ +#define evtimer_set(ev, cb, arg) event_set(ev, -1, 0, cb, arg) + + +/** + * Delete a timer event. + * + * @param ev the event struct to be disabled + */ +#define evtimer_del(ev) event_del(ev) +#define evtimer_pending(ev, tv) event_pending(ev, EV_TIMEOUT, tv) +#define evtimer_initialized(ev) ((ev)->ev_flags & EVLIST_INIT) + +/** + * Add a timeout event. + * + * @param ev the event struct to be disabled + * @param tv the timeout value, in seconds + */ +#define timeout_add(ev, tv) event_add(ev, tv) + + +/** + * Define a timeout event. + * + * @param ev the event struct to be defined + * @param cb the callback to be invoked when the timeout expires + * @param arg the argument to be passed to the callback + */ +#define timeout_set(ev, cb, arg) event_set(ev, -1, 0, cb, arg) + + +/** + * Disable a timeout event. + * + * @param ev the timeout event to be disabled + */ +#define timeout_del(ev) event_del(ev) + +#define timeout_pending(ev, tv) event_pending(ev, EV_TIMEOUT, tv) +#define timeout_initialized(ev) ((ev)->ev_flags & EVLIST_INIT) + +#define signal_add(ev, tv) event_add(ev, tv) +#define signal_set(ev, x, cb, arg) \ + event_set(ev, x, EV_SIGNAL|EV_PERSIST, cb, arg) +#define signal_del(ev) event_del(ev) +#define signal_pending(ev, tv) event_pending(ev, EV_SIGNAL, tv) +#define signal_initialized(ev) ((ev)->ev_flags & EVLIST_INIT) + +/** + Prepare an event structure to be added. + + The function event_set() prepares the event structure ev to be used in + future calls to event_add() and event_del(). The event will be prepared to + call the function specified by the fn argument with an int argument + indicating the file descriptor, a short argument indicating the type of + event, and a void * argument given in the arg argument. The fd indicates + the file descriptor that should be monitored for events. The events can be + either EV_READ, EV_WRITE, or both. Indicating that an application can read + or write from the file descriptor respectively without blocking. + + The function fn will be called with the file descriptor that triggered the + event and the type of event which will be either EV_TIMEOUT, EV_SIGNAL, + EV_READ, or EV_WRITE. The additional flag EV_PERSIST makes an event_add() + persistent until event_del() has been called. + + @param ev an event struct to be modified + @param fd the file descriptor to be monitored + @param event desired events to monitor; can be EV_READ and/or EV_WRITE + @param fn callback function to be invoked when the event occurs + @param arg an argument to be passed to the callback function + + @see event_add(), event_del(), event_once() + + */ +EVEXPORT void event_set(struct event *, int, short, void (*)(int, short, void *), void *); + +/** + Schedule a one-time event to occur. + + The function event_once() is similar to event_set(). However, it schedules + a callback to be called exactly once and does not require the caller to + prepare an event structure. + + @param fd a file descriptor to monitor + @param events event(s) to monitor; can be any of EV_TIMEOUT | EV_READ | + EV_WRITE + @param callback callback function to be invoked when the event occurs + @param arg an argument to be passed to the callback function + @param timeout the maximum amount of time to wait for the event, or NULL + to wait forever + @return 0 if successful, or -1 if an error occurred + @see event_set() + + */ +EVEXPORT int event_once(int, short, void (*)(int, short, void *), void *, + const struct timeval *); + + +/** + Schedule a one-time event (threadsafe variant) + + The function event_base_once() is similar to event_set(). However, it + schedules a callback to be called exactly once and does not require the + caller to prepare an event structure. + + @param base an event_base returned by event_init() + @param fd a file descriptor to monitor + @param events event(s) to monitor; can be any of EV_TIMEOUT | EV_READ | + EV_WRITE + @param callback callback function to be invoked when the event occurs + @param arg an argument to be passed to the callback function + @param timeout the maximum amount of time to wait for the event, or NULL + to wait forever + @return 0 if successful, or -1 if an error occurred + @see event_once() + */ +EVEXPORT int event_base_once(struct event_base *base, int fd, short events, + void (*callback)(int, short, void *), void *arg, + const struct timeval *timeout); + + +/** + Add an event to the set of monitored events. + + The function event_add() schedules the execution of the ev event when the + event specified in event_set() occurs or in at least the time specified in + the tv. If tv is NULL, no timeout occurs and the function will only be + called if a matching event occurs on the file descriptor. The event in the + ev argument must be already initialized by event_set() and may not be used + in calls to event_set() until it has timed out or been removed with + event_del(). If the event in the ev argument already has a scheduled + timeout, the old timeout will be replaced by the new one. + + @param ev an event struct initialized via event_set() + @param timeout the maximum amount of time to wait for the event, or NULL + to wait forever + @return 0 if successful, or -1 if an error occurred + @see event_del(), event_set() + */ +EVEXPORT int event_add(struct event *ev, const struct timeval *timeout); + + +/** + Remove an event from the set of monitored events. + + The function event_del() will cancel the event in the argument ev. If the + event has already executed or has never been added the call will have no + effect. + + @param ev an event struct to be removed from the working set + @return 0 if successful, or -1 if an error occurred + @see event_add() + */ +EVEXPORT int event_del(struct event *); + +EVEXPORT void event_active(struct event *, int, short); + + +/** + Checks if a specific event is pending or scheduled. + + @param ev an event struct previously passed to event_add() + @param event the requested event type; any of EV_TIMEOUT|EV_READ| + EV_WRITE|EV_SIGNAL + @param tv an alternate timeout (FIXME - is this true?) + + @return 1 if the event is pending, or 0 if the event has not occurred + + */ +EVEXPORT int event_pending(struct event *ev, short event, struct timeval *tv); + + +/** + Test if an event structure has been initialized. + + The event_initialized() macro can be used to check if an event has been + initialized. + + @param ev an event structure to be tested + @return 1 if the structure has been initialized, or 0 if it has not been + initialized + */ +#ifdef WIN32 +#define event_initialized(ev) ((ev)->ev_flags & EVLIST_INIT && (ev)->ev_fd != (int)INVALID_HANDLE_VALUE) +#else +#define event_initialized(ev) ((ev)->ev_flags & EVLIST_INIT) +#endif + + +/** + Get the libevent version number. + + @return a string containing the version number of libevent + */ +EVEXPORT const char *event_get_version(void); + + +/** + Get the kernel event notification mechanism used by libevent. + + @return a string identifying the kernel event mechanism (kqueue, epoll, etc.) + */ +EVEXPORT const char *event_get_method(void); + + +/** + Set the number of different event priorities. + + By default libevent schedules all active events with the same priority. + However, some time it is desirable to process some events with a higher + priority than others. For that reason, libevent supports strict priority + queues. Active events with a lower priority are always processed before + events with a higher priority. + + The number of different priorities can be set initially with the + event_priority_init() function. This function should be called before the + first call to event_dispatch(). The event_priority_set() function can be + used to assign a priority to an event. By default, libevent assigns the + middle priority to all events unless their priority is explicitly set. + + @param npriorities the maximum number of priorities + @return 0 if successful, or -1 if an error occurred + @see event_base_priority_init(), event_priority_set() + + */ +EVEXPORT int event_priority_init(int); + + +/** + Set the number of different event priorities (threadsafe variant). + + See the description of event_priority_init() for more information. + + @param eb the event_base structure returned by event_init() + @param npriorities the maximum number of priorities + @return 0 if successful, or -1 if an error occurred + @see event_priority_init(), event_priority_set() + */ +EVEXPORT int event_base_priority_init(struct event_base *, int); + + +/** + Assign a priority to an event. + + @param ev an event struct + @param priority the new priority to be assigned + @return 0 if successful, or -1 if an error occurred + @see event_priority_init() + */ +EVEXPORT int event_priority_set(struct event *, int); + + +/* These functions deal with buffering input and output */ + +struct evbuffer { + u_char *buffer; + u_char *orig_buffer; + + size_t misalign; + size_t totallen; + size_t off; + + void (*cb)(struct evbuffer *, size_t, size_t, void *); + void *cbarg; +}; + +/* Just for error reporting - use other constants otherwise */ +#define EVBUFFER_READ 0x01 +#define EVBUFFER_WRITE 0x02 +#define EVBUFFER_EOF 0x10 +#define EVBUFFER_ERROR 0x20 +#define EVBUFFER_TIMEOUT 0x40 + +struct bufferevent; +typedef void (*evbuffercb)(struct bufferevent *, void *); +typedef void (*everrorcb)(struct bufferevent *, short what, void *); + +struct event_watermark { + size_t low; + size_t high; +}; + +struct bufferevent { + struct event_base *ev_base; + + struct event ev_read; + struct event ev_write; + + struct evbuffer *input; + struct evbuffer *output; + + struct event_watermark wm_read; + struct event_watermark wm_write; + + evbuffercb readcb; + evbuffercb writecb; + everrorcb errorcb; + void *cbarg; + + int timeout_read; /* in seconds */ + int timeout_write; /* in seconds */ + + short enabled; /* events that are currently enabled */ +}; + + +/** + Create a new bufferevent. + + libevent provides an abstraction on top of the regular event callbacks. + This abstraction is called a buffered event. A buffered event provides + input and output buffers that get filled and drained automatically. The + user of a buffered event no longer deals directly with the I/O, but + instead is reading from input and writing to output buffers. + + Once initialized, the bufferevent structure can be used repeatedly with + bufferevent_enable() and bufferevent_disable(). + + When read enabled the bufferevent will try to read from the file descriptor + and call the read callback. The write callback is executed whenever the + output buffer is drained below the write low watermark, which is 0 by + default. + + If multiple bases are in use, bufferevent_base_set() must be called before + enabling the bufferevent for the first time. + + @param fd the file descriptor from which data is read and written to. + This file descriptor is not allowed to be a pipe(2). + @param readcb callback to invoke when there is data to be read, or NULL if + no callback is desired + @param writecb callback to invoke when the file descriptor is ready for + writing, or NULL if no callback is desired + @param errorcb callback to invoke when there is an error on the file + descriptor + @param cbarg an argument that will be supplied to each of the callbacks + (readcb, writecb, and errorcb) + @return a pointer to a newly allocated bufferevent struct, or NULL if an + error occurred + @see bufferevent_base_set(), bufferevent_free() + */ +EVEXPORT struct bufferevent *bufferevent_new(int fd, + evbuffercb readcb, evbuffercb writecb, everrorcb errorcb, void *cbarg); + + +/** + Assign a bufferevent to a specific event_base. + + @param base an event_base returned by event_init() + @param bufev a bufferevent struct returned by bufferevent_new() + @return 0 if successful, or -1 if an error occurred + @see bufferevent_new() + */ +EVEXPORT int bufferevent_base_set(struct event_base *base, struct bufferevent *bufev); + + +/** + Assign a priority to a bufferevent. + + @param bufev a bufferevent struct + @param pri the priority to be assigned + @return 0 if successful, or -1 if an error occurred + */ +EVEXPORT int bufferevent_priority_set(struct bufferevent *bufev, int pri); + + +/** + Deallocate the storage associated with a bufferevent structure. + + @param bufev the bufferevent structure to be freed. + */ +EVEXPORT void bufferevent_free(struct bufferevent *bufev); + + +/** + Changes the callbacks for a bufferevent. + + @param bufev the bufferevent object for which to change callbacks + @param readcb callback to invoke when there is data to be read, or NULL if + no callback is desired + @param writecb callback to invoke when the file descriptor is ready for + writing, or NULL if no callback is desired + @param errorcb callback to invoke when there is an error on the file + descriptor + @param cbarg an argument that will be supplied to each of the callbacks + (readcb, writecb, and errorcb) + @see bufferevent_new() + */ +EVEXPORT void bufferevent_setcb(struct bufferevent *bufev, + evbuffercb readcb, evbuffercb writecb, everrorcb errorcb, void *cbarg); + +/** + Changes the file descriptor on which the bufferevent operates. + + @param bufev the bufferevent object for which to change the file descriptor + @param fd the file descriptor to operate on +*/ +EVEXPORT void bufferevent_setfd(struct bufferevent *bufev, int fd); + +/** + Write data to a bufferevent buffer. + + The bufferevent_write() function can be used to write data to the file + descriptor. The data is appended to the output buffer and written to the + descriptor automatically as it becomes available for writing. + + @param bufev the bufferevent to be written to + @param data a pointer to the data to be written + @param size the length of the data, in bytes + @return 0 if successful, or -1 if an error occurred + @see bufferevent_write_buffer() + */ +EVEXPORT int bufferevent_write(struct bufferevent *bufev, + const void *data, size_t size); + + +/** + Write data from an evbuffer to a bufferevent buffer. The evbuffer is + being drained as a result. + + @param bufev the bufferevent to be written to + @param buf the evbuffer to be written + @return 0 if successful, or -1 if an error occurred + @see bufferevent_write() + */ +EVEXPORT int bufferevent_write_buffer(struct bufferevent *bufev, struct evbuffer *buf); + + +/** + Read data from a bufferevent buffer. + + The bufferevent_read() function is used to read data from the input buffer. + + @param bufev the bufferevent to be read from + @param data pointer to a buffer that will store the data + @param size the size of the data buffer, in bytes + @return the amount of data read, in bytes. + */ +EVEXPORT size_t bufferevent_read(struct bufferevent *bufev, void *data, size_t size); + +/** + Enable a bufferevent. + + @param bufev the bufferevent to be enabled + @param event any combination of EV_READ | EV_WRITE. + @return 0 if successful, or -1 if an error occurred + @see bufferevent_disable() + */ +EVEXPORT int bufferevent_enable(struct bufferevent *bufev, short event); + + +/** + Disable a bufferevent. + + @param bufev the bufferevent to be disabled + @param event any combination of EV_READ | EV_WRITE. + @return 0 if successful, or -1 if an error occurred + @see bufferevent_enable() + */ +EVEXPORT int bufferevent_disable(struct bufferevent *bufev, short event); + + +/** + Set the read and write timeout for a buffered event. + + @param bufev the bufferevent to be modified + @param timeout_read the read timeout + @param timeout_write the write timeout + */ +EVEXPORT void bufferevent_settimeout(struct bufferevent *bufev, + int timeout_read, int timeout_write); + + +/** + Sets the watermarks for read and write events. + + On input, a bufferevent does not invoke the user read callback unless + there is at least low watermark data in the buffer. If the read buffer + is beyond the high watermark, the buffevent stops reading from the network. + + On output, the user write callback is invoked whenever the buffered data + falls below the low watermark. + + @param bufev the bufferevent to be modified + @param events EV_READ, EV_WRITE or both + @param lowmark the lower watermark to set + @param highmark the high watermark to set +*/ + +EVEXPORT void bufferevent_setwatermark(struct bufferevent *bufev, short events, + size_t lowmark, size_t highmark); + +#define EVBUFFER_LENGTH(x) (x)->off +#define EVBUFFER_DATA(x) (x)->buffer +#define EVBUFFER_INPUT(x) (x)->input +#define EVBUFFER_OUTPUT(x) (x)->output + + +/** + Allocate storage for a new evbuffer. + + @return a pointer to a newly allocated evbuffer struct, or NULL if an error + occurred + */ +EVEXPORT struct evbuffer *evbuffer_new(void); + + +/** + Deallocate storage for an evbuffer. + + @param pointer to the evbuffer to be freed + */ +EVEXPORT void evbuffer_free(struct evbuffer *); + + +/** + Expands the available space in an event buffer. + + Expands the available space in the event buffer to at least datlen + + @param buf the event buffer to be expanded + @param datlen the new minimum length requirement + @return 0 if successful, or -1 if an error occurred +*/ +EVEXPORT int evbuffer_expand(struct evbuffer *, size_t); + + +/** + Append data to the end of an evbuffer. + + @param buf the event buffer to be appended to + @param data pointer to the beginning of the data buffer + @param datlen the number of bytes to be copied from the data buffer + */ +EVEXPORT int evbuffer_add(struct evbuffer *, const void *, size_t); + + + +/** + Read data from an event buffer and drain the bytes read. + + @param buf the event buffer to be read from + @param data the destination buffer to store the result + @param datlen the maximum size of the destination buffer + @return the number of bytes read + */ +EVEXPORT int evbuffer_remove(struct evbuffer *, void *, size_t); + + +/** + * Read a single line from an event buffer. + * + * Reads a line terminated by either '\r\n', '\n\r' or '\r' or '\n'. + * The returned buffer needs to be freed by the caller. + * + * @param buffer the evbuffer to read from + * @return pointer to a single line, or NULL if an error occurred + */ +EVEXPORT char *evbuffer_readline(struct evbuffer *); + + +/** + Move data from one evbuffer into another evbuffer. + + This is a destructive add. The data from one buffer moves into + the other buffer. The destination buffer is expanded as needed. + + @param outbuf the output buffer + @param inbuf the input buffer + @return 0 if successful, or -1 if an error occurred + */ +EVEXPORT int evbuffer_add_buffer(struct evbuffer *, struct evbuffer *); + + +/** + Append a formatted string to the end of an evbuffer. + + @param buf the evbuffer that will be appended to + @param fmt a format string + @param ... arguments that will be passed to printf(3) + @return The number of bytes added if successful, or -1 if an error occurred. + */ +EVEXPORT int evbuffer_add_printf(struct evbuffer *, const char *fmt, ...) +#ifdef __GNUC__ + __attribute__((format(printf, 2, 3))) +#endif +; + + +/** + Append a va_list formatted string to the end of an evbuffer. + + @param buf the evbuffer that will be appended to + @param fmt a format string + @param ap a varargs va_list argument array that will be passed to vprintf(3) + @return The number of bytes added if successful, or -1 if an error occurred. + */ +EVEXPORT int evbuffer_add_vprintf(struct evbuffer *, const char *fmt, va_list ap); + + +/** + Remove a specified number of bytes data from the beginning of an evbuffer. + + @param buf the evbuffer to be drained + @param len the number of bytes to drain from the beginning of the buffer + */ +EVEXPORT void evbuffer_drain(struct evbuffer *, size_t); + + +/** + Write the contents of an evbuffer to a file descriptor. + + The evbuffer will be drained after the bytes have been successfully written. + + @param buffer the evbuffer to be written and drained + @param fd the file descriptor to be written to + @return the number of bytes written, or -1 if an error occurred + @see evbuffer_read() + */ +EVEXPORT int evbuffer_write(struct evbuffer *, int); + + +/** + Read from a file descriptor and store the result in an evbuffer. + + @param buf the evbuffer to store the result + @param fd the file descriptor to read from + @param howmuch the number of bytes to be read + @return the number of bytes read, or -1 if an error occurred + @see evbuffer_write() + */ +EVEXPORT int evbuffer_read(struct evbuffer *, int, int); + + +/** + Find a string within an evbuffer. + + @param buffer the evbuffer to be searched + @param what the string to be searched for + @param len the length of the search string + @return a pointer to the beginning of the search string, or NULL if the search failed. + */ +EVEXPORT u_char *evbuffer_find(struct evbuffer *, const u_char *, size_t); + +/** + Set a callback to invoke when the evbuffer is modified. + + @param buffer the evbuffer to be monitored + @param cb the callback function to invoke when the evbuffer is modified + @param cbarg an argument to be provided to the callback function + */ +EVEXPORT void evbuffer_setcb(struct evbuffer *, void (*)(struct evbuffer *, size_t, size_t, void *), void *); + +/* + * Marshaling tagged data - We assume that all tags are inserted in their + * numeric order - so that unknown tags will always be higher than the + * known ones - and we can just ignore the end of an event buffer. + */ + +EVEXPORT void evtag_init(void); + +EVEXPORT void evtag_marshal(struct evbuffer *evbuf, ev_uint32_t tag, const void *data, + ev_uint32_t len); + +/** + Encode an integer and store it in an evbuffer. + + We encode integer's by nibbles; the first nibble contains the number + of significant nibbles - 1; this allows us to encode up to 64-bit + integers. This function is byte-order independent. + + @param evbuf evbuffer to store the encoded number + @param number a 32-bit integer + */ +EVEXPORT void encode_int(struct evbuffer *evbuf, ev_uint32_t number); + +EVEXPORT void evtag_marshal_int(struct evbuffer *evbuf, ev_uint32_t tag, + ev_uint32_t integer); + +EVEXPORT void evtag_marshal_string(struct evbuffer *buf, ev_uint32_t tag, + const char *string); + +EVEXPORT void evtag_marshal_timeval(struct evbuffer *evbuf, ev_uint32_t tag, + struct timeval *tv); + +EVEXPORT int evtag_unmarshal(struct evbuffer *src, ev_uint32_t *ptag, + struct evbuffer *dst); +EVEXPORT int evtag_peek(struct evbuffer *evbuf, ev_uint32_t *ptag); +EVEXPORT int evtag_peek_length(struct evbuffer *evbuf, ev_uint32_t *plength); +EVEXPORT int evtag_payload_length(struct evbuffer *evbuf, ev_uint32_t *plength); +EVEXPORT int evtag_consume(struct evbuffer *evbuf); + +EVEXPORT int evtag_unmarshal_int(struct evbuffer *evbuf, ev_uint32_t need_tag, + ev_uint32_t *pinteger); + +EVEXPORT int evtag_unmarshal_fixed(struct evbuffer *src, ev_uint32_t need_tag, + void *data, size_t len); + +EVEXPORT int evtag_unmarshal_string(struct evbuffer *evbuf, ev_uint32_t need_tag, + char **pstring); + +EVEXPORT int evtag_unmarshal_timeval(struct evbuffer *evbuf, ev_uint32_t need_tag, + struct timeval *ptv); + +#ifdef __cplusplus +} +#endif + +#endif /* _EVENT_H_ */ diff --git a/deps/libevent.evutil.h.cmake b/deps/libevent.evutil.h.cmake new file mode 100644 index 0000000..2657883 --- /dev/null +++ b/deps/libevent.evutil.h.cmake @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2007 Niels Provos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _EVUTIL_H_ +#define _EVUTIL_H_ + +/** @file evutil.h + + Common convenience functions for cross-platform portability and + related socket manipulations. + + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#ifdef _EVENT_HAVE_SYS_TIME_H +#include +#endif +#ifdef _EVENT_HAVE_STDINT_H +#include +#elif defined(_EVENT_HAVE_INTTYPES_H) +#include +#endif +#ifdef _EVENT_HAVE_SYS_TYPES_H +#include +#endif +#include + +#ifdef _EVENT_HAVE_UINT64_T +#define ev_uint64_t uint64_t +#define ev_int64_t int64_t +#elif defined(WIN32) +#define ev_uint64_t unsigned __int64 +#define ev_int64_t signed __int64 +#elif _EVENT_SIZEOF_LONG_LONG == 8 +#define ev_uint64_t unsigned long long +#define ev_int64_t long long +#elif _EVENT_SIZEOF_LONG == 8 +#define ev_uint64_t unsigned long +#define ev_int64_t long +#else +#error "No way to define ev_uint64_t" +#endif + +#ifdef _EVENT_HAVE_UINT32_T +#define ev_uint32_t uint32_t +#elif defined(WIN32) +#define ev_uint32_t unsigned int +#elif _EVENT_SIZEOF_LONG == 4 +#define ev_uint32_t unsigned long +#elif _EVENT_SIZEOF_INT == 4 +#define ev_uint32_t unsigned int +#else +#error "No way to define ev_uint32_t" +#endif + +#ifdef _EVENT_HAVE_UINT16_T +#define ev_uint16_t uint16_t +#elif defined(WIN32) +#define ev_uint16_t unsigned short +#elif _EVENT_SIZEOF_INT == 2 +#define ev_uint16_t unsigned int +#elif _EVENT_SIZEOF_SHORT == 2 +#define ev_uint16_t unsigned short +#else +#error "No way to define ev_uint16_t" +#endif + +#ifdef _EVENT_HAVE_UINT8_T +#define ev_uint8_t uint8_t +#else +#define ev_uint8_t unsigned char +#endif +#ifdef WIN32 +#ifdef BUILD_EVENT_DLL +#define EVEXPORT __declspec(dllexport) +#else +#define EVEXPORT __declspec(dllimport) +#endif +#endif +EVEXPORT int evutil_socketpair(int d, int type, int protocol, int sv[2]); +int evutil_make_socket_nonblocking(int sock); +#ifdef WIN32 +#define EVUTIL_CLOSESOCKET(s) closesocket(s) +#else +#define EVUTIL_CLOSESOCKET(s) close(s) +#endif + +#ifdef WIN32 +#define EVUTIL_SOCKET_ERROR() WSAGetLastError() +#define EVUTIL_SET_SOCKET_ERROR(errcode) \ + do { WSASetLastError(errcode); } while (0) +#else +#define EVUTIL_SOCKET_ERROR() (errno) +#define EVUTIL_SET_SOCKET_ERROR(errcode) \ + do { errno = (errcode); } while (0) +#endif + +/* + * Manipulation functions for struct timeval + */ +#ifdef _EVENT_HAVE_TIMERADD +#define evutil_timeradd(tvp, uvp, vvp) timeradd((tvp), (uvp), (vvp)) +#define evutil_timersub(tvp, uvp, vvp) timersub((tvp), (uvp), (vvp)) +#else +#define evutil_timeradd(tvp, uvp, vvp) \ + do { \ + (vvp)->tv_sec = (tvp)->tv_sec + (uvp)->tv_sec; \ + (vvp)->tv_usec = (tvp)->tv_usec + (uvp)->tv_usec; \ + if ((vvp)->tv_usec >= 1000000) { \ + (vvp)->tv_sec++; \ + (vvp)->tv_usec -= 1000000; \ + } \ + } while (0) +#define evutil_timersub(tvp, uvp, vvp) \ + do { \ + (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec; \ + (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec; \ + if ((vvp)->tv_usec < 0) { \ + (vvp)->tv_sec--; \ + (vvp)->tv_usec += 1000000; \ + } \ + } while (0) +#endif /* !_EVENT_HAVE_HAVE_TIMERADD */ + +#ifdef _EVENT_HAVE_TIMERCLEAR +#define evutil_timerclear(tvp) timerclear(tvp) +#else +#define evutil_timerclear(tvp) (tvp)->tv_sec = (tvp)->tv_usec = 0 +#endif + +#define evutil_timercmp(tvp, uvp, cmp) \ + (((tvp)->tv_sec == (uvp)->tv_sec) ? \ + ((tvp)->tv_usec cmp (uvp)->tv_usec) : \ + ((tvp)->tv_sec cmp (uvp)->tv_sec)) + +#ifdef _EVENT_HAVE_TIMERISSET +#define evutil_timerisset(tvp) timerisset(tvp) +#else +#define evutil_timerisset(tvp) ((tvp)->tv_sec || (tvp)->tv_usec) +#endif + + +/* big-int related functions */ +ev_int64_t evutil_strtoll(const char *s, char **endptr, int base); + + +#ifdef _EVENT_HAVE_GETTIMEOFDAY +#define evutil_gettimeofday(tv, tz) gettimeofday((tv), (tz)) +#else +int evutil_gettimeofday(struct timeval *tv, struct timezone *tz); +#endif + +int evutil_snprintf(char *buf, size_t buflen, const char *format, ...) +#ifdef __GNUC__ + __attribute__((format(printf, 3, 4))) +#endif + ; +int evutil_vsnprintf(char *buf, size_t buflen, const char *format, va_list ap); + +#ifdef __cplusplus +} +#endif + +#endif /* _EVUTIL_H_ */ diff --git a/doc/Admin.md b/doc/Admin.md new file mode 100644 index 0000000..936c37f --- /dev/null +++ b/doc/Admin.md @@ -0,0 +1,284 @@ +# Admin手册 + +## 写在前面 + +**有配置修改均能动态生效，配置更改后请务必修改原始配置文件，以确保下次重启时配置能够保留。** + +## 后端配置 + +### 查看后端 + +`SELECT * FROM backends` + +查看后端信息。 + +| backend_ndx | address | state | type | uuid | connected_clients | +|------------:|---------------------|-------|------|------|------------------:| +| 1 | 10.120.240.252:3309 | up | rw | NULL | 0 | +| 2 | 10.120.240.254:3310 | up | ro | NULL | 20 | + +结果说明： + +* backend_ndx: 后端序号，按照添加顺序排列； +* address: 后端地址，IP:PORT格式； +* state: 后端状态(unknown|up|down|maintaining|delete)； +* type: 读写类型(rw|ro)； +* uuid: 暂时无用； +* connected_clients: 客户查看后端连接状态端连接的数量(正在处理请求的连接)。 + +``` +状态说明 + +unknown: 后端初始状态，还未建立连接; +up: 能与后端正常建立连接； +down: 与后端无法联通(如果开启后端状态检测，能连通后自动变为UP); +maintaining: 后端正在维护，无法建立连接或自动切换状态(此状态由管理员手动设置); +delete: 后端已被删除，无法再建立连接。 +``` + +### 查看后端连接状态 + +`SELECT CONN_DETAILS FROM backends` + +查看每个用户占用和空闲的后端连接数。 + +| backend_ndx | username | idle_conns | total_used_conns | +|------------:|---------------|-----------:|-----------------:| +| 1 | lott_quanzi | 2 | 0 | +| 1 | zjhpettest | 11 | 0 | + +结果说明： + +* backend_ndx: 后端序号； +* username: 用户名； +* idle_conns: 空闲连接数； +* total_used_conns: 正在使用的连接数。 + +### 添加后端 + +`ADD MASTER [ip:port]` + +添加一个读写类型的后端。 + +>ADD MASTER 127.0.0.1:3307 + +`ADD SLAVE [ip:port]` + +添加一个只读类型的后端。 + +>ADD SLAVE 127.0.0.1:3360 + +`INSERT INTO backends VALUES ("ip:port", "ro|rw")` + +添加一个后端，同时指定读写类型。 + +>INSERT INTO backends values ("127.0.0.1:3306", "rw"); + +### 删除后端 + +`REMOVE BACKEND [backend_ndx]` 或 +`DELETE FROM BACKENDS where backend_ndx = [backend_ndx]` + +删除一个指定序号的后端。 + +>remove backend 1 + +`DELETE FROM BACKENDS where address = '[IP:PORT]'` + +删除一个指定地址的后端。 + +>delete from backends where address = '127.0.0.1:3306' + +### 修改后端 + +`UPDATE BACKENDS set type|state=[value] where address|backend_ndx=[value]` + +修改后端类型或状态。 + +>update backends set type="rw" where address="127.0.0.1:3306" + +>update backends set state="up" where backend_ndx=1 + +## 基本配置 + +### 获取基本配置 + +`config get common` + +能获取的配置包括: + +* check_slave_delay: 是否启用从库延迟检查（0为不检查） +* slave_delay_down: 从库延迟大于该时间时，设置从库状态为DOWN（单位：秒） +* slave_delay_recover: 从库延迟小于等于该时间时，设置从库状态为UP（单位：秒） + +### 修改基本配置 + +`config set common.[option] = [value]` + +例如 + +>config set common.slave_delay_down = 3 + +## 连接池配置 + +### 获取连接池配置 + +`config get pool` + +能获取的配置包括： + +* max_pool_size: 连接池最大连接数，超出后将不再维持超出部分的连接 +* default_pool_size: 默认连接数量，连接数达到此值之前会一直新建连接 +* master_preferred: 设置为1时仅访问主库(读写) +* max_resp_len: 允许每个后端返回数据的最大值 + +### 修改连接池配置 + +`config set pool.[option] = [value]` + +修改某项配置 + +>config set pool.max_pool_size = 200 + +## 查看连接信息 + +### 查看当前连接的详细信息 + +`SHOW CONNECTIONLIST` + +将当前全部连接的详细内容按表格显示出来。 + +| User | Host | Server | db | Command | Time | Trans | State | Info | +|---------------|----------------------|--------|------------------|---------|------|-------|------------|------| +| ddz_develop | 10.120.241.198:53728 | NULL | ddz_tower_test | Sleep | 0 | N | READ_QUERY | NULL | +| mail | 10.120.241.183:37960 | NULL | tech_mail_new_db | Sleep | 0 | N | READ_QUERY | NULL | + +结果说明： + +* User: 用户名; +* Host: 客户端的IP和端口; +* Server: 后端地址; +* db: 数据库名称; +* Command: 执行的sql，"Sleep"代表当前空闲; +* Time: 已执行的时间; +* Trans: 是否在事务中; +* State: 连接当前的状态，"READ_QUERY"代表在等待获取命令; +* Info: 暂未知。 + +## 用户/密码管理 + +### 密码查询 + +`SELECT * FROM user_pwd WHERE user=[username]` + +查询某个用户的后端密码。 +**注意由于密码是非明文的，仅能显示字节码。** + +>select * from user_pwd where user="root"; + +`SELECT * FROM app_user_pwd WHERE user=[username]` + +查询某个用户连接proxy的密码，同样是非明文。 + +>select * from app_user_pwd where user="test"; + +### 密码添加/修改 + +`UPDATE user_pwd SET password=[password1] WHERE user=[username]` + +添加或修改特定用户的后端密码(如果该用户不存在则添加，已存在则覆盖)。 +**需要Flush config生效** + +>update user_pwd set password="123456" where user="test"; + +>flush config; + +`UPDATE app_user_pwd SET password=[password1] WHERE user=[username]` + +添加或修改特定用户连接Proxy的密码(如果该用户不存在则添加，已存在则覆盖)。 +**需要Flush config生效** + +>update app_user_pwd set password="123456" where user="root"; + +>flush config; + +### 密码删除 + +`DELETE FROM user_pwd where user=[username]` + +删除特定用户的后端密码。 +**需要Flush config生效** + +>delete from user_pwd where user="root" + +>flush config; + +`DELETE FROM app_user_pwd where user=[username]` + +删除特定用户连接Proxy的密码。 +**需要Flush config生效** + +>delete from app_user_pwd where user="root" + +>flush config; + +### 查看未提交的修改 + +`SHOW CHANGES` + +查看已修改，但未flush的变更。 + +### 清除未提交的修改 + +`CLEAR CHANGES` + +清除已修改，但未flush的变更。 + +## IP许可 + +### 查看IP许可 + +`SHOW ALLOW_IP admin|proxy` + +查看Admin或者Proxy的IP许可。 +若列表为空，则代表没有任何限制。 + +### 增加IP许可 + +`ADD ALLOW_IP admin|proxy [[user@]IP]` + +增加一个IP许可。(不要加引号) + +* Admin: 仅能配置IP，不能限制用户(Admin有效用户只有一个)； +* Proxy: 仅配置IP，代表允许该IP来源所有用户的访问；配置User@IP，代表允许该IP来源的特定用户访问。 + +>add allow_ip admin 127.0.0.1 + +>add allow_ip proxy test@127.0.0.1 + +### 删除IP许可 + +`DELETE ALLOW_IP admin|proxy [[user@]IP]` + +删除一个IP许可。(不要加引号) + +>delete allow_ip admin 127.0.0.1 + +>delete allow_ip proxy test@127.0.0.1 + +## 远程管理 + +### 重载分库配置 + +`reload shard` + +需要"remote-config = true"和"disable-threads = false"启动选项。 +从远端配置库中重载Shard配置。 + +### 保存配置到本地文件 + +`SAVE SETTINGS [FILE]` + +保存当前配置到指定路径的本地文件中。 + +>save settings /tmp/proxy.cnf diff --git a/doc/Configuration.md b/doc/Configuration.md new file mode 100644 index 0000000..c6716ed --- /dev/null +++ b/doc/Configuration.md @@ -0,0 +1,369 @@ +# 配置文档 + +## 常规配置 + +### daemon + +Default: false + +通过守护进程启动。 + +> daemon = true + +### user + +Default: root + +启动进程的用户 + +> user = cetus + +### basedir + +基础路径，其它配置可以以此为基准配置相对路径。(必须是绝对路径) + +> basedir = /usr/lib/cetus + +### pid-file + +`必要` + +PID文件路径 + +> pid-file = /var/log/cetus.pid + +### log-file + +`必要` + +日志文件路径 + +> log-file = /var/log/cetus.log + +### log-level + +可选值: debug | info | message | warning | error | critical(default) + +日志级别 + +> log-level = info + +### plugins + +`可多项` + +加载模块名称。 + +> plugins = admin,proxy + +### plugin-dir + +库文件路径 + +> plugin-dir = /usr/lib/cetus/plugins + +## Proxy配置 + +### proxy-address + +Default: :4040 + +Proxy监听的IP和端口 + +> proxy-address = 127.0.0.1:4440 + +### proxy-allow-ip + +`可在Admin模块中动态更改` + +Proxy允许访问的"用户@IP" + +参数未设置时，没有限制；"User@IP"限制特定的用户和IP组合访问；"IP"允许该IP的所有用户访问 + +> proxy-allow-ip = root@127.0.0.1,10.238.7.6 + +### proxy-backend-addresses + +Default: 127.0.0.1:3306 + +`可多项` + +读写后端(主库)的IP和端口 + +> proxy-backend-addresses = 10.120.12.12:3306 + +若是分库模式，需要同时指定group + +> proxy-backend-addresses = 10.120.12.12:3306@data1 + +### proxy-read-only-backend-addresses + +`可多项` + +只读后端(从库)的IP和端口 + +> proxy-read-only-backend-addresses = 10.120.12.13:3307 + +若是分库模式，需要同时指定group + +> proxy-read-only-backend-addresses = 10.120.12.13:3307@data1 + +### default-username + +默认用户名，在Proxy启动时自动创建连接使用的用户名，在*user-pwd*中需要有对应的配置 + +> default-username = default_user + +### default-db + +默认数据库，当连接未指定db时，使用的默认数据库名称 + +> default-db = test + +### default-pool-size + +Default: 100 + +当前连接数不足此值时，会自动创建连接 + +> default-pool-size = 200 + +### max-pool-size + +Default: *default-pool-size * 2* + +连接池的最大连接数，超过此数目的连接不会放入连接池 + +> default-pool-size = 300 + +### max-resp-size + +Default: 10485760 (10MB) + +每个后端返回结果集的最大数量 + +> max-resp-size = 1024 + +### user-pwd + +`可在Admin模块动态更改` + +Proxy连接后端时使用的用户名、密码。用户名应与*app-user-pwd*中一一对应 + +> user-pwd = user1@password1,user2@password2 + +### app-user-pwd + +`可在Admin模块动态更改` + +客户端连接Proxy时使用的用户名、密码。用户名应与*user-pwd*中一一对应 + +> app-user-pwd = user1@apppass1,user2@apppass2 + +### crypt-pwd + +Default: false + +使用加密格式保存后端密码(user-pwd) + +> crypt-pwd = true + +### crypt-app-pwd + +Default: false + +使用加密格式保存客户端访问Proxy的密码(app-user-pwd) + +> crypt-app-pwd = true + +### disable-sharding-mode + +Default: false + +不启用分库功能(默认启用)，当只需要读写分离功能时设置为*true* + +> disable-sharding-mode = true + +### disable-auto-connect + +Default: false + +禁用自动创建连接,连接将在新请求到来时创建 + +> disable-auto-connect = false + +## Admin配置 + +### admin-address + +Default: :4041 + +管理模块的IP和端口 + +> admin-address = 127.0.0.1:4441 + +### admin-allow-ip + +`可在Admin模块中动态更改` + +参数未设置时，不作限制；仅能限制IP不区分用户 + +> admin-allow-ip = 127.0.0.1,10.238.7.6 + +### admin-lua-script + +`必要` + +管理模块对应的lua脚本路径 + +> admin-lua-script = /usr/lib/cetus/lua/admin.lua + +### admin-username + +`必要` + +管理模块的用户名 + +> admin-username = admin + +### admin-password + +`必要` + +管理模块的密码明文 + +> admin-password = admin_pass + +## 远端配置中心 + +目前分库配置仅能通过配置中心获取，因此要采用分库模式，必须配置远端db + +### config-remote + +Default: false + +是否启用远端配置中心 + +> config-remote = true + +### config-host + +Default: 127.0.0.1 + +配置中心host + +> config-host = 127.0.0.1 + +### config-port + +Default: 3306 + +配置中心端口 + +> config-port = 3310 + +### config-user + +Default: root + +配置中心用户 + +> config-user = test_user + +### config-pass + +配置中心密码 + +> config-pass = password + +### config-db + +Default: proxy_management + +配置中心db名称 + +> config-db = proxy_management + +### config-apply-interval + +Default: 0 (seconds) + +与配置中心同步配置的时间间隔。 +若设置为*0*，则配置更新不自动生效，仅能手动生效，且检测间隔为3秒。 + +> config-apply-interval = 0 + +### proxy-id + +Proxy的标识，在配置中心用来区分不同proxy的配置 + +> proxy-id = 001 + +## 辅助线程配置 + +### disable-threads + +Default: false + +禁用辅助线程，包括: 配置变更检测、后端存活检测和只读库延迟检测等 + +> disable-threads = true + +### connect-timeout + +Default: 2(seconds) + +检测线程连接后端mysql时的超时时间(秒) + +> connect-timeout = 10 + +### check-slave-delay + +Default: false + +是否检查从库延迟，需要配置`proxy-id`选项，否则不会生效 + +> check-slave-delay = true + +### slave-delay-down + +Default: 60 (seconds) + +从库延迟超过该秒，状态将被设置为DOWN + +> slave-delay-down = 120 + +### slave-delay-recover + +Default: *slave-delay-down / 2* (seconds) + +从库延迟少于该秒数，状态将恢复为UP + +> slave-delay-recover = 30 + +## 其它 + +### verbose-shutdown + +Default: false + +程序退出时，记录下退出代码。 + +> verbose-shutdown = true + +### keepalive + +Default: false + +当Proxy进程意外终止，会自动启动一个新进程 + +> keepalive = true + +### max-open-files + +Default: 根据操作系统 + +最大打开的文件数目(ulimit -n) + +> max-open-files = 10240 + diff --git a/doc/Makefile.am b/doc/Makefile.am new file mode 100644 index 0000000..a017421 --- /dev/null +++ b/doc/Makefile.am @@ -0,0 +1,30 @@ +SUBDIRS=chapter + +EXTRA_DIST = \ + lua-classes.dot \ + architecture.dot \ + architecture-overview.dot \ + architecture.txt \ + core.txt \ + chassis.txt \ + plugins.txt \ + protocol.txt \ + tests.txt \ + scripting.txt \ + lifecycle.msc + +clean-local: + rm -f *.html + +html-local: book.html protocol.html scripting.html + +## we use http://docutils.sourceforge.net/rst.html to generate the docs +book.html: book.txt chapter/scripting.txt chapter/protocol.txt + ${RST2HTML} $< $@ + +protocol.html: protocol.txt chapter/protocol.txt + ${RST2HTML} $< $@ + +scripting.html: scripting.txt chapter/scripting.txt + ${RST2HTML} $< $@ + diff --git a/doc/proxy.conf.example b/doc/proxy.conf.example new file mode 100644 index 0000000..c4774ba --- /dev/null +++ b/doc/proxy.conf.example @@ -0,0 +1,21 @@ +[cetus] +# Loaded Plugins +plugins=proxy,admin + +# Proxy Configuration +proxy-address=proxy-ip:proxy-port +proxy-backend-addresses=rw-ip:rw-port +proxy-read-only-backend-addresses=ro-ip:ro-port + +# Admin Configuration +admin-address=admin-ip:admin-port +admin-username=admin +admin-password=admin + +# Backend Configuration +default-db=test +default-username=dbtest + +# File and Log Configuration +log-file=cetus.log +log-level=debug diff --git a/doc/shard.conf.example b/doc/shard.conf.example new file mode 100644 index 0000000..a5669c9 --- /dev/null +++ b/doc/shard.conf.example @@ -0,0 +1,20 @@ +[cetus] +# Loaded Plugins +plugins=shard,admin + +# Proxy Configuration +proxy-address=proxy-ip:proxy-port +proxy-backend-addresses=ip1:port1@data1,ip2:port2@data2,ip3:port3@data3,ip4:port4@data4 + +# Admin Configuration +admin-address=admin-ip:admin-port +admin-username=admin +admin-password=admin + +# Backend Configuration +default-db=test +default-username=dbtest + +# Log Configuration +log-file=cetus.log +log-level=debug diff --git a/doc/sharding.json.example b/doc/sharding.json.example new file mode 100644 index 0000000..7bc63a6 --- /dev/null +++ b/doc/sharding.json.example @@ -0,0 +1,30 @@ +{ + "vdb": [ + { + "id": 1, + "type": "int", + "method": "hash", + "num": 8, + "partitions": {"data1": [0,1], "data2": [2,3], "data3": [4,5], "data4": [6,7]} + }, + { + "id": 2, + "type": "int", + "method": "range", + "num": 0, + "partitions": {"data1": 124999, "data2": 249999, "data3": 374999,"data4": 499999} + } + ], + "table": [ + {"vdb": 1, "db": "employees_hash", "table": "dept_emp", "pkey": "emp_no"}, + {"vdb": 1, "db": "employees_hash", "table": "employees", "pkey": "emp_no"}, + {"vdb": 1, "db": "employees_hash", "table": "titles", "pkey": "emp_no"}, + {"vdb": 2, "db": "employees_range", "table": "dept_emp", "pkey": "emp_no"}, + {"vdb": 2, "db": "employees_range", "table": "employees", "pkey": "emp_no"}, + {"vdb": 2, "db": "employees_range", "table": "titles", "pkey": "emp_no"} + ], + "single_tables": [ + {"table": "regioncode", "db": "employees_hash", "group": "data1"}, + {"table": "countries", "db": "employees_range", "group": "data2"} + ] +} diff --git a/doc/users.json.example b/doc/users.json.example new file mode 100644 index 0000000..4f74681 --- /dev/null +++ b/doc/users.json.example @@ -0,0 +1,11 @@ +{ + "users": [{ + "user": "test1", + "client_pwd": "123", + "server_pwd": "123456" + }, { + "user": "test2", + "client_pwd": "123456", + "server_pwd": "123456" + }] +} \ No newline at end of file diff --git a/doc/variables.json.example b/doc/variables.json.example new file mode 100644 index 0000000..55f349a --- /dev/null +++ b/doc/variables.json.example @@ -0,0 +1,17 @@ +{ + "variables": [ + { + "name": "sql_mode", + "type": "string-csv", + "allowed_values": ["STRICT_TRANS_TABLES", + "NO_AUTO_CREATE_USER", + "NO_ENGINE_SUBSTITUTION"] + }, + { + "name": "connect_timeout", + "type": "string", + "allowed_values": ["*"], + "silent_values": ["10", "100"] + } + ] +} diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt new file mode 100644 index 0000000..1c1f9a6 --- /dev/null +++ b/lib/CMakeLists.txt @@ -0,0 +1,90 @@ +# $%BEGINLICENSE%$ +# Copyright (c) 2009, 2012, Oracle and/or its affiliates. All rights reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2 of the +# License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA +# 02110-1301 USA +# +# $%ENDLICENSE%$ + +INCLUDE_DIRECTORIES(${GLIB_INCLUDE_DIRS}) +LINK_DIRECTORIES(${GLIB_LIBRARY_DIRS}) +LINK_DIRECTORIES(${LIBINTL_LIBRARY_DIRS}) + +INCLUDE_DIRECTORIES(${LUA_INCLUDE_DIRS}) +LINK_DIRECTORIES(${LUA_LIBRARY_DIRS}) + +INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/src/) +INCLUDE_DIRECTORIES(${PROJECT_BINARY_DIR}) # for config.h + +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) ## for the generated header file +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) ## for the packaged header file + +#lemon bin +add_executable(lemon ${CETUS_TOOLS_DIR}/lemon.c) + +option(SIMPLE_PARSER "use the simple parser") +if(SIMPLE_PARSER) + message("** Using simple parser [-DSIMPLE_PARSER=ON]") + set(LEMON_GRAMMAR_FILE "${CMAKE_CURRENT_SOURCE_DIR}/simple-parser.y") + set(LEMON_RAW_REPORT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/simple-parser.out") + add_custom_command(TARGET lemon PRE_BUILD COMMAND echo "@@@ USING SIMPLE PARSER [-DSIMPLE_PARSER=ON]") +else(SIMPLE_PARSER) + message("** Using full parser [-DSIMPLE_PARSER=OFF(default)]") + set(LEMON_GRAMMAR_FILE "${CMAKE_CURRENT_SOURCE_DIR}/myparser.y") + set(LEMON_RAW_REPORT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/myparser.out") + add_custom_command(TARGET lemon PRE_BUILD COMMAND echo "@@@ USING FULL PARSER [-DSIMPLE_PARSER=OFF]") +endif(SIMPLE_PARSER) + +#lemon parse +set(LEMON_PARSER_OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/myparser.y.c) +set(LEMON_OUTPUT_HEADER ${CMAKE_CURRENT_BINARY_DIR}/myparser.y.h) +set(LEMON_TEMPATE_FILE ${CETUS_TOOLS_DIR}/lempar.c) + +add_custom_command( + OUTPUT ${LEMON_PARSER_OUTPUT} ${SQL_TOKENS_HEADER} + DEPENDS ${LEMON_GRAMMAR_FILE} ${LEMON_TEMPATE_FILE} + ${CMAKE_CURRENT_SOURCE_DIR}/extra_tokens.inc + COMMAND lemon -s -T${LEMON_TEMPATE_FILE} ${LEMON_GRAMMAR_FILE} -o${LEMON_PARSER_OUTPUT} -h${LEMON_OUTPUT_HEADER} + + COMMAND cat ${CMAKE_CURRENT_SOURCE_DIR}/extra_tokens.inc >> ${LEMON_OUTPUT_HEADER} + ) + +set_source_files_properties(${LEMON_PARSER_OUTPUT} PROPERTIES GENERATED 1) +set_source_files_properties(${LEMON_OUTPUT_HEADER} PROPERTIES GENERATED 1) + +#flex -- depends on parse.h generated by lemon +find_package(FLEX) +set(FLEX_OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/mylexer.l.c) +FLEX_TARGET(MyLexer mylexer.l ${FLEX_OUTPUT} COMPILE_FLAGS + "--header-file=${CMAKE_CURRENT_BINARY_DIR}/mylexer.l.h") +set_source_files_properties(${FLEX_OUTPUT} + PROPERTIES GENERATED 1) + +set(SQL_PARSER_SOURCES + sql-expression.c + sql-operation.c + sql-property.c + sql-context.c + sql-construction.c + sql-filter-variables.c + ${FLEX_MyLexer_OUTPUTS} + ${LEMON_PARSER_OUTPUT} +) + +ADD_LIBRARY(sqlparser SHARED ${SQL_PARSER_SOURCES}) + +add_dependencies(sqlparser lemon) + +CHASSIS_INSTALL_TARGET(sqlparser) diff --git a/lib/extra_tokens.inc b/lib/extra_tokens.inc new file mode 100644 index 0000000..2388908 --- /dev/null +++ b/lib/extra_tokens.inc @@ -0,0 +1,12 @@ + +/* imported from extra_tokens.inc */ +#define TK_CUSTOM 250 /*TODO: get the last number automatically */ +#define TK_FUNCTION (TK_CUSTOM + 1) +#define TK_UMINUS (TK_CUSTOM + 2) +#define TK_UPLUS (TK_CUSTOM + 3) +#define TK_ISNOT (TK_CUSTOM + 4) + +#define TK_PROPERTY (TK_CUSTOM + 8) +#define TK_PROPERTY_START (TK_CUSTOM + 9) +#define TK_PROPERTY_END (TK_CUSTOM + 10) +#define TK_MYSQL_HINT (TK_CUSTOM + 11) diff --git a/lib/mylexer.l b/lib/mylexer.l new file mode 100644 index 0000000..50cacf7 --- /dev/null +++ b/lib/mylexer.l @@ -0,0 +1,298 @@ +%{ +#include +#include +#include "myparser.y.h" +%} + +%option reentrant +%option noyywrap +%option never-interactive + /*yyin and yyout set to NULL*/ +%option nostdinit + /*%option outfile="mylexer.c" header-file="mylexer.h"*/ +%option nodefault +%option warn +%option case-insensitive + +%x PROPERTY +%x COMMENT +%x MYSQL_HINT + +HEX_DIGIT [0-9a-fA-F] + +%% + +[ \t\r\n]+ /* ignore whitespace */; +";" return TK_SEMI; +"(" return TK_LP ; +")" return TK_RP ; +"," return TK_COMMA ; +"<>" return TK_NE ; +"!=" return TK_NE ; +"=" return TK_EQ ; +"==" return TK_EQ ; +">" return TK_GT ; +"<=" return TK_LE ; +"<" return TK_LT ; +">=" return TK_GE ; +"&" return TK_BITAND ; +"|" return TK_BITOR ; +"~" return TK_BITNOT ; +"<<" return TK_LSHIFT ; +">>" return TK_RSHIFT ; +"+" return TK_PLUS ; +"-" return TK_MINUS ; +"*" return TK_STAR ; +"/" return TK_SLASH ; +"%" return TK_REM ; /*REMAIN */ +"." return TK_DOT ; +"?" return TK_VARIABLE ; + +"TABLE" return TK_TABLE ; +"CREATE" return TK_CREATE ; +"IF" return TK_IF ; +"NOT" return TK_NOT ; +"!" return TK_NOT; +"EXISTS" return TK_EXISTS ; +"TEMP" return TK_TEMP ; +"AS" return TK_AS ; +"WITHOUT" return TK_WITHOUT ; +"OR" return TK_OR ; +"||" return TK_OR; +"AND" return TK_AND ; +"&&" return TK_AND; +"IS" return TK_IS ; +"MATCH" return TK_MATCH ; +"LIKE" return TK_LIKE_KW ; +"REGEXP" return TK_LIKE_KW; +"BETWEEN" return TK_BETWEEN ; +"IN" return TK_IN ; + /*"ISNULL" return TK_ISNULL ;*/ + /*"NOTNULL" return TK_NOTNULL ;*/ + /*"||" return TK_CONCAT ;*/ /*for ansi sql_mode*/ +"COLLATE" return TK_COLLATE ; +"ABORT" return TK_ABORT ; +"ACTION" return TK_ACTION ; +"AFTER" return TK_AFTER ; +"ANALYZE" return TK_ANALYZE ; +"ASC" return TK_ASC ; +"ATTACH" return TK_ATTACH ; +"BEFORE" return TK_BEFORE ; +"BEGIN" return TK_BEGIN; +"BY" return TK_BY ; +"CASCADE" return TK_CASCADE ; +"CAST" return TK_CAST ; +"COLUMN" return TK_COLUMNKW ; +"CONFLICT" return TK_CONFLICT ; +"DATABASE" return TK_DATABASE ; +"SCHEMA" return TK_SCHEMA; +"DESC" return TK_DESC ; +"DETACH" return TK_DETACH ; +"EACH" return TK_EACH ; +"END" return TK_END ; +"FAIL" return TK_FAIL ; +"FOR" return TK_FOR ; +"IGNORE" return TK_IGNORE ; +"INITIALLY" return TK_INITIALLY ; +"INSTEAD" return TK_INSTEAD ; +"NO" return TK_NO ; +"PLAN" return TK_PLAN ; +"QUERY" return TK_QUERY ; +"KEY" return TK_KEY ; +"OF" return TK_OF ; +"TO" return TK_TO; +"OFFSET" return TK_OFFSET ; +"PRAGMA" return TK_PRAGMA ; +"RAISE" return TK_RAISE ; +"RECURSIVE" return TK_RECURSIVE ; +"RELEASE" return TK_RELEASE ; +"REPLACE" return TK_REPLACE ; +"RESTRICT" return TK_RESTRICT ; +"ROW" return TK_ROW ; + +"TRANSACTION" return TK_TRANSACTION; +"START" return TK_START; +"COMMIT" return TK_COMMIT; +"ROLLBACK" return TK_ROLLBACK ; +"SAVEPOINT" return TK_SAVEPOINT ; + +"TRIGGER" return TK_TRIGGER ; +"VACUUM" return TK_VACUUM ; +"VIEW" return TK_VIEW ; +"VIRTUAL" return TK_VIRTUAL ; +"WITH" return TK_WITH ; +"INDEX" return TK_INDEX ; +"RENAME" return TK_RENAME ; + /* +"DATE" return TK_CTIME_KW ; +"TIME" return TK_CTIME_KW ; +"NOW" return TK_CTIME_KW ; +"TIMESTAMP" return TK_CTIME_KW ; +"TIMEDIFF" return TK_CTIME_KW ; + */ +"LEFT" return TK_JOIN_KW; +"RIGHT" return TK_JOIN_KW; +"INNER" return TK_JOIN_KW; +"OUTER" return TK_JOIN_KW; +"FULL" return TK_JOIN_KW; + +"ANY" return TK_ANY ; +"CONSTRAINT" return TK_CONSTRAINT ; +"DEFAULT" return TK_DEFAULT ; +"CHECK" return TK_CHECK ; +"AUTO_INCREMENT" return TK_AUTO_INCREMENT ; +"PRIMARY" return TK_PRIMARY ; +"UNIQUE" return TK_UNIQUE ; +"FOREIGN" return TK_FOREIGN ; +"DROP" return TK_DROP ; +"SELECT" return TK_SELECT ; +"DISTINCT" return TK_DISTINCT ; +"FROM" return TK_FROM ; +"JOIN" return TK_JOIN ; +"ON" return TK_ON ; +"USING" return TK_USING ; +"ORDER" return TK_ORDER ; +"GROUP" return TK_GROUP ; +"HAVING" return TK_HAVING ; +"LIMIT" return TK_LIMIT ; +"DELETE" return TK_DELETE ; +"WHERE" return TK_WHERE ; +"UPDATE" return TK_UPDATE ; +"SET" return TK_SET ; +"INTO" return TK_INTO ; +"VALUES" return TK_VALUES ; +"INSERT" return TK_INSERT ; +"NULL" return TK_NULL ; +"BLOB" return TK_BLOB ; +"CASE" return TK_CASE ; +"WHEN" return TK_WHEN ; +"THEN" return TK_THEN ; +"ELSE" return TK_ELSE ; +"EXPLAIN" return TK_EXPLAIN; +"DESCRIBE" return TK_DESCRIBE; +"SHOW" return TK_SHOW; +"SQL_CALC_FOUND_ROWS" return TK_SQL_CALC_FOUND_ROWS; +"NAMES" return TK_NAMES; +"USE" return TK_USE; +"CALL" return TK_CALL; +"CHARACTER" return TK_CHARACTER; +"CHARSET" return TK_CHARSET; +"SESSION" return TK_SESSION; +"GLOBAL" return TK_GLOBAL; +"KILL" return TK_KILL; +"READ" return TK_READ; +"WRITE" return TK_WRITE; +"ONLY" return TK_ONLY; +"UNION" return TK_UNION; +"ALL" return TK_ALL; +"DUPLICATE" return TK_DUPLICATE; +"INTERVAL" return TK_INTERVAL; +"DAY" return TK_TIME_UNIT; +"MONTH" return TK_TIME_UNIT; +"YEAR" return TK_TIME_UNIT; +"HOUR" return TK_TIME_UNIT; +"MINUTE" return TK_TIME_UNIT; +"SECOND" return TK_TIME_UNIT; +"SHARD_EXPLAIN" return TK_SHARD_EXPLAIN; +"ALTER" return TK_ALTER; +"FORCE" return TK_FORCE; +"COLUMNS" return TK_COLUMNS; +"FIELDS" return TK_FIELDS; +"COMMENT" return TK_COMMENT_KW; +"LOCK" return TK_LOCK;/*R*/ +"SHARE" return TK_SHARE; +"MODE" return TK_MODE; +"UNLOCK" return TK_UNLOCK;/*R*/ +"TABLES" return TK_TABLES; +"LOCAL" return TK_LOCAL; +"LOW_PRIORITY" return TK_LOW_PRIORITY;/*R*/ +"ISOLATION" return TK_ISOLATION; +"LEVEL" return TK_LEVEL; +"COMMITTED" return TK_COMMITTED; +"UNCOMMITTED" return TK_UNCOMMITTED; +"SERIALIZABLE" return TK_SERIALIZABLE; +"REPEATABLE" return TK_REPEATABLE; +"TRIM" return TK_TRIM; +"BOTH" return TK_TRIM_SPEC; +"LEADING" return TK_TRIM_SPEC; +"TRAILING" return TK_TRIM_SPEC; +"POSITION" return TK_POSITION; +"TRUNCATE" return TK_TRUNCATE; +"XA" return TK_XA; +"RECOVER" return TK_RECOVER; +"CURRENT_DATE" return TK_CURRENT_DATE; +"SIGNED" return TK_SIGNED; +"UNSIGNED" return TK_UNSIGNED; +"DECIMAL" return TK_DECIMAL; +"BINARY" return TK_BINARY; +"NCHAR" return TK_NCHAR; +"INT" return TK_INT_SYM; +"ESCAPE" return TK_ESCAPE; +"CETUS_SEQUENCE" return TK_CETUS_SEQUENCE; +"CETUS_VERSION" return TK_CETUS_VERSION; +"WARNINGS" return TK_WARNINGS; + +[0-9]+ return TK_INTEGER; /*sign symbol is handled in parser*/ +0[xX]{HEX_DIGIT}+ return TK_INTEGER; +[xX]'({HEX_DIGIT}{HEX_DIGIT})+' return TK_HEX_NUM; /* X'even num of digit' */ +0b[01]+ return TK_BIN_NUM; +[bB]'[01]+' return TK_BIN_NUM; + +[0-9]*\.[0-9]+([eE][-+]?[0-9]+)? return TK_FLOAT; + +[a-zA-Z][a-zA-Z0-9_]* return TK_ID; +"@" return TK_AT_SIGN; + +'([^'\\]|\\.|'')*' return TK_STRING; +\"([^"\\]|\"\"|\\.)*\" return TK_STRING; +`([^`]|``)*` return TK_ID; + + +"/*#" { BEGIN(PROPERTY); return TK_PROPERTY_START;} +[a-zA-Z0-9_.-]+ return TK_ID; +"=" return TK_EQ; +\"([^"\\]|\"\"|\\.)*\" return TK_STRING; +'([^'\\]|\\.|'')*' return TK_STRING; +[ \t\n\r,;]+ /* ignore */ +. { g_warning("### bad char in property comment: %02x\n", yytext[0]);} +"*"+"/" { BEGIN(INITIAL); return TK_PROPERTY_END;} + + +"/*" { BEGIN(COMMENT); } +[^*]* +"*"+[^*/]* +"*"+"/" { BEGIN(INITIAL); } + +"/*!" { BEGIN(MYSQL_HINT); } +[^*]* +"*"+[^*/]* +"*"+"/" { BEGIN(INITIAL); return TK_MYSQL_HINT;} + + +"#".* { /* useless comment */ } +"--"[ \t\n].* { /* useless comment */ } + +. {g_warning("### flex: bad input character: %02x\n", yytext[0]);return 0;} + +%% +/*hack function to restore user allocated string. +input string will be modified in the process of lexing +the string will be restored in the end of lexing, however, +if we break in the middle of lex process, the input +string will be left dirty, flex does not supply a +function to restore it. we use this function as a hack, +its declaration is in sql-expression.h +*/ + +void yylex_restore_buffer(void* yyscanner) +{ + struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; + + if ( YY_CURRENT_BUFFER ) + { + /* Flush out information for old buffer. */ + *yyg->yy_c_buf_p = yyg->yy_hold_char; + YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = yyg->yy_c_buf_p; + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = yyg->yy_n_chars; + } +} diff --git a/lib/myparser.y b/lib/myparser.y new file mode 100644 index 0000000..52e1070 --- /dev/null +++ b/lib/myparser.y @@ -0,0 +1,1299 @@ +// All token codes are small integers with #defines that begin with "TK_" +%token_prefix TK_ + +// The type of the data attached to each token is Token. This is also the +// default type for non-terminals. +// +%token_type {sql_token_t} +%default_type {sql_token_t} + +// The generated parser function takes a 4th argument as follows: +%extra_argument {sql_context_t *context} + +// This code runs whenever there is a syntax error +// +%syntax_error { + UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ + #define MAX_LEN 256 + char msg[MAX_LEN] = {0}; + snprintf(msg, MAX_LEN, "near \"%s\": syntax error", TOKEN.z); + sql_context_set_error(context, PARSE_SYNTAX_ERR, msg); +} + +%stack_overflow { + context->rc = PARSE_ERROR; + sql_context_append_msg(context, "parser stack overflow"); +} + +// The name of the generated procedure that implements the parser +// is as follows: +%name sqlParser + +// The following text is included near the beginning of the C source +// code file that implements the parser. +// +%include { +#include +#include +#include +#include +#include "sql-context.h" +#include "sql-expression.h" +#include "sql-operation.h" + +#define UNUSED_PARAMETER(x) (void)(x) +/* +** Disable all error recovery processing in the parser push-down +** automaton. +*/ +#define YYNOERRORRECOVERY 1 + +/* +** Make yytestcase() the same as testcase() +*/ +/*#define yytestcase(X) testcase(X)*/ + +/* +** Indicate that sqlParserFree() will never be called with a null +** pointer. +*/ +#define YYPARSEFREENEVERNULL 1 + +/* +** Alternative datatype for the argument to the malloc() routine passed +** into sqlParserAlloc(). The default is size_t. +*/ +#define YYMALLOCARGTYPE uint64_t + +/* +** An instance of this structure holds information about the +** LIMIT clause of a SELECT statement. +*/ +struct LimitVal { + sql_expr_t *pLimit; /* The LIMIT expression. NULL if there is no limit */ + sql_expr_t *pOffset; /* The OFFSET expression. NULL if there is none */ +}; + +/* +** An instance of this structure is used to store the LIKE, +** GLOB, NOT LIKE, and NOT GLOB operators. +*/ +struct LikeOp { + sql_token_t eOperator; /* "like" or "glob" or "regexp" */ + int bNot; /* True if the NOT keyword is present */ +}; + +struct transact_feature_t { + int rw_feature; + int isolation_level; +}; + +} // end %include + +// Input is a single SQL command +input ::= cmdlist. +cmdlist ::= cmdlist ecmd. +cmdlist ::= ecmd. +ecmd ::= SEMI. +ecmd ::= cmdx SEMI. { + context->stmt_count += 1; + if (context->stmt_count > 1) { + sql_context_set_error(context, PARSE_NOT_SUPPORT, + "multi-statement not support"); + } +} + +cmdx ::= cmd. +cmdx ::= select_stmt. +cmdx ::= update_stmt. +cmdx ::= delete_stmt. +cmdx ::= insert_stmt. + +///////////////////// EXPLAIN syntax //////////////////////////// +cmd ::= explain fullname(X) opt_col_name. { + context->rw_flag |= CF_READ; + sql_context_add_stmt(context, STMT_EXPLAIN_TABLE, X); +} +opt_col_name ::= ID|STRING. +opt_col_name ::= . + +cmd ::= explain explainable_stmt. +explainable_stmt ::= select_stmt. +explainable_stmt ::= insert_stmt. +explainable_stmt ::= update_stmt. +explainable_stmt ::= delete_stmt. + +explain ::= EXPLAIN. {context->explain = TK_EXPLAIN;} +explain ::= DESCRIBE. {context->explain = TK_EXPLAIN;} +explain ::= DESC. {context->explain = TK_EXPLAIN;} + +//////////////////// SHARD_EXPLAIN syntax ////////////////////// +cmd ::= SHARD_EXPLAIN explainable_stmt. { + context->explain = TK_SHARD_EXPLAIN; + context->clause_flags |= CF_LOCAL_QUERY; +} + +///////////////////// SHOW syntax ////////////////////////////// +cmd_head ::= SHOW opt_full. { + context->rw_flag |= CF_READ; + sql_context_add_stmt(context, STMT_SHOW, NULL); + context->rc = PARSE_HEAD; +} +cmd ::= SHOW opt_full COLUMNS|FIELDS FROM fullname(X) opt_db opt_wild_or_where. { + context->rw_flag |= CF_READ; + sql_context_add_stmt(context, STMT_SHOW_COLUMNS, X); +} +cmd ::= SHOW CREATE VIEW|TABLE fullname(X). { + context->rw_flag |= CF_READ; + sql_context_add_stmt(context, STMT_SHOW_CREATE, X); +} +cmd ::= SHOW WARNINGS. { + context->rw_flag |= CF_READ; + sql_context_add_stmt(context, STMT_SHOW_WARNINGS, NULL); +} + +opt_full ::= . +opt_full ::= JOIN_KW. + +opt_db ::= . +opt_db ::= FROM|IN ID. + +opt_wild_or_where ::= . +opt_wild_or_where ::= LIKE_KW STRING. +opt_wild_or_where ::= WHERE expr. /* destructor expr */ + +cmd ::= USE ID(X). {sql_use_database(context, sql_token_dup(X));} +cmd ::= CALL expr(X). { + context->rw_flag |= CF_READ; + sql_context_add_stmt(context, STMT_CALL, X); +} + +cmd ::= XA RECOVER. +cmd ::= XA COMMIT STRING. +cmd ::= XA ROLLBACK STRING. + +///////////////////// SET Command /////////////////////////////// +cmd ::= SET option_value_list(X). { + sql_set_variable(context, X); +} +cmd ::= SET NAMES ID|STRING(X) opt_collate. { + sql_set_names(context, sql_token_dup(X)); +} +opt_collate ::= COLLATE ID|STRING. +opt_collate ::= . + +cmd ::= SET opt_var_scope TRANSACTION transact_feature. { + sql_context_add_stmt(context, STMT_SET_TRANSACTION, NULL); +} + +%type transact_feature { struct transact_feature_t } +transact_feature(A) ::= READ ONLY. { + A.rw_feature = TF_READ_ONLY; + A.isolation_level = 0; +} +transact_feature(A) ::= READ WRITE. { + A.rw_feature = TF_READ_WRITE; + A.isolation_level = 0; +} +transact_feature(A) ::= ISOLATION LEVEL isolation_level(X). { + A.isolation_level = X; + A.rw_feature = 0; +} +%type isolation_level {int} +isolation_level(A) ::= REPEATABLE READ. {A = TF_REPEATABLE_READ;} +isolation_level(A) ::= READ COMMITTED. {A = TF_READ_COMMITTED;} +isolation_level(A) ::= READ UNCOMMITTED. {A = TF_READ_UNCOMMITTED;} +isolation_level(A) ::= SERIALIZABLE. {A = TF_SERIALIZABLE;} +%type opt_var_scope {enum sql_var_scope_t} +opt_var_scope(A) ::= GLOBAL. { A = SCOPE_GLOBAL; } +opt_var_scope(A) ::= SESSION. { A = SCOPE_SESSION; } +opt_var_scope(A) ::= . { A = SCOPE_TRANSIENT; } + + +%type option_type {enum sql_var_scope_t} +option_type(A) ::= GLOBAL. { A = SCOPE_GLOBAL; } +option_type(A) ::= SESSION. { A = SCOPE_SESSION; } +option_type(A) ::= LOCAL. {A = SCOPE_SESSION;} + +%type opt_var_ident_type {enum sql_var_scope_t} +opt_var_ident_type(A) ::= AT_SIGN AT_SIGN. {A = SCOPE_SESSION;} +opt_var_ident_type(A) ::= AT_SIGN AT_SIGN GLOBAL DOT. {A = SCOPE_GLOBAL;} +opt_var_ident_type(A) ::= AT_SIGN AT_SIGN SESSION DOT. {A = SCOPE_SESSION;} +opt_var_ident_type(A) ::= AT_SIGN AT_SIGN LOCAL DOT. {A = SCOPE_SESSION;} + +%type option_value {sql_expr_t*} // the [option = value] pair +%destructor option_value { sql_expr_free($$); } + +// set var=xx +option_value(A) ::= internal_variable_name(X) EQ set_expr_or_default(Y). { + X->var_scope = SCOPE_SESSION; + A=spanBinaryExpr(TK_EQ, X, Y); +} +// set SCOPE var=xx +option_value(A) ::= option_type(S) internal_variable_name(X) EQ set_expr_or_default(Y).{ + X->var_scope = S; + A=spanBinaryExpr(TK_EQ, X, Y); +} +// set @user_var=xx +option_value(A) ::= AT_SIGN ID(X) EQ expr(Y). { + sql_expr_t* lhs = sql_expr_new(TK_ID, &X); + lhs->var_scope = SCOPE_USER; + A=spanBinaryExpr(TK_EQ, lhs, Y); +} +// set @@scope.var=xx +option_value(A) ::= opt_var_ident_type(S) internal_variable_name(X) EQ set_expr_or_default(Y). { + X->var_scope = S; + A=spanBinaryExpr(TK_EQ, X, Y); +} + +%type internal_variable_name {sql_expr_t*} +%destructor internal_variable_name { sql_expr_free($$); } +internal_variable_name(A) ::= ID(X). { A = sql_expr_new(@X, &X); } +internal_variable_name(A) ::= ID(X) DOT ID(Y). { + sql_expr_t* p1 = sql_expr_new(TK_ID, &X); + sql_expr_t* p2 = sql_expr_new(TK_ID, &Y); + A = sql_expr_new(TK_DOT, 0); + sql_expr_attach_subtrees(A, p1, p2); +} + +%type set_expr_or_default {sql_expr_t*} +%destructor set_expr_or_default { sql_expr_free($$); } +set_expr_or_default(A) ::= expr(A). +set_expr_or_default(A) ::= DEFAULT(X). { A = sql_expr_new(@X, &X); } +set_expr_or_default(A) ::= BINARY(X). { A = sql_expr_new(@X, &X); } +set_expr_or_default(A) ::= ON(X). {A = sql_expr_new(@X, &X);} // set xx=on +set_expr_or_default(A) ::= ALL(X). { A = sql_expr_new(@X, &X); } + +%type option_value_list {sql_expr_list_t*} +%destructor option_value_list {sql_expr_list_free($$);} + +option_value_list(A) ::= option_value_list(A) COMMA option_value(Y). + {A = sql_expr_list_append(A,Y);} +option_value_list(A) ::= option_value(Y). + {A = sql_expr_list_append(0,Y); /*A-overwrites-Y*/} + +//////////////////////// KILL ! not supported ! //////////////////////////// +cmd ::= cmd_head ANY. +cmd_head ::= KILL. { + sql_context_set_error(context, PARSE_NOT_SUPPORT, + "KILL not yet supported by proxy"); +} + +///////////////////// Begin and end transactions. //////////////////////////// +// +cmd ::= START trans_opt. {sql_start_transaction(context);} +trans_opt ::= . +trans_opt ::= TRANSACTION. +trans_opt ::= TRANSACTION nm. + +cmd ::= BEGIN. {sql_start_transaction(context);} + +cmd ::= COMMIT trans_opt. {sql_commit_transaction(context);} +cmd ::= ROLLBACK trans_opt. {sql_rollback_transaction(context);} + +savepoint_opt ::= SAVEPOINT. +savepoint_opt ::= . +cmd ::= SAVEPOINT nm(X). { + sql_savepoint(context, TK_SAVEPOINT, sql_token_dup(X)); +} +cmd ::= RELEASE savepoint_opt nm(X). { + sql_savepoint(context, TK_RELEASE, sql_token_dup(X)); +} +cmd ::= ROLLBACK trans_opt TO savepoint_opt nm(X). { + sql_savepoint(context, TK_ROLLBACK, sql_token_dup(X)); +} + +%include { + /* This routine constructs a binary expression node out of two ExprSpan + ** objects and uses the result to populate a new ExprSpan object. + */ + static sql_expr_t* spanBinaryExpr( + int op, /* The binary operation */ + sql_expr_t* pLeft, /* The left operand*/ + sql_expr_t *pRight /* The right operand */ + ){ + sql_expr_t* the_op = sql_expr_new(op, 0); + sql_expr_attach_subtrees(the_op, pLeft, pRight); + return the_op; // output + } + + /* If doNot is true, then add a TK_NOT Expr-node wrapper around the + ** outside of *ppExpr. -- NOT BETWEEN/LIKE/IN -- + */ + static sql_expr_t* exprNot(int doNot, sql_expr_t *expr){ + if (doNot) { + sql_expr_t* not_op = sql_expr_new(TK_NOT, 0); + expr->flags |= EP_NOT; + sql_expr_attach_subtrees(not_op, expr, NULL); // not->start <- expr->start + not_op->end = expr->end; + return not_op; // output + } + return expr; + } +} + +// Define operator precedence early so that this is the first occurrence +// of the operator tokens in the grammer. Keeping the operators together +// causes them to be assigned integer values that are close together, +// which keeps parser tables smaller. +// +// The token values assigned to these symbols is determined by the order +// in which lemon first sees them. It must be the case that ISNULL/NOTNULL, +// NE/EQ, GT/LE, and GE/LT are separated by only a single value. See +// the sqlite3sql_expr_tIfFalse() routine for additional information on this +// constraint. +// +%left OR. +%left AND. +%right NOT. +%left IS MATCH LIKE_KW BETWEEN IN NE EQ. +%left GT LE LT GE. +%right ESCAPE. +%left BITAND BITOR LSHIFT RSHIFT. +%left PLUS MINUS. +%left STAR SLASH REM. +%left CONCAT. +%left COLLATE. +%right BITNOT. + + +// The following directive causes tokens ABORT, AFTER, ASC, etc. to +// fallback to ID if they will not parse as their original value. +// This obviates the need for the "id" nonterminal. +// +%fallback ID + ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BY CASCADE CAST COLUMNKW + CONFLICT DESC DETACH EACH END FAIL FOR + INITIALLY INSTEAD LIKE_KW MATCH NO PLAN + QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW + ROLLBACK SAVEPOINT START TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT + + RENAME IF + EXPLAIN DESCRIBE SHOW NAMES USE CALL + CHARACTER CHARSET SESSION GLOBAL KILL ONLY + DUPLICATE INTERVAL TIME_UNIT JOIN_KW SHARD_EXPLAIN + COLUMNS FIELDS COMMENT_KW SHARE MODE TABLES LOCAL + ISOLATION LEVEL COMMITTED UNCOMMITTED SERIALIZABLE REPEATABLE + XA RECOVER WARNINGS +// HACK fallback + CONSTRAINT CHECK AUTO_INCREMENT FOREIGN + TRIM POSITION TRUNCATE SIGNED NCHAR + . +%wildcard ANY. + +// The name of a column or table can be any of the following: +%type nm {sql_token_t} +nm(A) ::= ID(A). +nm(A) ::= JOIN_KW(A). + +// A typetoken is really zero or more tokens that form a type name such +// as can be found after the column name in a CREATE TABLE statement. +// Multiple tokens are concatenated to form the value of the typetoken. +// + +%token_class number INTEGER|FLOAT. + +// "carglist" is a list of additional constraints that come after the +// column name and column type in a CREATE TABLE statement. +// + +////////////////////////// The DROP TABLE ///////////////////////////////////// +// +cmd_head ::= ddl_cmd_head. { + context->stmt_type = STMT_COMMON_DDL; + context->rc = PARSE_HEAD; + context->rw_flag |= CF_WRITE|CF_DDL; +} + +ddl_cmd_head ::= DROP INDEX. +ddl_cmd_head ::= CREATE opt_unique INDEX. + +ddl_cmd_head ::= CREATE VIEW. +ddl_cmd_head ::= ALTER VIEW. +ddl_cmd_head ::= DROP VIEW. + +%token_class db_schema DATABASE|SCHEMA. +ddl_cmd_head ::= CREATE db_schema. +ddl_cmd_head ::= DROP db_schema. +ddl_cmd_head ::= ALTER db_schema. + +ddl_cmd_head ::= CREATE TABLE. +ddl_cmd_head ::= DROP TABLE. +ddl_cmd_head ::= TRUNCATE. /* optional TABLE */ +ddl_cmd_head ::= ALTER TABLE. + +opt_unique ::= UNIQUE. +opt_unique ::= . + +//////////////////////// The SELECT statement ///////////////////////////////// +// +select_stmt ::= select(X). { + context->rw_flag |= CF_READ; + sql_select(context, X); +} + +%type select { sql_select_t* } +%destructor select { sql_select_free($$); } +%type oneselect { sql_select_t* } +%destructor oneselect { sql_select_free($$); } + +select(A) ::= oneselect(A). + +select(A) ::= select(A) multiselect_op(Y) oneselect(Z). { + sql_select_t* rhs = Z; + sql_select_t* lhs = A; + if (rhs) { + rhs->op = Y; + rhs->prior = lhs;//single list + } + A = rhs; +} + +%type multiselect_op {int} +multiselect_op(A) ::= UNION. {A = TK_UNION;} +multiselect_op(A) ::= UNION ALL. {A = TK_UNION;} +multiselect_op(A) ::= UNION DISTINCT. {A = TK_UNION;} + +oneselect(A) ::= SELECT select_options(D) selcollist(C) from(F) where_opt(W) + groupby_opt(G) having_opt(H) orderby_opt(O) limit_opt(L) lock_read(R). { + A = sql_select_new(); + A->flags |= D; + A->columns = C; + A->from_src = F; + A->where_clause = W; + A->groupby_clause = G; + A->having_clause = H; + A->orderby_clause = O; + A->limit = L.pLimit; + A->offset = L.pOffset; + A->lock_read = R; +} + +oneselect(A) ::= values(A). + +%type values {sql_select_t*} +%destructor values { sql_select_free($$); } +values(A) ::= VALUES LP nexprlist(X) RP. { + A = sql_select_new(); + A->columns = X; +} +values(A) ::= values(A) COMMA LP exprlist(Y) RP. { + sql_select_t *right, *left = A; + right = sql_select_new(); + if (right) { + right->columns = Y; + right->flags |= SF_MULTI_VALUE; + right->prior = left; + A = right; + } else { + A = left; + } +} + +// The "distinct" nonterminal is true (1) if the DISTINCT keyword is +// present and false (0) if it is not. +// +%type select_options {int} +%type select_option {int} +select_options(A) ::= . { + A = 0; + context->parsing_place = SELECT_COLUMN; +} +select_options(A) ::= select_options(X) select_option(Y). { + A = X|Y; + context->parsing_place = SELECT_COLUMN; +} +select_option(A) ::= DISTINCT. {A = SF_DISTINCT;} +select_option(A) ::= ALL. {A = SF_ALL;} +select_option(A) ::= SQL_CALC_FOUND_ROWS. {A = SF_CALC_FOUND_ROWS;} + +%type distinct {int} +distinct(A) ::= DISTINCT. {A=1;} +distinct(A) ::= . {A=0;} + +%type lock_read {int} +lock_read(A) ::= FOR UPDATE. {A=1;} +lock_read(A) ::= LOCK IN SHARE MODE. {A=1;} +lock_read(A) ::=. {A=0;} + +// selcollist is a list of expressions that are to become the return +// values of the SELECT statement. The "*" in statements like +// "SELECT * FROM ..." is encoded as a special expression with an +// opcode of TK_ASTERISK. +// +%type selcollist { sql_expr_list_t* } +%destructor selcollist { sql_expr_list_free($$); $$ = 0; } +%type sclp { sql_expr_list_t* } +%destructor sclp { sql_expr_list_free($$); $$ = 0; } +sclp(A) ::= selcollist(A) COMMA. +sclp(A) ::= . { A = 0; } +selcollist(A) ::= sclp(A) expr(B) as(C). { + B->alias = sql_token_dup(C); + A = sql_expr_list_append(A, B); + context->parsing_place = SELECT_FROM; +} +selcollist(A) ::= sclp(A) STAR(B). { + sql_expr_t* p = sql_expr_new(@B, &B); + A = sql_expr_list_append(A, p); + context->parsing_place = SELECT_FROM; +} +selcollist(A) ::= sclp(A) nm(B) DOT STAR(C). { + sql_expr_t* left = sql_expr_new(TK_ID, &B); + sql_expr_t* right = sql_expr_new(@C, &C); + sql_expr_t* dot = sql_expr_new(TK_DOT, 0); + sql_expr_attach_subtrees(dot, left, right); + A = sql_expr_list_append(A, dot); + context->parsing_place = SELECT_FROM; +} + +// An option "AS " phrase that can follow one of the expressions that +// define the result set, or one of the tables in the FROM clause. +// +%type as {sql_token_t} +as(X) ::= AS nm(Y). {X = Y;} +as(X) ::= AS STRING(Y). {X = Y;} +as(X) ::= ID(X). +as(X) ::= . {X.z = 0; X.n = 0;} + + +%type seltablist {sql_src_list_t*} +%destructor seltablist { sql_src_list_free($$); } +%type stl_prefix {sql_src_list_t*} +%destructor stl_prefix { sql_src_list_free($$); } +%type from {sql_src_list_t*} +%destructor from { sql_src_list_free($$); } + +// A complete FROM clause. +// +from(A) ::= . { A = 0; } +from(A) ::= FROM seltablist(X). { A = X; } + +// "seltablist" is a "Select Table List" - the content of the FROM clause +// in a SELECT statement. "stl_prefix" is a prefix of this list. +// +stl_prefix(A) ::= seltablist(A) joinop(Y). { + if (A && A->len > 0) { + sql_src_item_t* item = g_ptr_array_index(A, A->len-1); + item->jointype = Y; + } +} +stl_prefix(A) ::= . {A = 0;} +seltablist(A) ::= stl_prefix(A) nm(Y) dotnm(D) as(Z) index_hint on_opt(N) using_opt(U). { + if (D.n) + A = sql_src_list_append(A,&D,&Y,&Z,0,N,U); + else + A = sql_src_list_append(A,&Y,0,&Z,0,N,U); +} +seltablist(A) ::= stl_prefix(A) nm(Y) dotnm(D) LP exprlist(E) RP as(Z) + on_opt(N) using_opt(U). { + if (D.n) + A = sql_src_list_append(A,&D,&Y,&Z,0,N,U); + else + A = sql_src_list_append(A,&Y,0,&Z,0,N,U); + sql_src_item_t* item = g_ptr_array_index(A, A->len-1); + item->func_arg = E; +} + +seltablist(A) ::= stl_prefix(A) LP select(S) RP + as(Z) on_opt(N) using_opt(U). { + context->clause_flags |= CF_SUBQUERY; + A = sql_src_list_append(A,0,0,&Z,S,N,U); +} +seltablist(A) ::= stl_prefix(A) LP seltablist(F) RP + as(Z) on_opt(N) using_opt(U). { + if (A==0 && Z.n==0 && N==0 && U==0) { + A = F; + } else if (F->len == 1) { + A = sql_src_list_append(A,0,0,&Z,0,N,U); + if (A) { + printf("not implemented");//TODO; + } + sql_src_list_free(F); + } else { + printf("not implemented");//TODO + } +} + +%type dotnm {sql_token_t} +dotnm(A) ::= . {A.z = 0; A.n = 0;} +dotnm(A) ::= DOT nm(X). {A = X;} + +%type fullname {sql_src_list_t*} +%destructor fullname { sql_src_list_free($$);} +fullname(A) ::= nm(B) dotnm(C). { + if (C.n) + A = sql_src_list_append(0,&C,&B,0,0,0,0); + else + A = sql_src_list_append(0,&B,0,0,0,0,0); +} + +%type joinop {int} +joinop(A) ::= COMMA|JOIN. { A = JT_INNER; } //TODO: JOIN TYPE +joinop(A) ::= JOIN_KW(X) JOIN. { + A = sql_join_type(X); +} +joinop(A) ::= JOIN_KW(X) nm(Y) JOIN.{ + A = sql_join_type(X)|sql_join_type(Y); +} +joinop(A) ::= JOIN_KW(X) nm(Y) nm(Z) JOIN.{ + A = sql_join_type(X)|sql_join_type(Y)|sql_join_type(Z); +} + +%type on_opt {sql_expr_t*} +%destructor on_opt {sql_expr_free($$);} +on_opt(N) ::= ON expr(E). {N = E;} +on_opt(N) ::= . {N = 0;} + + +%type using_opt {sql_id_list_t*} +%destructor using_opt {sql_id_list_free($$);} +using_opt(U) ::= USING LP idlist(L) RP. {U = L;} +using_opt(U) ::= . {U = 0;} + + +%type orderby_opt {sql_column_list_t*} +%destructor orderby_opt {sql_column_list_free($$);} + +// the sortlist non-terminal stores a list of expression where each +// expression is optionally followed by ASC or DESC to indicate the +// sort order. +// +%type sortlist {sql_column_list_t*} +%destructor sortlist {sql_column_list_free($$);} + +orderby_opt(A) ::= . {A = 0;} +orderby_opt(A) ::= ORDER BY sortlist(X). {A = X;} +sortlist(A) ::= sortlist(A) COMMA expr(Y) sortorder(Z). { + sql_column_t* col = sql_column_new(); + col->expr = Y; + col->sort_order = Z; + A = sql_column_list_append(A, col); +} +sortlist(A) ::= expr(Y) sortorder(Z). { + sql_column_t* col = sql_column_new(); + col->expr = Y; + col->sort_order = Z; + A = sql_column_list_append(0, col); +} + +%type sortorder {int} + +sortorder(A) ::= ASC. {A = SQL_SO_ASC;} +sortorder(A) ::= DESC. {A = SQL_SO_DESC;} +sortorder(A) ::= . {A = SQL_SO_ASC;/*default to asc*/} + +%type groupby_opt {sql_expr_list_t*} +%destructor groupby_opt {sql_expr_list_free($$);} +groupby_opt(A) ::= . {A = 0;} +groupby_opt(A) ::= GROUP BY nexprlist(X). {A = X;} + +%type having_opt {sql_expr_t*} +%destructor having_opt {sql_expr_free($$);} +having_opt(A) ::= . {A = 0;} +having_opt(A) ::= HAVING expr(X). {A = X;} + +%type limit_opt {struct LimitVal} + +// The destructor for limit_opt will never fire in the current grammar. +// The limit_opt non-terminal only occurs at the end of a single production +// rule for SELECT statements. As soon as the rule that create the +// limit_opt non-terminal reduces, the SELECT statement rule will also +// reduce. So there is never a limit_opt non-terminal on the stack +// except as a transient. So there is never anything to destroy. +// +//%destructor limit_opt { +// sqlite3ExprDelete(pParse->db, $$.pLimit); +// sqlite3ExprDelete(pParse->db, $$.pOffset); +//} +limit_opt(A) ::= . {A.pLimit = 0; A.pOffset = 0;} +limit_opt(A) ::= LIMIT expr(X). {A.pLimit = X; A.pOffset = 0;} +limit_opt(A) ::= LIMIT expr(X) OFFSET expr(Y). + {A.pLimit = X; A.pOffset = Y;} +limit_opt(A) ::= LIMIT expr(X) COMMA expr(Y). + {A.pOffset = X; A.pLimit = Y;} + +/////////////////////////// The DELETE statement ///////////////////////////// +// +delete_stmt ::= DELETE FROM fullname(X) where_opt(W) orderby_opt(O) limit_opt(L). { + sql_delete_t* del = sql_delete_new(); + del->from_src = X; + del->where_clause = W; + del->orderby_clause = O; + del->limit = L.pLimit; + del->offset = L.pOffset; + sql_delete(context, del); +} + +%type where_opt {sql_expr_t*} +%destructor where_opt {sql_expr_free($$);} + +where_opt(A) ::= . {A = 0;} +where_opt(A) ::= where_sym expr(X). { + A = X; + if (context->where_flags & EP_LAST_INSERT_ID) { + sql_context_set_error(context, PARSE_NOT_SUPPORT, + "(proxy unsupported) last_insert_id in WHERE clause"); + } +} + +where_sym ::= WHERE. { + context->where_flags = 0; +} + + +////////////////////////// The UPDATE command //////////////////////////////// +// +update_stmt ::= UPDATE table_reference(X) SET update_list(Y) where_opt(W) orderby_opt(O) limit_opt(L). { + sql_update_t* p = sql_update_new(); + p->table = X; + p->set_list = Y; + p->where_clause = W; + p->orderby_clause = O; + p->limit = L.pLimit; + p->offset = L.pOffset; + sql_update(context, p); +} + + +%type update_list {sql_expr_list_t*} +%destructor update_list {sql_expr_list_free($$);} +update_list(A) ::= update_list(A) COMMA update_elem(Y). { + A = sql_expr_list_append(A, Y); +} +update_list(A) ::= update_elem(Y). { + A = sql_expr_list_append(0, Y); +} + + +%type update_elem {sql_expr_t*} +%destructor update_elem { sql_expr_free($$); } +update_elem(A) ::= simple_ident_nospvar(X) EQ expr(Y). { + A = spanBinaryExpr(TK_EQ, X, Y); +} + + +%type simple_ident_nospvar {sql_expr_t*} // not stored program variable +%destructor simple_ident_nospvar { sql_expr_free($$); } +simple_ident_nospvar(A) ::= ID(X). { + A = sql_expr_new(@X, &X); +} +simple_ident_nospvar(A) ::= simple_ident_q(A). + + +%type simple_ident_q {sql_expr_t*} // qualified id +%destructor simple_ident_q { sql_expr_free($$); } +simple_ident_q(A) ::= ID(X) DOT ID(Y). { + sql_expr_t* p1 = sql_expr_new(TK_ID, &X); + sql_expr_t* p2 = sql_expr_new(TK_ID, &Y); + A = sql_expr_new(TK_DOT, 0); + sql_expr_attach_subtrees(A, p1, p2); +} +simple_ident_q(A) ::= DOT ID(X) DOT ID(Y). { + sql_expr_t* p1 = sql_expr_new(TK_ID, &X); + sql_expr_t* p2 = sql_expr_new(@Y, &Y); + A = sql_expr_new(TK_DOT, 0); + sql_expr_attach_subtrees(A, p1, p2); +} +simple_ident_q(A) ::= ID(X) DOT ID(Y) DOT ID(Z). { + sql_expr_t* p1 = sql_expr_new(TK_ID, &X); + sql_expr_t* p2 = sql_expr_new(TK_ID, &Y); + sql_expr_t* p3 = sql_expr_new(TK_ID, &Z); + sql_expr_t* p4 = sql_expr_new(TK_DOT, 0); + sql_expr_attach_subtrees(p4, p2, p3); + A = sql_expr_new(TK_DOT, 0); + sql_expr_attach_subtrees(A, p1, p4); +} + + +%type table_reference {sql_src_list_t*} +%destructor table_reference {sql_src_list_free($$);} +table_reference(A) ::= fullname(A) as index_hint. +index_hint ::= . +index_hint ::= USE INDEX|KEY index_for LP index_list RP. +index_hint ::= IGNORE INDEX|KEY index_for LP index_list RP. +index_hint ::= FORCE INDEX|KEY index_for LP index_list RP. +index_for ::= . +index_for ::= FOR JOIN. +index_for ::= FOR ORDER BY. +index_for ::= FOR GROUP BY. +index_list ::= index_list COMMA ID. +index_list ::= ID|PRIMARY. + +////////////////////////// The INSERT command ///////////////////////////////// +// +insert_stmt ::= insert_cmd(R) INTO fullname(X) idlist_opt(F) select(S). { + sql_insert_t* p = sql_insert_new(); + p->is_replace = R; + p->table = X; + p->columns = F; + p->sel_val = S; + sql_insert(context, p); +} +insert_stmt ::= insert_cmd(R) INTO fullname(X) idlist_opt(F) DEFAULT VALUES. { + sql_insert_t* p = sql_insert_new(); + p->is_replace = R; + p->table = X; + p->columns = F; + p->sel_val = 0; + sql_insert(context, p); +} +insert_stmt ::= insert_cmd(R) INTO fullname(X) idlist_opt(F) values(S) + ON DUPLICATE KEY UPDATE exprlist. { + sql_insert_t* p = sql_insert_new(); + p->is_replace = R; + p->table = X; + p->columns = F; + p->sel_val = S; + sql_insert(context, p); +} + +%type insert_cmd {int} +insert_cmd(A) ::= INSERT. {A=0;} +insert_cmd(A) ::= REPLACE. {A = 1;} +insert_cmd(A) ::= INSERT IGNORE. {A=1;} + +%type idlist_opt {sql_id_list_t*} +%destructor idlist_opt {sql_id_list_free($$);} +%type idlist {sql_id_list_t*} +%destructor idlist {sql_id_list_free($$);} + +idlist_opt(A) ::= . {A = 0;} +idlist_opt(A) ::= LP idlist(X) RP. {A = X;} +idlist(A) ::= idlist(A) COMMA nm(Y). + {A = sql_id_list_append(A,&Y);} +idlist(A) ::= nm(Y). + {A = sql_id_list_append(0,&Y); /*A-overwrites-Y*/} + +%type concat_str {sql_token_t} +concat_str(A) ::= STRING(A). +concat_str(A) ::= concat_str(A) STRING. // TODO: collect all + +/////////////////////////// sql_expression Processing ///////////////////////////// +// +%include { + static void spanSet(sql_expr_t *pOut, sql_token_t *pStart, sql_token_t *pEnd){ + pOut->start = pStart->z; + pOut->end = &pEnd->z[pEnd->n]; + } + + static sql_expr_t* function_expr_new(sql_token_t* name, + sql_expr_list_t* args, sql_token_t* rparenth) + { + sql_expr_t* func_expr = sql_expr_new(TK_FUNCTION, name); + func_expr->flags |= EP_FUNCTION; + func_expr->list = args; + if (rparenth) { + func_expr->end = &rparenth->z[rparenth->n]; + } + return func_expr; + } +} + +%type expr {sql_expr_t*} +%destructor expr { sql_expr_free($$); } +%type term {sql_expr_t*} +%destructor term { sql_expr_free($$); } + +expr(A) ::= term(A). +expr(A) ::= LP(B) expr(X) RP(E). { + A = X; + spanSet(A, &B, &E); +} +term(A) ::= NULL(X). {A = sql_expr_new(@X, &X);} +expr(A) ::= ID(X). {A = sql_expr_new(@X, &X);} +expr(A) ::= JOIN_KW(X). {A = sql_expr_new(@X, &X);} +expr(A) ::= nm(X) DOT nm(Y). { + sql_expr_t* p1 = sql_expr_new(TK_ID, &X); + sql_expr_t* p2 = sql_expr_new(TK_ID, &Y); + A = sql_expr_new(TK_DOT, 0); + sql_expr_attach_subtrees(A, p1, p2); +} + +expr(A) ::= nm(X) DOT ANY(Y). {// allow x.reserved-keyword + sql_expr_t* p1 = sql_expr_new(TK_ID, &X); + sql_expr_t* p2 = sql_expr_new(@Y, &Y); + A = sql_expr_new(TK_DOT, 0); + sql_expr_attach_subtrees(A, p1, p2); +} + +expr(A) ::= nm(X) DOT nm(Y) DOT nm(Z). { + sql_expr_t* p1 = sql_expr_new(TK_ID, &X); + sql_expr_t* p2 = sql_expr_new(TK_ID, &Y); + sql_expr_t* p3 = sql_expr_new(TK_ID, &Z); + sql_expr_t* p4 = sql_expr_new(TK_DOT, 0); + sql_expr_attach_subtrees(p4, p2, p3); + A = sql_expr_new(TK_DOT, 0); + sql_expr_attach_subtrees(A, p1, p4); +} +term(A) ::= INTEGER|FLOAT|BIN_NUM|HEX_NUM|BLOB(X). {A = sql_expr_new(@X, &X);} +term(A) ::= concat_str(X). {A = sql_expr_new(TK_STRING, &X);} //TODO: span error +expr(A) ::= VARIABLE(X). {A = sql_expr_new(@X, &X);} +expr(A) ::= expr(A) COLLATE ID|STRING(X). { + sql_expr_t* coll = sql_expr_new(@X, &X); + sql_expr_attach_subtrees(A, coll, NULL); + A->end = &X.z[X.n]; +} + +%type func_expr {sql_expr_t*} +%destructor func_expr { sql_expr_free($$); } + +expr(A) ::= func_expr(X). { + A = X; + context->where_flags |= EP_FUNCTION; +} + +func_expr(A) ::= ID(X) LP distinct(D) exprlist(Y) RP(R). { + A = function_expr_new(&X, Y, &R); + if (strncasecmp(X.z, "last_insert_id", X.n) == 0) { + sql_context_set_error(context, PARSE_NOT_SUPPORT, + "(proxy)LAST_INSERT_ID() not supported"); + //TODO: parse interupted, func_expr free? + } + if (sql_func_type(X.z) != FT_UNKNOWN) { + A->flags |= EP_AGGREGATE; + } + if (D) { + A->flags |= EP_DISTINCT; + context->clause_flags |= CF_DISTINCT_AGGR; + } +} +func_expr(A) ::= ID(X) LP STAR RP(R). { + A = function_expr_new(&X, 0, &R); + if (sql_func_type(X.z) != FT_UNKNOWN) { + A->flags |= EP_AGGREGATE; + } +} +func_expr(A) ::= JOIN_KW(N) LP expr(X) COMMA expr(Y) RP(R). { + sql_expr_list_t *args = sql_expr_list_append(0, X); + sql_expr_list_append(args, Y); + A = function_expr_new(&N, args, &R); +} +func_expr(A) ::= INSERT(N) LP expr(X) COMMA expr(Y) COMMA expr(Z) COMMA expr(W) RP(R). { + sql_expr_list_t *args = sql_expr_list_append(0, X); + sql_expr_list_append(args, Y); + sql_expr_list_append(args, Z); + sql_expr_list_append(args, W); + A = function_expr_new(&N, args, &R); +} +func_expr(A) ::= TRIM(N) LP expr RP(R). { + A = function_expr_new(&N, 0, &R); +} +func_expr(A) ::= TRIM(N) LP expr FROM expr RP(R). { + A = function_expr_new(&N, 0, &R); +} +func_expr(A) ::= TRIM(N) LP TRIM_SPEC expr FROM expr RP(R). { + A = function_expr_new(&N, 0, &R); +} +func_expr(A) ::= TRIM(N) LP TRIM_SPEC FROM expr RP(R). { + A = function_expr_new(&N, 0, &R); +} +func_expr(A) ::= POSITION(N) LP STRING IN expr RP(R). { + A = function_expr_new(&N, 0, &R); +} +func_expr(A) ::= CURRENT_DATE(N) opt_parentheses. { + A = function_expr_new(&N, 0, NULL); + if (context->parsing_place == SELECT_COLUMN) { + context->clause_flags |= CF_LOCAL_QUERY; + } +} +func_expr(A) ::= CETUS_SEQUENCE(N) opt_parentheses. { + A = function_expr_new(&N, 0, NULL); + if (context->parsing_place == SELECT_COLUMN) { + context->clause_flags |= CF_LOCAL_QUERY; + } +} +func_expr(A) ::= CETUS_VERSION(N) opt_parentheses. { + A = function_expr_new(&N, 0, NULL); + if (context->parsing_place == SELECT_COLUMN) { + context->clause_flags |= CF_LOCAL_QUERY; + } +} +func_expr(A) ::= CAST(N) LP expr AS cast_type RP(R). { + A = function_expr_new(&N, 0, &R); +} +func_expr(A) ::= DATABASE(N) LP RP(R). { + A = function_expr_new(&N, 0, &R); +} + +opt_parentheses ::= LP RP. +opt_parentheses ::= . + +expr(A) ::= expr(A) AND expr(Y). {A=spanBinaryExpr(TK_AND, A, Y);} +expr(A) ::= expr(A) OR expr(Y). {A=spanBinaryExpr(TK_OR, A, Y);} +expr(A) ::= expr(A) LT|GT|GE|LE(OP) expr(Y). + {A=spanBinaryExpr(@OP, A, Y);} +expr(A) ::= expr(A) EQ|NE(OP) expr(Y). {A=spanBinaryExpr(@OP, A, Y);} +expr(A) ::= expr(A) BITAND|BITOR|LSHIFT|RSHIFT(OP) expr(Y). + {A=spanBinaryExpr(@OP, A, Y);} +expr(A) ::= expr(A) PLUS|MINUS(OP) expr(Y). + {A=spanBinaryExpr(@OP, A, Y);} +expr(A) ::= expr(A) STAR|SLASH|REM(OP) expr(Y). + {A=spanBinaryExpr(@OP, A, Y);} +expr(A) ::= expr(A) CONCAT(OP) expr(Y). {A=spanBinaryExpr(@OP, A, Y);} + +expr(A) ::= variable(A). + +%type variable {sql_expr_t*} +%destructor variable {sql_expr_free($$);} + +variable(A) ::= opt_var_ident_type(S) ID(X). { // sys var + A = sql_expr_new(@X, &X); //TODO: span error + A->var_scope = S; + if (strcasecmp(X.z, "last_insert_id") == 0) { + context->where_flags |= EP_LAST_INSERT_ID; + } +} +variable(A) ::= AT_SIGN ID(X). {A = sql_expr_new(@X, &X);} // user var + +expr(A) ::= predicate(A). + +%type likeop {struct LikeOp} +likeop(A) ::= LIKE_KW|MATCH(X). {A.eOperator = X; A.bNot = 0;/*A-overwrites-X*/} +likeop(A) ::= NOT LIKE_KW|MATCH(X). {A.eOperator = X; A.bNot = 1;} + +%type predicate {sql_expr_t*} +%destructor predicate {sql_expr_free($$);} +predicate(A) ::= expr(X) likeop(OP) expr(Y). [LIKE_KW] { + sql_expr_list_t *args = sql_expr_list_append(0, X); + args = sql_expr_list_append(args, Y); + sql_expr_t* like_expr = sql_expr_new(TK_LIKE_KW, 0); + if (like_expr) { + like_expr->list = args; + like_expr->start = X->start; + like_expr->end = Y->end; + } else { + sql_expr_list_free(args); + } + A = exprNot(OP.bNot, like_expr); +} +predicate(A) ::= expr(X) likeop(OP) expr(Y) ESCAPE expr(E). [LIKE_KW] { + sql_expr_list_t *args = sql_expr_list_append(0, X); + args = sql_expr_list_append(args, Y); + args = sql_expr_list_append(args, E); + sql_expr_t* like_expr = sql_expr_new(TK_LIKE_KW, 0); + if (like_expr) { + like_expr->list = args; + like_expr->start = X->start; + like_expr->end = E->end; + } else { + sql_expr_list_free(args); + } + A = exprNot(OP.bNot, like_expr); +} + +expr(A) ::= expr(A) IS expr(Y). { + A=spanBinaryExpr(TK_IS,A,Y); +} +expr(A) ::= expr(A) IS NOT expr(Y). { + A=spanBinaryExpr(TK_ISNOT,A,Y); +} + +%include { + /* Construct an expression node for a unary prefix operator + */ + static sql_expr_t* spanUnaryPrefix( + int op, /* The operator */ + sql_expr_t* pOperand, /* The operand */ + sql_token_t* token + ){ + sql_expr_t* the_op = sql_expr_new(op, 0); + sql_expr_attach_subtrees(the_op, pOperand, NULL); + the_op->start = token->z; + the_op->end = pOperand->end; + return the_op; + } +} + + +expr(A) ::= NOT(B) expr(X). + {A=spanUnaryPrefix(TK_NOT, X, &B);/*A-overwrites-B*/} +expr(A) ::= BITNOT(B) expr(X). + {A=spanUnaryPrefix(TK_BITNOT, X, &B);/*A-overwrites-B*/} +expr(A) ::= MINUS(B) expr(X). [BITNOT] + {A=spanUnaryPrefix(TK_UMINUS, X, &B);/*A-overwrites-B*/} +expr(A) ::= PLUS(B) expr(X). [BITNOT] + {A=spanUnaryPrefix(TK_UPLUS, X, &B);/*A-overwrites-B*/} + +%type between_op {int} +between_op(A) ::= BETWEEN. {A = 0;} +between_op(A) ::= NOT BETWEEN. {A = 1;} +expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] { + sql_expr_list_t* pList = sql_expr_list_append(0, X); + pList = sql_expr_list_append(pList, Y); + sql_expr_t* betw_op = sql_expr_new(TK_BETWEEN, 0); + if (betw_op) { + sql_expr_attach_subtrees(betw_op, A, NULL); + betw_op->list = pList; + betw_op->end = Y->end; + A = betw_op; + } else { + sql_expr_list_free(pList); + } + A = exprNot(N, A); +} + +%type in_op {int} +in_op(A) ::= IN. {A = 0;} +in_op(A) ::= NOT IN. {A = 1;} +expr(A) ::= expr(B) in_op(N) LP exprlist(Y) RP(R). [IN] { + A = sql_expr_new(TK_IN, 0); + if (A) { + sql_expr_attach_subtrees(A, B, NULL); + A->list = Y; + A->end = &R.z[R.n]; + } else { + sql_expr_list_free(Y); + } + A = exprNot(N, A); +} +expr(A) ::= LP(L) select(X) RP(R). { + context->clause_flags |= CF_SUBQUERY; + A = sql_expr_new(TK_SELECT, 0); + if (A) { + spanSet(A, &L, &R); + A->select = X; + } else { + sql_expr_free(X); + }; +} +/* example: (id, name) IN (select ..) */ +expr(A) ::= LP(L) expr(B) COMMA nexprlist RP in_op(N) LP select(Y) RP(R). [IN] { + A = sql_expr_new(TK_IN, 0); + if (A) { + A->select = Y; + sql_expr_attach_subtrees(A, B, NULL); + spanSet(A, &L, &R); + } else { + sql_select_free(Y); + sql_expr_free(B); + } + A = exprNot(N, A); +} +expr(A) ::= expr(B) in_op(N) LP select(Y) RP(R). [IN] { + A = sql_expr_new(TK_IN, 0); + if (A) { + sql_expr_attach_subtrees(A, B, NULL); + A->end = &R.z[R.n]; + A->select = Y; + } else { + sql_select_free(Y); + sql_expr_free(B); + } + A = exprNot(N, A); +} +expr(A) ::= EXISTS(E) LP select(Y) RP(R). { + A = sql_expr_new(TK_EXISTS, 0); + A->select = Y; + spanSet(A, &E, &R); +} + +expr(A) ::= INTERVAL(I) expr TIME_UNIT(T). { + context->where_flags |= EP_INTERVAL; + A = sql_expr_new(TK_INTERVAL, 0); //TODO: interval? + spanSet(A, &I, &T); +} + +/* CASE expressions */ +expr(A) ::= CASE(C) case_operand(X) case_exprlist(Y) case_else(Z) END(E). { + A = sql_expr_new(TK_CASE, 0); + if (A) { + sql_expr_attach_subtrees(A, X, NULL); + A->list = Z ? sql_expr_list_append(Y, Z) : Y; + spanSet(A, &C, &E); + } else { + sql_expr_list_free(Y); + sql_expr_free(Z); + } + context->where_flags |= EP_CASE_WHEN; +} + +%type case_exprlist {sql_expr_list_t*} +%destructor case_exprlist {sql_expr_list_free($$);} +case_exprlist(A) ::= case_exprlist(A) WHEN expr(Y) THEN expr(Z). { + A = sql_expr_list_append(A, Y); + A = sql_expr_list_append(A, Z); +} +case_exprlist(A) ::= WHEN expr(Y) THEN expr(Z). { + A = sql_expr_list_append(0, Y); + A = sql_expr_list_append(A, Z); +} + +%type case_else {sql_expr_t*} +%destructor case_else {sql_expr_free($$);} +case_else(A) ::= ELSE expr(X). {A = X;} //TODO: span +case_else(A) ::= . {A = 0;} + +%type case_operand {sql_expr_t*} +%destructor case_operand {sql_expr_free($$);} +case_operand(A) ::= expr(X). {A = X; /*A-overwrites-X*/} +case_operand(A) ::= . {A = 0;} + +%type exprlist {sql_expr_list_t*} +%destructor exprlist {sql_expr_list_free($$);} +%type nexprlist {sql_expr_list_t*} +%destructor nexprlist {sql_expr_list_free($$);} + +exprlist(A) ::= nexprlist(A). +exprlist(A) ::= . {A = 0;} +nexprlist(A) ::= nexprlist(A) COMMA expr(Y). + {A = sql_expr_list_append(A,Y);} +nexprlist(A) ::= expr(Y). + {A = sql_expr_list_append(0,Y); /*A-overwrites-Y*/} + + +// The eidlist non-terminal (sql_expr_tession Id List) generates an sql_expr_list_t +// from a list of identifiers. The identifier names are in sql_expr_list_t.a[].zName. +// This list is stored in an sql_expr_list_t rather than an sql_id_list_t so that it +// can be easily sent to sqlite3Columnssql_expr_list_t(). +// +// eidlist is grouped with CREATE INDEX because it used to be the non-terminal +// used for the arguments to an index. That is just an historical accident. +// +// IMPORTANT COMPATIBILITY NOTE: Some prior versions of SQLite accepted +// COLLATE clauses and ASC or DESC keywords on ID lists in inappropriate +// places - places that might have been stored in the sqlite_master schema. +// Those extra features were ignored. But because they might be in some +// (busted) old databases, we need to continue parsing them when loading +// historical schemas. +// +%type eidlist {sql_expr_list_t*} +%destructor eidlist {sql_expr_list_free($$);} +%type eidlist_opt {sql_expr_list_t*} +%destructor eidlist_opt {sql_expr_list_free($$);} + + +///////////////////////LOCK TABLES/////////////////////////// +cmd ::= LOCK TABLES lock_tables. { + sql_context_set_error(context, PARSE_NOT_SUPPORT, + "(cetus) LOCK TABLES not supported"); +} +lock_tables ::= fullname as lock_type. +lock_tables ::= lock_tables COMMA fullname as lock_type. +lock_type ::= READ opt_local. +lock_type ::= opt_priority WRITE. +opt_local ::= LOCAL. +opt_local ::= . +opt_priority ::= LOW_PRIORITY. +opt_priority ::= . +cmd ::= UNLOCK TABLES. { + sql_context_set_error(context, PARSE_NOT_SUPPORT, + "(cetus) UNLOCK TABLES not supported"); +} + +cast_type ::= SIGNED. +cast_type ::= UNSIGNED. +cast_type ::= SIGNED INT_SYM. +cast_type ::= UNSIGNED INT_SYM. +cast_type ::= BINARY opt_field_length. +cast_type ::= NCHAR opt_field_length. +cast_type ::= DECIMAL float_options. + +float_options ::= . +float_options ::= field_length. +float_options ::= precision. + +precision ::= LP INTEGER COMMA INTEGER RP. + +field_length ::= LP INTEGER RP. +opt_field_length ::= . +opt_field_length ::= field_length. diff --git a/lib/simple-parser.y b/lib/simple-parser.y new file mode 100644 index 0000000..a311ec5 --- /dev/null +++ b/lib/simple-parser.y @@ -0,0 +1,1032 @@ +// All token codes are small integers with #defines that begin with "TK_" +%token_prefix TK_ + +// The type of the data attached to each token is Token. This is also the +// default type for non-terminals. +// +%token_type {sql_token_t} +%default_type {sql_token_t} + +// The generated parser function takes a 4th argument as follows: +%extra_argument {sql_context_t *context} + +// This code runs whenever there is a syntax error +// +%syntax_error { + UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ + // since this is an incomplete parser, we don't know for sure whether a clause + // has syntax error, just leave it to the backend + context->rc = PARSE_UNRECOGNIZED; + context->rw_flag |= CF_WRITE;// unrecognized sql direct to WRITE server +} + +%stack_overflow { + context->rc = PARSE_ERROR; + sql_context_append_msg(context, "parser stack overflow"); +} + +// The name of the generated procedure that implements the parser +// is as follows: +%name sqlParser + +// The following text is included near the beginning of the C source +// code file that implements the parser. +// +%include { +#include +#include +#include +#include +#include "sql-context.h" +#include "sql-operation.h" + +#define UNUSED_PARAMETER(x) (void)(x) +/* +** Disable all error recovery processing in the parser push-down +** automaton. +*/ +#define YYNOERRORRECOVERY 1 + +/* +** Make yytestcase() the same as testcase() +*/ +/*#define yytestcase(X) testcase(X)*/ + +/* +** Indicate that sqlParserFree() will never be called with a null +** pointer. +*/ +#define YYPARSEFREENEVERNULL 1 + +/* +** Alternative datatype for the argument to the malloc() routine passed +** into sqlParserAlloc(). The default is size_t. +*/ +#define YYMALLOCARGTYPE uint64_t + +/* +** An instance of this structure holds information about the +** LIMIT clause of a SELECT statement. +*/ +struct LimitVal { + sql_expr_t *pLimit; /* The LIMIT expression. NULL if there is no limit */ + sql_expr_t *pOffset; /* The OFFSET expression. NULL if there is none */ +}; + +/* +** An instance of this structure is used to store the LIKE, +** GLOB, NOT LIKE, and NOT GLOB operators. +*/ +struct LikeOp { + sql_token_t eOperator; /* "like" or "glob" or "regexp" */ + int bNot; /* True if the NOT keyword is present */ +}; + +struct transact_feature_t { + int rw_feature; + int isolation_level; +}; + +} // end %include + +// Input is a single SQL command +input ::= cmdlist. +cmdlist ::= cmdlist ecmd. +cmdlist ::= ecmd. +ecmd ::= SEMI. +ecmd ::= cmdx SEMI. { + context->stmt_count += 1; +} + +cmdx ::= cmd. +cmdx ::= select_stmt. +cmdx ::= update_stmt. +cmdx ::= delete_stmt. +cmdx ::= insert_stmt. + +///////////////////// EXPLAIN syntax //////////////////////////// +cmd ::= explain fullname opt_col_name. +opt_col_name ::= ID|STRING. +opt_col_name ::= . + +cmd ::= explain explainable_stmt. +explainable_stmt ::= select_stmt. +explainable_stmt ::= insert_stmt. +explainable_stmt ::= update_stmt. +explainable_stmt ::= delete_stmt. + +explain ::= EXPLAIN. {context->explain = TK_EXPLAIN;} +explain ::= DESCRIBE. {context->explain = TK_EXPLAIN;} +explain ::= DESC. {context->explain = TK_EXPLAIN;} + +//////////////////// SHARD_EXPLAIN syntax ////////////////////// +cmd ::= SHARD_EXPLAIN explainable_stmt. { + context->explain = TK_SHARD_EXPLAIN; +} + +///////////////////// Mysql Special ////////////////////////////// +cmd_head ::= SHOW. { + context->rw_flag |= CF_READ; + sql_context_add_stmt(context, STMT_SHOW, NULL); + context->rc = PARSE_HEAD; +} +cmd ::= SHOW WARNINGS. { + context->rw_flag |= CF_READ; + sql_context_add_stmt(context, STMT_SHOW_WARNINGS, NULL); +} +cmd ::= USE ID(X). {sql_use_database(context, sql_token_dup(X));} +cmd ::= CALL expr(X). { + context->rw_flag |= CF_READ; + sql_context_add_stmt(context, STMT_CALL, X); +} + +anylist ::= ANY. +anylist ::= anylist ANY. + +///////////////////// SET Command /////////////////////////////// +cmd ::= SET nexprlist(X). { + sql_set_variable(context, X); +} +cmd ::= SET NAMES ID|STRING(X). { + sql_set_names(context, sql_token_dup(X)); +} + +cmd ::= SET opt_var_scope(S) TRANSACTION transact_feature(T). { + context->rc = PARSE_HEAD; + sql_set_transaction(context, S, T.rw_feature, T.isolation_level); +} + +%type transact_feature { struct transact_feature_t } +transact_feature(A) ::= READ ONLY. { + A.rw_feature = TF_READ_ONLY; + A.isolation_level = 0; +} +transact_feature(A) ::= READ WRITE. { + A.rw_feature = TF_READ_WRITE; + A.isolation_level = 0; +} +transact_feature(A) ::= ISOLATION LEVEL isolation_level(X). { + A.isolation_level = X; + A.rw_feature = 0; +} +%type isolation_level {int} +isolation_level(A) ::= REPEATABLE READ. {A = TF_REPEATABLE_READ;} +isolation_level(A) ::= READ COMMITTED. {A = TF_READ_COMMITTED;} +isolation_level(A) ::= READ UNCOMMITTED. {A = TF_READ_UNCOMMITTED;} +isolation_level(A) ::= SERIALIZABLE. {A = TF_SERIALIZABLE;} + +%type var_scope {enum sql_var_scope_t} +var_scope(A) ::= GLOBAL. { A = SCOPE_GLOBAL; } +var_scope(A) ::= SESSION. { A = SCOPE_SESSION; } +var_scope(A) ::= AT_SIGN AT_SIGN GLOBAL DOT. { A = SCOPE_GLOBAL; } +var_scope(A) ::= AT_SIGN AT_SIGN SESSION DOT. { A = SCOPE_SESSION; } +var_scope(A) ::= AT_SIGN AT_SIGN. { A = SCOPE_SESSION; } +var_scope(A) ::= AT_SIGN. { A = SCOPE_USER; } + +%type opt_var_scope {enum sql_var_scope_t} +opt_var_scope(A) ::= GLOBAL. { A = SCOPE_GLOBAL; } +opt_var_scope(A) ::= SESSION. { A = SCOPE_SESSION; } +opt_var_scope(A) ::= . { A = SCOPE_TRANSIENT; } + + +//////////////////////// KILL ! not supported ! //////////////////////////// +cmd ::= cmd_head ANY. +cmd_head ::= KILL. { + sql_context_set_error(context, PARSE_NOT_SUPPORT, + "KILL not yet supported by proxy"); +} + +///////////////////// Begin and end transactions. //////////////////////////// +// + +%token_class begin_trans BEGIN|START. +cmd ::= begin_trans trans_opt. {sql_start_transaction(context);} +trans_opt ::= . +trans_opt ::= TRANSACTION. +trans_opt ::= TRANSACTION nm. + +cmd ::= COMMIT trans_opt. {sql_commit_transaction(context);} +cmd ::= ROLLBACK trans_opt. {sql_rollback_transaction(context);} + +savepoint_opt ::= SAVEPOINT. +savepoint_opt ::= . +cmd ::= SAVEPOINT nm(X). { + sql_savepoint(context, TK_SAVEPOINT, sql_token_dup(X)); +} +cmd ::= RELEASE savepoint_opt nm(X). { + sql_savepoint(context, TK_RELEASE, sql_token_dup(X)); +} +cmd ::= ROLLBACK trans_opt TO savepoint_opt nm(X). { + sql_savepoint(context, TK_ROLLBACK, sql_token_dup(X)); +} + +///////////////////// The CREATE TABLE statement //////////////////////////// +// +cmd ::= create_table create_table_args. +create_table ::= CREATE temp TABLE ifnotexists nm dotnm. { + context->rw_flag |= CF_WRITE; +} + +%type ifnotexists {int} +ifnotexists(A) ::= . {A = 0;} +ifnotexists(A) ::= IF NOT EXISTS. {A = 1;} +%type temp {int} +temp(A) ::= TEMP. {A = 1;} +temp(A) ::= . {A = 0;} + +create_table_args ::= LP columnlist conslist_opt RP table_options. +create_table_args ::= AS select. +//%type table_options {sql_expr_list_t*} +//%destructor table_options {sql_expr_list_free($$);} +table_options ::= . +table_options ::= table_optlist. + +table_optlist ::= table_opt. +table_optlist ::= table_optlist COMMA table_opt. +table_optlist ::= table_optlist table_opt. + +%include { + /* This routine constructs a binary expression node out of two ExprSpan + ** objects and uses the result to populate a new ExprSpan object. + */ + static sql_expr_t* spanBinaryExpr( + int op, /* The binary operation */ + sql_expr_t* pLeft, /* The left operand*/ + sql_expr_t *pRight /* The right operand */ + ){ + sql_expr_t* the_op = sql_expr_new(op, 0); + sql_expr_attach_subtrees(the_op, pLeft, pRight); + return the_op; // output + } + + /* If doNot is true, then add a TK_NOT Expr-node wrapper around the + ** outside of *ppExpr. + */ + static sql_expr_t* exprNot(int doNot, sql_expr_t *pSpan){ + if (doNot) { + sql_expr_t* not_op = sql_expr_new(TK_NOT, 0); + sql_expr_attach_subtrees(not_op, pSpan, NULL); + return not_op; // output + } + return pSpan; + } +} + +table_opt ::= ID opt_equal expr. +table_opt ::= ct_opt_charset opt_equal expr. + +ct_opt_charset ::= opt_default CHARACTER SET. +ct_opt_charset ::= opt_default CHARSET. +opt_default ::= . +opt_default ::= DEFAULT. + +opt_equal ::= . +opt_equal ::= EQ. + +columnlist ::= columnlist COMMA columnname carglist. +columnlist ::= columnname carglist. +columnname ::= nm typetoken. + +// Define operator precedence early so that this is the first occurrence +// of the operator tokens in the grammer. Keeping the operators together +// causes them to be assigned integer values that are close together, +// which keeps parser tables smaller. +// +// The token values assigned to these symbols is determined by the order +// in which lemon first sees them. It must be the case that ISNULL/NOTNULL, +// NE/EQ, GT/LE, and GE/LT are separated by only a single value. See +// the sqlite3sql_expr_tIfFalse() routine for additional information on this +// constraint. +// +%left OR. +%left AND. +%right NOT. +%left IS MATCH LIKE_KW BETWEEN IN NE EQ. +%left GT LE LT GE. +%right ESCAPE. +%left BITAND BITOR LSHIFT RSHIFT. +%left PLUS MINUS. +%left STAR SLASH REM. +%left CONCAT. +%left COLLATE. +%right BITNOT. + + +// The following directive causes tokens ABORT, AFTER, ASC, etc. to +// fallback to ID if they will not parse as their original value. +// This obviates the need for the "id" nonterminal. +// +%fallback ID + ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW + CONFLICT DATABASE DESC DETACH EACH END FAIL FOR + IGNORE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN + QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW + ROLLBACK SAVEPOINT START TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT + + RENAME IF + EXPLAIN DESCRIBE SHOW NAMES USE CALL + CHARACTER CHARSET SESSION GLOBAL KILL ONLY + DUPLICATE INTERVAL TIME_UNIT SHARD_EXPLAIN + COLUMNS FIELDS COMMENT_KW SHARE MODE TABLES LOCAL + ISOLATION LEVEL COMMITTED UNCOMMITTED SERIALIZABLE REPEATABLE + XA RECOVER WARNINGS +// all terminals in the full parser should be here, so that the +// generated header is compatible +// these keywords are recognized but not parsed as is +// JOIN ON USING WHERE ORDER GROUP HAVING LIMIT + INTO TRIM_SPEC TRIM POSITION TRUNCATE SIGNED UNSIGNED DECIMAL + BINARY NCHAR INT_SYM CETUS_SEQUENCE SCHEMA + . +%wildcard ANY. + + +// And "ids" is an identifer-or-string. +// +%token_class ids ID.//|STRING. + +// The name of a column or table can be any of the following: +%type nm {sql_token_t} +nm(A) ::= ID(A). +nm(A) ::= JOIN_KW(A). + +// A typetoken is really zero or more tokens that form a type name such +// as can be found after the column name in a CREATE TABLE statement. +// Multiple tokens are concatenated to form the value of the typetoken. +// +%type typetoken {sql_token_t} +typetoken(A) ::= . {A.z = 0; A.n = 0;} +typetoken(A) ::= typename(A). +typetoken(A) ::= typename(A) LP signed RP. +typetoken(A) ::= typename(A) LP signed COMMA signed RP. +%type typename {sql_token_t} +typename(A) ::= ids(A). +typename(A) ::= typename(A) ids. +signed ::= plus_num. +signed ::= minus_num. + +%token_class number INTEGER|FLOAT. +plus_num(A) ::= PLUS number(X). {A = X;} +plus_num(A) ::= number(A). +minus_num(A) ::= MINUS number(X). {A = X;} + +// "carglist" is a list of additional constraints that come after the +// column name and column type in a CREATE TABLE statement. +// +carglist ::= carglist ccons. +carglist ::= . +ccons ::= CONSTRAINT nm. +ccons ::= DEFAULT term. +ccons ::= DEFAULT LP expr RP. +ccons ::= DEFAULT PLUS term. +ccons ::= DEFAULT MINUS term. +ccons ::= DEFAULT ID. + +// In addition to the type name, we also care about the primary key and +// UNIQUE constraints. +// +ccons ::= NULL. +ccons ::= NOT NULL autoinc. +ccons ::= PRIMARY KEY sortorder autoinc. +ccons ::= UNIQUE. +ccons ::= CHECK LP expr RP. + +ccons ::= COLLATE ids. +ccons ::= COMMENT_KW concat_str. + +// The optional AUTOINCREMENT keyword +%type autoinc {int} +autoinc(X) ::= . {X = 0;} +autoinc(X) ::= AUTO_INCREMENT. {X = 1;} + +conslist_opt(A) ::= . {A.z = 0; A.n = 0;} +conslist_opt(A) ::= COMMA(A) conslist. +conslist ::= conslist tconscomma tcons. +conslist ::= tcons. +tconscomma ::= COMMA. +tconscomma ::= . +tcons ::= CONSTRAINT nm. +tcons ::= PRIMARY KEY LP sortlist autoinc RP. +tcons ::= UNIQUE LP sortlist RP. +tcons ::= UNIQUE KEY LP sortlist RP. +tcons ::= CHECK LP expr RP. +tcons ::= FOREIGN KEY LP eidlist RP. + +////////////////////////// The DROP TABLE ///////////////////////////////////// +// +cmd ::= DROP TABLE ifexists nm. { + context->rw_flag |= CF_WRITE; + sql_context_add_stmt(context, STMT_COMMON_DDL, NULL); +} +%type ifexists {int} +ifexists(A) ::= IF EXISTS. {A = 1;} +ifexists(A) ::= . {A = 0;} + +//////////////////////// ALTER TABLE ////////////////////////////////// +cmd ::= ALTER TABLE. { + context->rw_flag |= CF_WRITE; + sql_context_add_stmt(context, STMT_COMMON_DDL, NULL); + context->rc = PARSE_HEAD; +} + +//////////////////////// The SELECT statement ///////////////////////////////// +// +select_stmt ::= select(X). { + context->rw_flag |= CF_READ; + sql_select(context, X); +} + +%type select { sql_select_t* } +%destructor select { sql_select_free($$); } +%type oneselect { sql_select_t* } +%destructor oneselect { sql_select_free($$); } + +select(A) ::= oneselect(A). + +select(A) ::= select(A) multiselect_op(Y) oneselect(Z). { + sql_select_t* rhs = Z; + sql_select_t* lhs = A; + if (rhs) { + rhs->op = Y; + rhs->prior = lhs;//single list + } + A = rhs; +} + +%type multiselect_op {int} +multiselect_op(A) ::= UNION. {A = TK_UNION;} +multiselect_op(A) ::= UNION ALL. {A = TK_UNION;} +multiselect_op(A) ::= UNION DISTINCT. {A = TK_UNION;} + +oneselect(A) ::= SELECT select_options(D) selcollist(C) from where_opt + groupby_opt having_opt orderby_opt limit_opt lock_read(R). { + A = sql_select_new(); + A->flags |= D; + A->columns = C; + A->lock_read = R; + if (R) { + context->rw_flag |= CF_WRITE; + } +} + +oneselect(A) ::= values(A). + +%type values {sql_select_t*} +%destructor values { sql_select_free($$); } +values(A) ::= VALUES LP nexprlist(X) RP. { + A = sql_select_new(); + A->columns = X; +} +values(A) ::= values(A) COMMA LP exprlist(Y) RP. { + sql_select_t *pRight, *pLeft = A; + pRight = sql_select_new(); + //if (pLeft)pLeft->selFlags &= ~SF_MultiValue; + if (pRight) { + pRight->columns = Y; + //pRight->op = TK_ALL; + pRight->prior = pLeft; + A = pRight; + } else { + A = pLeft; + } +} + +// The "distinct" nonterminal is true (1) if the DISTINCT keyword is +// present and false (0) if it is not. +// +%type select_options {int} +%type select_option {int} +select_options(A) ::= . {A = 0;} +select_options(A) ::= select_options(X) select_option(Y). { A = X|Y; } +select_option(A) ::= DISTINCT. {A = SF_DISTINCT;} +select_option(A) ::= ALL. {A = SF_ALL;} +select_option(A) ::= SQL_CALC_FOUND_ROWS. {A = SF_CALC_FOUND_ROWS;} + +%type distinct {int} +distinct(A) ::= DISTINCT. {A=1;} +distinct(A) ::= . {A=0;} + +%type lock_read {int} +lock_read(A) ::= FOR UPDATE. {A=1;} +lock_read(A) ::= LOCK IN SHARE MODE. {A=1;} +lock_read(A) ::=. {A=0;} + +// selcollist is a list of expressions that are to become the return +// values of the SELECT statement. The "*" in statements like +// "SELECT * FROM ..." is encoded as a special expression with an +// opcode of TK_ASTERISK. +// +%type selcollist { sql_expr_list_t* } +%destructor selcollist { sql_expr_list_free($$); $$ = 0; } +%type sclp { sql_expr_list_t* } +%destructor sclp { sql_expr_list_free($$); $$ = 0; } +sclp(A) ::= selcollist(A) COMMA. +sclp(A) ::= . { A = 0; } +selcollist(A) ::= sclp(A) expr(B) as(C). { + B->alias = sql_token_dup(C); + A = sql_expr_list_append(A, B); +} +selcollist(A) ::= sclp(A) STAR(B). { + sql_expr_t* p = sql_expr_new(@B, &B); + A = sql_expr_list_append(A, p); +} +selcollist(A) ::= sclp(A) nm(B) DOT STAR(C). { + sql_expr_t* left = sql_expr_new(TK_ID, &B); + sql_expr_t* right = sql_expr_new(@C, &C); + sql_expr_t* dot = sql_expr_new(TK_DOT, 0); + sql_expr_attach_subtrees(dot, left, right); + A = sql_expr_list_append(A, dot); +} + +// An option "AS " phrase that can follow one of the expressions that +// define the result set, or one of the tables in the FROM clause. +// +%type as {sql_token_t} +as(X) ::= AS nm(Y). {X = Y;} +as(X) ::= ids(X). +as(X) ::= . {X.z = 0; X.n = 0;} + +// A complete FROM clause. +// +from ::= . +from ::= FROM seltablist. + +// "seltablist" is a "Select Table List" - the content of the FROM clause +// in a SELECT statement. "stl_prefix" is a prefix of this list. +// +stl_prefix ::= seltablist joinop. +stl_prefix ::= . +seltablist ::= stl_prefix nm dotnm as index_hint on_opt using_opt. +seltablist ::= stl_prefix nm dotnm LP exprlist RP as on_opt using_opt. +seltablist ::= stl_prefix LP select RP as on_opt using_opt. +seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt. + +%type dotnm {sql_token_t} +dotnm(A) ::= . {A.z = 0; A.n = 0;} +dotnm(A) ::= DOT nm(X). {A = X;} + +%type fullname {sql_src_list_t*} +%destructor fullname { sql_src_list_free($$);} +fullname(A) ::= nm(B) dotnm(C). { + if (C.n) + A = sql_src_list_append(0,&C,&B,0,0,0,0); + else + A = sql_src_list_append(0,&B,0,0,0,0,0); +} + +%type joinop {int} +joinop(X) ::= COMMA|JOIN. { X = JT_LEFT; } //TODO: JOIN TYPE +joinop(X) ::= JOIN_KW JOIN. { X = JT_LEFT; } +joinop(X) ::= JOIN_KW nm JOIN. { X = JT_LEFT; } +joinop(X) ::= JOIN_KW nm nm JOIN. { X = JT_LEFT; } + +%type on_opt {sql_expr_t*} +%destructor on_opt {sql_expr_free($$);} +on_opt(N) ::= ON expr(E). {N = E;} +on_opt(N) ::= . {N = 0;} + + +%type using_opt {sql_id_list_t*} +%destructor using_opt {sql_id_list_free($$);} +using_opt(U) ::= USING LP idlist(L) RP. {U = L;} +using_opt(U) ::= . {U = 0;} + + +%type orderby_opt {sql_column_list_t*} +%destructor orderby_opt {sql_column_list_free($$);} + +// the sortlist non-terminal stores a list of expression where each +// expression is optionally followed by ASC or DESC to indicate the +// sort order. +// +%type sortlist {sql_column_list_t*} +%destructor sortlist {sql_column_list_free($$);} + +orderby_opt(A) ::= . {A=0;} +orderby_opt(A) ::= ORDER BY sortlist. {A=0;} +sortlist(A) ::= sortlist COMMA expr sortorder. { A = 0;} +sortlist(A) ::= expr sortorder. { A = 0;} + +%type sortorder {int} + +sortorder(A) ::= ASC. {A = 0;} +sortorder(A) ::= DESC. {A = 1;} +sortorder(A) ::= . {A = -1;} + +//%type groupby_opt {sql_expr_list_t*} +//%destructor groupby_opt {sql_expr_list_free($$);} +groupby_opt ::= . +groupby_opt ::= GROUP BY nexprlist. + +//%type having_opt {sql_expr_t*} +//%destructor having_opt {sql_expr_free($$);} +having_opt ::= . +having_opt ::= HAVING expr. + +%type limit_opt {struct LimitVal} + +// The destructor for limit_opt will never fire in the current grammar. +// The limit_opt non-terminal only occurs at the end of a single production +// rule for SELECT statements. As soon as the rule that create the +// limit_opt non-terminal reduces, the SELECT statement rule will also +// reduce. So there is never a limit_opt non-terminal on the stack +// except as a transient. So there is never anything to destroy. +// +//%destructor limit_opt { +// sqlite3ExprDelete(pParse->db, $$.pLimit); +// sqlite3ExprDelete(pParse->db, $$.pOffset); +//} +limit_opt ::= . +limit_opt ::= LIMIT expr. +limit_opt ::= LIMIT expr OFFSET expr. +limit_opt ::= LIMIT expr COMMA expr. + + +/////////////////////////// The DELETE statement ///////////////////////////// +// +delete_stmt ::= delete_kw FROM fullname where_opt orderby_opt limit_opt. +delete_kw ::= DELETE. { + context->rw_flag |= CF_WRITE; + sql_context_add_stmt(context, STMT_DELETE, NULL); +} + +where_opt ::= . +where_opt ::= where_sym expr. { + if (context->where_flags & EP_LAST_INSERT_ID) { + sql_context_set_error(context, PARSE_NOT_SUPPORT, + "(proxy unsupported) last_insert_id in WHERE clause"); + } +} + +where_sym ::= WHERE. { + context->where_flags = 0; +} +////////////////////////// The UPDATE command //////////////////////////////// +// +update_stmt ::= update_kw table_reference SET setlist where_opt orderby_opt limit_opt. +update_kw ::= UPDATE. { + context->rw_flag |= CF_WRITE; + sql_context_add_stmt(context, STMT_UPDATE, NULL); +} + +setlist ::= setlist COMMA expr. +setlist ::= expr. + +table_reference ::= fullname as index_hint. +index_hint ::= . +index_hint ::= USE INDEX|KEY index_for LP index_list RP. +index_hint ::= IGNORE INDEX|KEY index_for LP index_list RP. +index_hint ::= FORCE INDEX|KEY index_for LP index_list RP. +index_for ::= . +index_for ::= FOR JOIN. +index_for ::= FOR ORDER BY. +index_for ::= FOR GROUP BY. +index_list ::= index_list COMMA ID. +index_list ::= ID|PRIMARY. + +////////////////////////// The INSERT command ///////////////////////////////// +// +insert_stmt ::= insert_cmd anylist. +insert_cmd ::= INSERT|REPLACE. { + context->rw_flag |= CF_WRITE; + sql_context_add_stmt(context, STMT_INSERT, NULL); +} + +%type idlist_opt {sql_id_list_t*} +%destructor idlist_opt {sql_id_list_free($$);} +%type idlist {sql_id_list_t*} +%destructor idlist {sql_id_list_free($$);} + +idlist(A) ::= idlist(A) COMMA nm(Y). + {A = sql_id_list_append(A,&Y);} +idlist(A) ::= nm(Y). + {A = sql_id_list_append(0,&Y); /*A-overwrites-Y*/} + +%type concat_str {sql_token_t} +concat_str(A) ::= STRING(A). +concat_str(A) ::= concat_str(A) STRING. // TODO: collecet all + +/////////////////////////// sql_expression Processing ///////////////////////////// +// + +%type expr {sql_expr_t*} +%destructor expr { sql_expr_free($$); } +%type term {sql_expr_t*} +%destructor term { sql_expr_free($$); } + +expr(A) ::= term(A). +expr(A) ::= LP expr(X) RP. {A = X;} +term(A) ::= NULL(X). {A = sql_expr_new(@X, &X);} +term(A) ::= ON(X). {A = sql_expr_new(@X, &X);} // set xx=on +expr(A) ::= ID(X). {A = sql_expr_new(@X, &X);} +expr(A) ::= JOIN_KW(X). {A = sql_expr_new(@X, &X);} +expr(A) ::= nm(X) DOT nm(Y). { + sql_expr_t* p1 = sql_expr_new(TK_ID, &X); + sql_expr_t* p2 = sql_expr_new(TK_ID, &Y); + A = sql_expr_new(TK_DOT, 0); + sql_expr_attach_subtrees(A, p1, p2); +} + +expr(A) ::= nm(X) DOT ANY(Y). {// allow x.reserved-keyword + sql_expr_t* p1 = sql_expr_new(TK_ID, &X); + sql_expr_t* p2 = sql_expr_new(@Y, &Y); + A = sql_expr_new(TK_DOT, 0); + sql_expr_attach_subtrees(A, p1, p2); +} + +expr(A) ::= nm(X) DOT nm(Y) DOT nm(Z). { + sql_expr_t* p1 = sql_expr_new(TK_ID, &X); + sql_expr_t* p2 = sql_expr_new(TK_ID, &Y); + sql_expr_t* p3 = sql_expr_new(TK_ID, &Z); + sql_expr_t* p4 = sql_expr_new(TK_DOT, 0); + sql_expr_attach_subtrees(p4, p2, p3); + A = sql_expr_new(TK_DOT, 0); + sql_expr_attach_subtrees(A, p1, p4); +} +term(A) ::= INTEGER|FLOAT|BIN_NUM|HEX_NUM|BLOB(X). {A = sql_expr_new(@X, &X);} +term(A) ::= concat_str(X). {A = sql_expr_new(TK_STRING, &X);} +expr(A) ::= VARIABLE(X). {A = sql_expr_new(@X, &X);} +expr(A) ::= expr(A) COLLATE ID|STRING(X). { + sql_expr_t* coll = sql_expr_new(@X, &X); + sql_expr_attach_subtrees(A, coll, NULL); +} + +%include { + static sql_expr_t* function_expr_new(sql_token_t* name, sql_expr_list_t* args) + { + sql_expr_t* func_expr = sql_expr_new(TK_FUNCTION, name); + func_expr->list = args; + return func_expr; + } +} + +expr(A) ::= ID(X) LP distinct exprlist(Y) RP. { + A = function_expr_new(&X, Y); + context->where_flags |= EP_FUNCTION; + if (strncasecmp(X.z, "last_insert_id", X.n) == 0) { + context->where_flags |= EP_LAST_INSERT_ID; + } + //A->distinct = D; +} +expr(A) ::= ID(X) LP STAR RP. { + A = function_expr_new(&X, 0); +} +expr(A) ::= JOIN_KW(N) LP expr COMMA expr RP. { + A = function_expr_new(&N, 0); +} +expr(A) ::= INSERT(N) LP expr COMMA expr COMMA expr COMMA expr RP. { + A = function_expr_new(&N, 0); +} +expr(A) ::= TRIM(N) LP expr RP. { + A = function_expr_new(&N, 0); +} +expr(A) ::= TRIM(N) LP expr FROM expr RP. { + A = function_expr_new(&N, 0); +} +expr(A) ::= TRIM(N) LP TRIM_SPEC expr FROM expr RP. { + A = function_expr_new(&N, 0); +} +expr(A) ::= TRIM(N) LP TRIM_SPEC FROM expr RP. { + A = function_expr_new(&N, 0); +} +expr(A) ::= POSITION(N) LP STRING IN expr RP. { + A = function_expr_new(&N, 0); +} +expr(A) ::= CURRENT_DATE(N) opt_parentheses. { + A = function_expr_new(&N, 0); + context->clause_flags |= CF_LOCAL_QUERY; +} +expr(A) ::= CETUS_VERSION(N) opt_parentheses. { + A = function_expr_new(&N, 0); + context->clause_flags |= CF_LOCAL_QUERY; +} + +opt_parentheses ::= LP RP. +opt_parentheses ::= . + +expr(A) ::= expr(A) AND expr(Y). {A=spanBinaryExpr(TK_AND, A, Y);} +expr(A) ::= expr(A) OR expr(Y). {A=spanBinaryExpr(TK_OR, A, Y);} +expr(A) ::= expr(A) LT|GT|GE|LE(OP) expr(Y). + {A=spanBinaryExpr(@OP, A, Y);} +expr(A) ::= expr(A) EQ|NE(OP) expr(Y). {A=spanBinaryExpr(@OP, A, Y);} +expr(A) ::= expr(A) BITAND|BITOR|LSHIFT|RSHIFT(OP) expr(Y). + {A=spanBinaryExpr(@OP, A, Y);} +expr(A) ::= expr(A) PLUS|MINUS(OP) expr(Y). + {A=spanBinaryExpr(@OP, A, Y);} +expr(A) ::= expr(A) STAR|SLASH|REM(OP) expr(Y). + {A=spanBinaryExpr(@OP, A, Y);} +expr(A) ::= expr(A) CONCAT(OP) expr(Y). {A=spanBinaryExpr(@OP, A, Y);} + +expr(A) ::= var_scope(S) ID(X). { + A = sql_expr_new(@X, &X); + A->var_scope = S; + if (strcasecmp(X.z, "last_insert_id") == 0) { + context->where_flags |= EP_LAST_INSERT_ID; + } +} + +%type likeop {struct LikeOp} +likeop(A) ::= LIKE_KW|MATCH(X). {A.eOperator = X; A.bNot = 0;/*A-overwrites-X*/} +likeop(A) ::= NOT LIKE_KW|MATCH(X). {A.eOperator = X; A.bNot = 1;} +expr(A) ::= expr(A) likeop(OP) expr(Y). [LIKE_KW] { + sql_expr_list_t *pList = sql_expr_list_append(0, Y); + pList = sql_expr_list_append(pList, A); + A = function_expr_new(&OP.eOperator, pList); + A = exprNot(OP.bNot, A); +} +expr(A) ::= expr(A) likeop(OP) expr(Y) ESCAPE expr(E). [LIKE_KW] { + sql_expr_list_t *pList = sql_expr_list_append(0, Y); + pList = sql_expr_list_append(pList, A); + pList = sql_expr_list_append(pList, E); + A = function_expr_new(&OP.eOperator, pList); + A = exprNot(OP.bNot, A); +} + +expr(A) ::= expr(A) IS expr(Y). { + A=spanBinaryExpr(TK_IS,A,Y); +} +expr(A) ::= expr(A) IS NOT expr(Y). { + A=spanBinaryExpr(TK_ISNOT,A,Y); +} + +%include { + /* Construct an expression node for a unary prefix operator + */ + static sql_expr_t* spanUnaryPrefix( + int op, /* The operator */ + sql_expr_t* pOperand /* The operand */ + ){ + sql_expr_t* the_op = sql_expr_new(op, 0); + sql_expr_attach_subtrees(the_op, pOperand, NULL); + return the_op; + } +} + + +expr(A) ::= NOT expr(X). + {A=spanUnaryPrefix(TK_NOT,X);/*A-overwrites-B*/} +expr(A) ::= BITNOT expr(X). + {A=spanUnaryPrefix(TK_BITNOT,X);/*A-overwrites-B*/} +expr(A) ::= MINUS expr(X). [BITNOT] + {A=spanUnaryPrefix(TK_UMINUS,X);/*A-overwrites-B*/} +expr(A) ::= PLUS expr(X). [BITNOT] + {A=spanUnaryPrefix(TK_UPLUS,X);/*A-overwrites-B*/} + +%type between_op {int} +between_op(A) ::= BETWEEN. {A = 0;} +between_op(A) ::= NOT BETWEEN. {A = 1;} +expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] { + sql_expr_list_t* pList = sql_expr_list_append(0, X); + pList = sql_expr_list_append(pList, Y); + sql_expr_t* betw_op = sql_expr_new(TK_BETWEEN, 0); + if (betw_op) { + sql_expr_attach_subtrees(betw_op, A, NULL); + betw_op->list = pList; + A = betw_op; + } else { + sql_expr_list_free(pList); + } + A = exprNot(N, A); +} + +%type in_op {int} +in_op(A) ::= IN. {A = 0;} +in_op(A) ::= NOT IN. {A = 1;} +expr(A) ::= expr(B) in_op(N) LP exprlist(Y) RP. [IN] { + A = sql_expr_new(TK_IN, 0); + if (A) { + sql_expr_attach_subtrees(A, B, NULL); + A->list = Y; + } else { + sql_expr_list_free(Y); + } + A = exprNot(N, A); +} +expr(A) ::= LP select(X) RP. { + A = sql_expr_new(TK_SELECT, 0); + if (A) { + A->select = X; + } else { + sql_expr_free(X); + }; +} +expr(A) ::= expr(B) in_op(N) LP select(Y) RP. [IN] { + A = sql_expr_new(TK_IN, 0); + if (A) { + sql_expr_attach_subtrees(A, B, NULL); + A->select = Y; + } else { + sql_expr_free(Y); + } + A = exprNot(N, A); +} +expr(A) ::= EXISTS LP select(Y) RP. { + A = sql_expr_new(TK_EXISTS, 0); + A->select = Y; +} + +expr(A) ::= INTERVAL expr TIME_UNIT. { A = 0; } + +/* CASE expressions */ +expr(A) ::= CASE case_operand(X) case_exprlist(Y) case_else(Z) END. { + A = sql_expr_new(TK_CASE, 0); + if (A) { + sql_expr_attach_subtrees(A, X, NULL); + A->list = Z ? sql_expr_list_append(Y, Z) : Y; + } else { + sql_expr_list_free(Y); + sql_expr_free(Z); + } +} +%type case_exprlist {sql_expr_list_t*} +%destructor case_exprlist {sql_expr_list_free($$);} +case_exprlist(A) ::= case_exprlist(A) WHEN expr(Y) THEN expr(Z). { + A = sql_expr_list_append(A, Y); + A = sql_expr_list_append(A, Z); +} +case_exprlist(A) ::= WHEN expr(Y) THEN expr(Z). { + A = sql_expr_list_append(0, Y); + A = sql_expr_list_append(A, Z); +} +%type case_else {sql_expr_t*} +%destructor case_else {sql_expr_free($$);} +case_else(A) ::= ELSE expr(X). {A = X;} +case_else(A) ::= . {A = 0;} +%type case_operand {sql_expr_t*} +%destructor case_operand {sql_expr_free($$);} +case_operand(A) ::= expr(X). {A = X; /*A-overwrites-X*/} +case_operand(A) ::= . {A = 0;} + +%type exprlist {sql_expr_list_t*} +%destructor exprlist {sql_expr_list_free($$);} +%type nexprlist {sql_expr_list_t*} +%destructor nexprlist {sql_expr_list_free($$);} + +exprlist(A) ::= nexprlist(A). +exprlist(A) ::= . {A = 0;} +nexprlist(A) ::= nexprlist(A) COMMA expr(Y). + {A = sql_expr_list_append(A,Y);} +nexprlist(A) ::= expr(Y). + {A = sql_expr_list_append(0,Y); /*A-overwrites-Y*/} + + +// The eidlist non-terminal (sql_expr_tession Id List) generates an sql_expr_list_t +// from a list of identifiers. The identifier names are in sql_expr_list_t.a[].zName. +// This list is stored in an sql_expr_list_t rather than an sql_id_list_t so that it +// can be easily sent to sqlite3Columnssql_expr_list_t(). +// +// eidlist is grouped with CREATE INDEX because it used to be the non-terminal +// used for the arguments to an index. That is just an historical accident. +// +// IMPORTANT COMPATIBILITY NOTE: Some prior versions of SQLite accepted +// COLLATE clauses and ASC or DESC keywords on ID lists in inappropriate +// places - places that might have been stored in the sqlite_master schema. +// Those extra features were ignored. But because they might be in some +// (busted) old databases, we need to continue parsing them when loading +// historical schemas. +// +%type eidlist {sql_expr_list_t*} +%destructor eidlist {sql_expr_list_free($$);} + +%include { + /* Add a single new term to an sql_expr_list_t that is used to store a + ** list of identifiers. Report an error if the ID list contains + ** a COLLATE clause or an ASC or DESC keyword, except ignore the + ** error while parsing a legacy schema. + */ + static sql_expr_list_t *parserAddExprIdListTerm( + sql_expr_list_t *pPrior, + sql_token_t *pIdToken, + int hasCollate, + int sortOrder + ){ + sql_expr_list_t *p = sql_expr_list_append(pPrior, 0); + if (hasCollate || sortOrder != -1) { + printf("syntax error after column name \"%.*s\"", + pIdToken->n, pIdToken->z); + } + return p; + } +} // end %include + +eidlist(A) ::= eidlist(A) COMMA nm(Y) collate(C) sortorder(Z). { + A = parserAddExprIdListTerm(A, &Y, C, Z); +} +eidlist(A) ::= nm(Y) collate(C) sortorder(Z). { + A = parserAddExprIdListTerm(0, &Y, C, Z); /*A-overwrites-Y*/ +} + +%type collate {int} +collate(C) ::= . {C = 0;} +collate(C) ::= COLLATE ids. {C = 1;} + +///////////////////////LOCK TABLES/////////////////////////// +cmd ::= LOCK TABLES lock_tables. +lock_tables ::= fullname as lock_type. +lock_tables ::= lock_tables COMMA fullname as lock_type. +lock_type ::= READ opt_local. +lock_type ::= opt_priority WRITE. +opt_local ::= LOCAL. +opt_local ::= . +opt_priority ::= LOW_PRIORITY. +opt_priority ::= . +cmd ::= UNLOCK TABLES. diff --git a/lib/sql-construction.c b/lib/sql-construction.c new file mode 100644 index 0000000..d36ea39 --- /dev/null +++ b/lib/sql-construction.c @@ -0,0 +1,407 @@ +#include "sql-construction.h" +#include + +#include "myparser.y.h" + +static void sql_expr_traverse(GString *s, sql_expr_t *expr); + +static void string_append_quoted(GString *s, const char *p, char quote) +{ + g_string_append_c(s, quote); + for (; *p != '\0'; ++p) { + if (*p == quote) { /* escape quote inside string */ + g_string_append_c(s, '\\'); + } + g_string_append_c(s, *p); + } + g_string_append_c(s, quote); +} + +/* sql construction */ +static void sql_append_expr(GString *s, sql_expr_t *p) +{ + switch (p->op) { + case TK_ID: + g_string_append(s, " "); + g_string_append(s, p->token_text); + break; + case TK_EQ: g_string_append(s, "="); break; + case TK_LT: g_string_append(s, "<"); break; + case TK_GT: g_string_append(s, ">"); break; + case TK_LE: g_string_append(s, "<="); break; + case TK_GE: g_string_append(s, ">="); break; + case TK_NE: g_string_append(s, "<>"); break; + case TK_AND: g_string_append(s, " AND "); break; + case TK_OR: g_string_append(s, " OR "); break; + case TK_DOT: + g_string_append(s, " "); + g_string_append(s, p->left->token_text); + g_string_append(s, "."); + g_string_append(s, p->right->token_text); + break; + case TK_UPLUS: + case TK_UMINUS: + case TK_INTEGER: { + char valstr[32] = {0}; + char *pstr = valstr; + if (p->op == TK_UMINUS) {*pstr='-'; ++pstr;} + sprintf(pstr, "%" PRIu64, p->num_value); + g_string_append(s, valstr); + break; + } + case TK_STRING: + if (sql_is_quoted_string(p->token_text)) {/* TODO: dequote all */ + g_string_append(s, " "); + g_string_append(s, p->token_text); + } else { + g_string_append_c(s, ' '); + string_append_quoted(s, p->token_text, '\''); + } + break; + case TK_FUNCTION: { + g_string_append(s, " "); + g_string_append(s, p->token_text); + g_string_append(s, "("); + sql_expr_list_t *args = p->list; + if (args) { + int i = 0; + for (i = 0; i < args->len; ++i) { + sql_expr_t *arg = g_ptr_array_index(args, i); + sql_expr_traverse(s, arg); + if (i < args->len-1) { + g_string_append(s, ","); + } + } + } + g_string_append(s, ")"); + break; + } + case TK_BETWEEN: { + sql_append_expr(s, p->left); + g_string_append(s, " BETWEEN "); + sql_expr_list_t *args = p->list; + if (args && args->len == 2) { + sql_expr_t *low = g_ptr_array_index(args, 0); + sql_expr_t *high = g_ptr_array_index(args, 1); + sql_expr_traverse(s, low); + g_string_append(s, " AND "); + sql_expr_traverse(s, high); + } + break; + } + case TK_IN: { + sql_append_expr(s, p->left); + g_string_append(s, " IN ("); + if (p->list) { + sql_expr_list_t *args = p->list; + int i; + for (i = 0; args && i < args->len; ++i) { + sql_expr_t *arg = g_ptr_array_index(args, i); + sql_append_expr(s, arg); + if (i < args->len-1) { + g_string_append_c(s, ','); + } + } + } else if (p->select) { + GString *sel = sql_construct_select(p->select); + if (sel) { + g_string_append(s, sel->str); + g_string_free(sel, TRUE); + } + } + g_string_append(s, ")"); + break; + } + case TK_EXISTS: { + g_string_append(s, " EXISTS ("); + GString *sel = sql_construct_select(p->select); + if (sel) { + g_string_append(s, sel->str); + g_string_free(sel, TRUE); + } + g_string_append(s, ")"); + break; + } + case TK_LIKE_KW: + if (p->list) { + sql_expr_list_t *args = p->list; + if (args->len > 0) { + sql_expr_t *arg = g_ptr_array_index(args, 0); + sql_expr_traverse(s, arg); + } + g_string_append(s, " LIKE "); + if (args->len > 1) { + sql_expr_t *arg = g_ptr_array_index(args, 1); + sql_append_expr(s, arg); + } + if (args->len > 2) { + sql_expr_t *arg = g_ptr_array_index(args, 2); + g_string_append(s, " ESCAPE "); + sql_append_expr(s, arg); + } + } + break; + case TK_NOT: + g_string_append(s, " NOT("); + sql_expr_traverse(s, p->left); + g_string_append_c(s, ')'); + break; + case TK_SELECT: {/* subselect as an expression */ + g_string_append(s, "("); + GString *sel = sql_construct_select(p->select); + if (sel) { + g_string_append(s, sel->str); + g_string_free(sel, TRUE); + } + g_string_append(s, ")"); + break; + } + case TK_IS: g_string_append(s, " IS ");break; + case TK_ISNOT: g_string_append(s, " IS NOT ");break; + case TK_PLUS: g_string_append_c(s, '+');break; + case TK_MINUS: g_string_append_c(s, '-');break; + default: + g_string_append(s, " "); + g_string_append(s, p->token_text); + } +} + +/* these expr will be processed as leaf node */ +static gboolean sql_expr_is_leaf_node(sql_expr_t *expr) +{ + return (expr->op == TK_DOT && expr->left && expr->right) /* db.table */ + || (expr->op == TK_UMINUS && expr->left) /* -3 */ + || (expr->op == TK_UPLUS && expr->left) /* +4 */ + || (expr->op == TK_BETWEEN) + || (expr->op == TK_NOT) + || (expr->op == TK_EXISTS) + || (expr->op == TK_IN); +} + +static void sql_expr_traverse(GString *s, sql_expr_t *expr) +{ + if (!expr) + return; + if (sql_expr_is_leaf_node(expr)) { + sql_append_expr(s, expr); + return; + } + if (expr->left) { + gboolean parenth = (expr->op == TK_AND && expr->left->op == TK_OR); + if (parenth) + g_string_append_c(s, '('); + + sql_expr_traverse(s, expr->left); + if (parenth) + g_string_append_c(s, ')'); + } + + sql_append_expr(s, expr); + + if (expr->right) { + gboolean parenth = (expr->op == TK_AND && expr->right->op == TK_OR); + if (parenth) + g_string_append_c(s, '('); + + sql_expr_traverse(s, expr->right); + if (parenth) + g_string_append_c(s, ')'); + } +} + +static void sql_construct_join(GString *s, int flag) +{ + if (flag == JT_INNER) { + g_string_append(s, " JOIN "); + return; + } + if (flag & JT_INNER) { + g_string_append(s, "INNER "); + } + if (flag & JT_CROSS) { + g_string_append(s, "CROSS "); + } + if (flag & JT_NATURAL) { + g_string_append(s, "NATURAL "); + } + if (flag & JT_LEFT) { + g_string_append(s, "LEFT "); + } + if (flag & JT_RIGHT) { + g_string_append(s, "RIGHT "); + } + if (flag & JT_OUTER) { + g_string_append(s, "OUTER "); + } + g_string_append(s, "JOIN "); +} + +static inline void append_sql_expr(GString *s, sql_expr_t *expr) +{ + g_string_append_len(s, expr->start, expr->end - expr->start); +} + +GString *sql_construct_select(sql_select_t *select) +{ + int i = 0; + GString *s = g_string_new(NULL); + g_string_append(s, "SELECT "); + if (select->columns) { + if (select->flags & SF_DISTINCT) { + g_string_append(s, "DISTINCT "); + } + for (i = 0; i < select->columns->len; ++i) { + sql_expr_t *expr = g_ptr_array_index(select->columns, i); + append_sql_expr(s, expr); + if (expr->alias) { + g_string_append(s, " AS "); + g_string_append(s, expr->alias); + } + if (i != select->columns->len - 1) + g_string_append(s, ","); + } + } + if (select->from_src) { + g_string_append(s, " FROM "); + for (i = 0; i < select->from_src->len; ++i) { + sql_src_item_t *src = g_ptr_array_index(select->from_src, i); + if (src->table_name) { + if (src->dbname) { + g_string_append(s, src->dbname); + g_string_append(s, "."); + } + g_string_append(s, src->table_name); + g_string_append(s, " "); + } else if (src->select) { + GString *sub = sql_construct_select(src->select); + g_string_append_c(s, '('); + g_string_append_len(s, sub->str, sub->len); + g_string_append_c(s, ')'); + g_string_free(sub, TRUE); + } + + if (src->table_alias) { + g_string_append(s, " AS "); + g_string_append(s, src->table_alias); + g_string_append(s, " "); + } + if (src->on_clause) { + g_string_append(s, " ON "); + append_sql_expr(s, src->on_clause); + g_string_append_c(s, ' '); + } + if (src->jointype != 0) { + sql_construct_join(s, src->jointype); + } + } + } + if (select->where_clause) { + g_string_append(s, " WHERE "); + append_sql_expr(s, select->where_clause); + } + if (select->groupby_clause) { + sql_expr_list_t *groupby = select->groupby_clause; + g_string_append(s, " GROUP BY "); + for (i = 0; i < groupby->len; ++i) { + sql_expr_t *expr = g_ptr_array_index(groupby, i); + append_sql_expr(s, expr); + if (i < groupby->len - 1) { + g_string_append(s, ","); + } + } + } + if (select->having_clause) { + g_string_append(s, " HAVING "); + append_sql_expr(s, select->having_clause); + } + if (select->orderby_clause) { + sql_expr_list_t *orderby = select->orderby_clause; + g_string_append(s, " ORDER BY "); + for (i = 0; i < orderby->len; ++i) { + sql_column_t *col = g_ptr_array_index(orderby, i); + + /* this might be duped from column, @see sql_modify_orderby + not using append_sql_expr() to reserve possible alias */ + sql_expr_t *expr = col->expr; + sql_expr_traverse(s, expr); + if (col->sort_order && col->sort_order == SQL_SO_DESC) { + g_string_append(s, " DESC "); + } + if (i < orderby->len - 1) { + g_string_append(s, ","); + } + } + } + + /* don't use append_sql_expr() for LIMIT/OFFSET, the expression has changed */ + if (select->limit) { + g_string_append(s, " LIMIT "); + sql_expr_traverse(s, select->limit); + } + if (select->offset) { + g_string_append(s, " OFFSET "); + sql_expr_traverse(s, select->offset); + } + if (select->lock_read) { + g_string_append(s, " FOR UPDATE"); + } + return s; +} + +/* format as " expr1,expr2,expr3 "*/ +void sql_append_expr_list(GString *s, sql_expr_list_t *exprlist) +{ + int i = 0; + for (i = 0; i < exprlist->len; ++i) { + sql_expr_t *expr = g_ptr_array_index(exprlist, i); + append_sql_expr(s, expr); + if (i != exprlist->len -1) { + g_string_append_c(s, ','); + } + } +} + +void sql_construct_insert(GString *s, sql_insert_t *p) +{ + g_string_append(s, "INSERT INTO "); + if (p->table && p->table->len > 0) { + sql_src_item_t *src = g_ptr_array_index(p->table, 0); + if (src->dbname) { + g_string_append(s, src->dbname); + g_string_append_c(s, '.'); + } + g_string_append(s, src->table_name); + g_string_append_c(s, ' '); + } + if (p->columns && p->columns->len > 0) { + g_string_append_c(s, '('); + int i = 0; + for (i =0; i < p->columns->len; ++i) { + const char *col = g_ptr_array_index(p->columns, i); + g_string_append(s, col); + if (i != p->columns->len - 1) { + g_string_append_c(s, ','); + } + } + g_string_append(s, ") "); + } + if (p->sel_val) { + if (p->sel_val->from_src) { + /* select as values */ + GString *select = sql_construct_select(p->sel_val); + g_string_append(s, select->str); + g_string_free(select, TRUE); + } else { + /* expression values */ + g_string_append(s, "VALUES"); + sql_select_t *values = p->sel_val; + for (;values; values = values->prior) { + sql_expr_list_t *cols = values->columns; + g_string_append_c(s, '('); + sql_append_expr_list(s, cols); + g_string_append(s, "),"); + } + s->str[s->len-1] = ' ';/* no comma at the end */ + } + } +} diff --git a/lib/sql-construction.h b/lib/sql-construction.h new file mode 100644 index 0000000..464b2ed --- /dev/null +++ b/lib/sql-construction.h @@ -0,0 +1,14 @@ +#ifndef _SQL_CONSTRUCTION_ +#define _SQL_CONSTRUCTION_ + +#include "sql-expression.h" + + +GString *sql_construct_expr(sql_expr_t *expr); + +/* TODO: don't alloc inside function */ +GString *sql_construct_select(sql_select_t *); + +void sql_construct_insert(GString *, sql_insert_t *); + +#endif /*_SQL_CONSTRUCTION_*/ diff --git a/lib/sql-context.c b/lib/sql-context.c new file mode 100644 index 0000000..f51ae54 --- /dev/null +++ b/lib/sql-context.c @@ -0,0 +1,224 @@ +#include "sql-context.h" + +#include "mylexer.l.h" +#include "sql-property.h" + +void sqlParser(void *yyp,int yymajor, sql_token_t yyminor, sql_context_t *); +void sqlParserFree(void *p,void(*freeProc)(void *)); +void *sqlParserAlloc(void *(*mallocProc)(size_t)); +void sqlParserTrace(FILE *TraceFILE, char *zTracePrompt); +void yylex_restore_buffer(void *); + +void sql_context_init(sql_context_t *p) +{ + memset(p, 0, sizeof(sql_context_t)); +} + +void sql_context_destroy(sql_context_t *p) +{ + if (p->sql_statement) + sql_statement_free(p->sql_statement, p->stmt_type); + if (p->message) + g_free(p->message); + if (p->property) + sql_property_free(p->property); +} + +void sql_context_reset(sql_context_t *p) +{ + sql_context_destroy(p); + sql_context_init(p); +} + +void sql_context_append_msg(sql_context_t *p, char *msg) +{ + int len = strlen(msg); + int orig_len = 0; + if (p->message == NULL) { + p->message = g_malloc0(len + 1); + } else { + char *orig_msg = p->message; + orig_len = strlen(orig_msg); + p->message = g_malloc0(orig_len + len + 1); + strcpy(p->message, orig_msg); + g_free(orig_msg); + } + strcpy(p->message + orig_len, msg); +} + +void sql_context_set_error(sql_context_t *p, int err, char *msg) +{ + p->rc = err; + sql_context_append_msg(p, msg); +} + +void sql_context_add_stmt(sql_context_t *p, + enum sql_stmt_type_t type, void *clause) +{ + if (p->stmt_count == 0) { + p->stmt_type = type; + p->sql_statement = clause; + } else { + if (clause) { + sql_statement_free(clause, type); + } + } +} + +gboolean sql_context_has_sharding_property(sql_context_t *p) +{ + return p && p->property && (p->property->table || p->property->group); +} + +static void parse_token(sql_context_t *context, int code, sql_token_t token, + void *parser, sql_property_parser_t *prop_parser) +{ + if (code == TK_PROPERTY_START) { + prop_parser->is_parsing = TRUE; + if (context->property == NULL) { + context->property = g_new0(sql_property_t, 1); + } + return; + } else if (code == TK_PROPERTY_END) { + prop_parser->is_parsing = FALSE; + sql_property_t *prop = context->property; + if (prop->mode == MODE_READWRITE) { + context->rw_flag |= CF_FORCE_MASTER; + } else if (prop->mode == MODE_READONLY) { + context->rw_flag |= CF_FORCE_SLAVE; + } + if (!sql_property_is_valid(context->property)) { + sql_property_free(context->property); + context->property = NULL; + g_message(G_STRLOC ":invalid comment"); + sql_context_set_error(context, PARSE_SYNTAX_ERR, "comment error"); + } + return; + } else if (code == TK_MYSQL_HINT) { + context->rw_flag |= CF_WRITE; + return; + } + + if (prop_parser->is_parsing) {/*# K = V */ + int rc = sql_property_parser_parse(prop_parser, + token.z, token.n, context->property); + if (!rc) { + sql_property_free(context->property); + context->property = NULL; + sql_context_set_error(context, PARSE_SYNTAX_ERR, "comment error"); + } + } else { + sqlParser(parser, code, token, context); + } +} + +#define PARSER_TRACE 0 + +/* Parse user allocated sql string + sql->str must be terminated with 2 NUL + sql->len is length including the 2 NUL */ +void sql_context_parse_len(sql_context_t *context, GString *sql) +{ + yyscan_t scanner; + yylex_init(&scanner); + YY_BUFFER_STATE buf_state = yy_scan_buffer(sql->str, sql->len, scanner); +#if PARSER_TRACE + sqlParserTrace(stdout, "---ParserTrace: "); +#endif + + void *parser = sqlParserAlloc(malloc); + sql_context_reset(context); + + static sql_property_parser_t comment_parser; + sql_property_parser_reset(&comment_parser); + + int code; + int last_parsed_token; + sql_token_t token; + while ((code = yylex(scanner)) > 0) { /* 0 on EOF */ + token.z = yyget_text(scanner); + token.n = yyget_leng(scanner); +#if PARSER_TRACE + printf("***LexerTrace: code: %d, yytext: %.*s\n", code, token.n, token.z); + printf("***LexerTrace: yytext addr: %p\n", token.z); +#endif + parse_token(context, code, token, parser, &comment_parser); + + last_parsed_token = code; + + if (context->rc != PARSE_OK) {/* break on PARSE_HEAD, other error */ + yylex_restore_buffer(scanner);/* restore the input string */ + break; + } + } + if (context->rc == PARSE_OK) { + /* grammar require semicolon as ending token */ + if (last_parsed_token != TK_SEMI) { + sqlParser(parser, TK_SEMI, token, context); + } + sqlParser(parser, 0, token, context); + } + sqlParserFree(parser, free); + yy_delete_buffer(buf_state, scanner); + yylex_destroy(scanner); +} + +gboolean sql_context_is_autocommit_on(sql_context_t *context) +{ + if (context->stmt_type == STMT_SET) { + sql_expr_list_t *set_list = context->sql_statement; + if (set_list && set_list->len > 0) { + sql_expr_t *expr = g_ptr_array_index(set_list, 0); + if (expr->op == TK_EQ && sql_expr_is_id(expr->left, "AUTOCOMMIT")) { + gboolean on; + if (sql_expr_is_boolean(expr->right, &on)) { + return on; + } + } + } + } + return FALSE; +} + +gboolean sql_context_is_autocommit_off(sql_context_t *context) +{ + if (context->stmt_type == STMT_SET) { + sql_expr_list_t *set_list = context->sql_statement; + if (set_list && set_list->len > 0) { + sql_expr_t *expr = g_ptr_array_index(set_list, 0); + if (expr->op == TK_EQ && sql_expr_is_id(expr->left, "AUTOCOMMIT")) { + gboolean on; + if (sql_expr_is_boolean(expr->right, &on)) { + return on == FALSE; + } + } + } + } + return FALSE; +} + +gboolean sql_context_is_single_node_trx(sql_context_t *context) +{ + return context && context->property + && context->property->transaction == TRX_SINGLE_NODE; +} + +gboolean sql_context_is_cacheable(sql_context_t *context) +{ + if (context->stmt_type != STMT_SELECT) + return FALSE; + if (context->clause_flags & CF_SUBQUERY) + return FALSE; + sql_select_t *select = context->sql_statement; + if (!select->from_src) + return FALSE; + if (select->lock_read) + return FALSE; + int i = 0; + for (i = 0; i < select->columns->len; ++i) { /* only allow aggreate functions */ + sql_expr_t *col = g_ptr_array_index(select->columns, i); + if (col->flags & EP_FUNCTION && !(col->flags & EP_AGGREGATE)) + return FALSE; + } + return TRUE; +} diff --git a/lib/sql-context.h b/lib/sql-context.h new file mode 100644 index 0000000..5050dfd --- /dev/null +++ b/lib/sql-context.h @@ -0,0 +1,71 @@ +#ifndef SQL_CONTEXT_H +#define SQL_CONTEXT_H + +#include "sql-expression.h" +#include "myparser.y.h" + +enum sql_parse_state_code_t { + PARSE_NOT_SUPPORT = -3, + PARSE_SYNTAX_ERR = -2, + PARSE_ERROR = -1, + PARSE_OK = 0, + PARSE_HEAD, /* only extract head, leave the tail unparsed */ + PARSE_UNRECOGNIZED, /* confused, send to backend for further validation */ +}; + +enum sql_parsing_place_t { + SELECT_OPTION, + SELECT_COLUMN, + SELECT_FROM, + SELECT_WHERE, +}; + +struct sql_property_t; + +typedef struct sql_context_t { + enum sql_parse_state_code_t rc; + char *message; + int explain; + void *user_data; + void *sql_statement;/* opaque statement pointer */ + sql_stmt_type_t stmt_type; + short stmt_count; + + enum sql_clause_flag_t rw_flag; + enum sql_clause_flag_t clause_flags; + enum sql_expr_flags_t where_flags; + enum sql_parsing_place_t parsing_place; + + struct sql_property_t *property; +} sql_context_t; + + +void sql_context_init(sql_context_t *); + +void sql_context_reset(sql_context_t *); + +void sql_context_destroy(sql_context_t *); + +void sql_context_append_msg(sql_context_t *, char *msg); + +void sql_context_set_error(sql_context_t *, int err, char *msg); + +void sql_context_add_stmt(sql_context_t *, + enum sql_stmt_type_t, void *); + +gboolean sql_context_using_property(sql_context_t *); + +gboolean sql_context_has_sharding_property(sql_context_t *p); + +void sql_context_parse_len(sql_context_t *, GString *sql); + +gboolean sql_context_is_autocommit_on(sql_context_t *); + +gboolean sql_context_is_autocommit_off(sql_context_t *); + +gboolean sql_context_is_single_node_trx(sql_context_t *); + +gboolean sql_context_is_cacheable(sql_context_t *); + + +#endif /* SQL_CONTEXT_H */ diff --git a/lib/sql-expression.c b/lib/sql-expression.c new file mode 100644 index 0000000..9a9c881 --- /dev/null +++ b/lib/sql-expression.c @@ -0,0 +1,863 @@ +#include "sql-expression.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "myparser.y.h" + +char *sql_token_dup(sql_token_t token) +{ + if (token.n == 0) + return NULL; + char *s = g_malloc0(token.n + 1); + memcpy(s, token.z, token.n); + sql_string_dequote(s); + return s; +} + +static int64_t sql_token_to_int(sql_token_t token) +{ + /* TODO: HEX */ + int64_t value = 0; + int sign = 1; + const char *c = token.z; + int i = 0; + if (*c == '+' || *c == '-' ) { + if (*c == '-' ) sign = -1; + c++; + i++; + } + while (isdigit(*c) && i++ < token.n) { + value *= 10; + value += (int) (*c - '0'); + c++; + } + return (value *sign); +} + +void sql_string_dequote(char *z) +{ + int quote; + int i, j; + if (z == 0) return; + quote = z[0]; + switch(quote) { + case '\'': break; + case '"': break; + case '`': break; /* For MySQL compatibility */ + case '[': quote = ']'; break; /* For MS SqlServer compatibility */ + default: return; + } + for (i = 1, j = 0; z[i]; i++) { + if (z[i] == quote) { + if (z[i+1] == quote) { /* quote escape */ + z[j++] = quote; + i++; + } else { + z[j++] = 0; + break; + } + } else if (z[i] == '\\') { /* slash escape */ + i++; + z[j++] = z[i]; + } else { + z[j++] = z[i]; + } + } +} + +sql_expr_t *sql_expr_new(int op, const sql_token_t *token) +{ + int extra = 0; + if (token && op != TK_INTEGER) { + extra = token->n + 1; + } + sql_expr_t *expr = g_malloc0(sizeof(sql_expr_t) + extra); + if (expr) { + expr->op = op; + if (token) { + if (extra == 0) { + expr->num_value = sql_token_to_int(*token); + } else { + expr->token_text = (char *)&expr[1]; + assert(token != NULL); + if (token->n) { + strncpy(expr->token_text, token->z, token->n); + if (op == TK_STRING || op == TK_ID) { + sql_string_dequote(expr->token_text); + } + } + } + expr->start = token->z; + expr->end = &token->z[token->n]; + } + expr->height = 1; + expr->var_scope = SCOPE_SESSION; + } + return expr; +} + +/** + * Only duplicate the root node + */ +sql_expr_t *sql_expr_dup(const sql_expr_t *p) +{ + int extra = 0; + if (p->token_text) { + extra = strlen(p->token_text) + 1; + } + int size = sizeof(sql_expr_t) + extra; + sql_expr_t *expr = g_malloc0(size); + if (expr) { + memcpy(expr, p, size); + if (p->op == TK_DOT) { + expr->left = sql_expr_dup(p->left); + expr->right = sql_expr_dup(p->right); + } else { + expr->left = 0; + expr->right = 0; + } + expr->list = 0; + expr->select = 0; + expr->alias = 0; + } + return expr; +} + +static void height_of_expr(sql_expr_t *p, int *height) +{ + if (p) { + if (p->height > *height) { + *height = p->height; + } + } +} + +static void expr_set_height(sql_expr_t *p) +{ + int height = 0; + height_of_expr(p->left, &height); + height_of_expr(p->right, &height); + p->height = height + 1; +} + +void sql_expr_attach_subtrees(sql_expr_t *root, sql_expr_t *left, sql_expr_t *right) +{ + if (root == NULL) { + sql_expr_free(left); + sql_expr_free(right); + } else { + if (left) { + root->left = left; + root->start = left->start; + } + if (right) { + root->right = right; + root->end = right->end; + } + expr_set_height(root); + } +} + +void sql_expr_free(void *p) +{ + if (p) { + sql_expr_t *exp = (sql_expr_t *)p; + if (exp->left) + sql_expr_free(exp->left); + if (exp->right) + sql_expr_free(exp->right); + if (exp->list) + g_ptr_array_free(exp->list, TRUE); + if (exp->select) + sql_select_free(exp->select); + if (exp->alias) + g_free(exp->alias); + g_free(exp); + } +} + +gboolean sql_expr_get_int(sql_expr_t *p, gint64 *value) +{ + gboolean rc = FALSE; + if (p == NULL) + return FALSE; + switch (p->op) { + case TK_INTEGER: + if (value) { + *value = p->num_value; + } + rc = TRUE; + break; + case TK_UMINUS: + rc = sql_expr_get_int(p->left, value); + *value = -(*value); + break; + case TK_UPLUS: + rc = sql_expr_get_int(p->left, value); + break; + default: + rc = FALSE; + } + return rc; +} + +gboolean sql_expr_is_boolean(sql_expr_t *p, gboolean *value) +{ + gint64 int_val = -1; + if (sql_expr_get_int(p, &int_val)) { + *value = (int_val != 0); + return TRUE; + } else if (sql_expr_is_id(p, "off") || sql_expr_is_id(p, "false")) { + *value = FALSE; + return TRUE; + } else if (sql_expr_is_id(p, "on") || sql_expr_is_id(p, "true")) { + *value = TRUE; + return TRUE; + } else { + return FALSE; + } +} + +/** + * @param name NULL or expected token name + * @return + * if name is NULL: + * return whether p is of type TK_ID + * if name not NULL: + * return TRUE only if p is TK_ID and has same name + */ +gboolean sql_expr_is_id(const sql_expr_t *p, const char *name) +{ + const char *id = sql_expr_id(p); + if (name == NULL) { + return id != NULL; + } else { + if (id) { + return strcasecmp(id, name) == 0; + } + return FALSE; + } +} + +gboolean sql_expr_is_function(const sql_expr_t *p, const char *name) +{ + const char *func = NULL; + if (p && p->op == TK_FUNCTION) { + func = p->token_text; + } + if (name == NULL) { + return func != NULL; + } else { + if (func) { + return strcasecmp(func, name) == 0; + } + return FALSE; + } +} + +/** + * prefix and suffix is NULLable + * @return + * if prefix or suffix is NULL, return true if the expression is of type: + * TK_DOT + * |--TK_ID + * |--TK_ID + * + * if prefix or suffix not NULL, return ture only if expr is "preifx.suffix" + */ +gboolean sql_expr_is_dotted_name(const sql_expr_t *p, const char *prefix, const char *suffix) +{ + if (p && p->op == TK_DOT) { + return sql_expr_is_id(p->left, prefix) + && sql_expr_is_id(p->right, suffix); + } + return FALSE; +} + +gboolean sql_expr_is_field_name(const sql_expr_t *p) +{ + if (p) { + if (p->op == TK_ID) { + return TRUE; + } else if (p->op == TK_DOT) { /* qualified name */ + if (p->right->op == TK_DOT) { /* db.table.col */ + return sql_expr_is_id(p->left,0) + && sql_expr_is_dotted_name(p->right,0,0); + } else { /* table.col */ + return sql_expr_is_dotted_name(p,0,0); + } + } + } + return FALSE; +} + +sql_expr_list_t *sql_expr_list_append(sql_expr_list_t *list, sql_expr_t *expr) +{ + if (expr == NULL) + return list; + if (list == NULL) { + list = g_ptr_array_new_with_free_func(sql_expr_free); + } + g_ptr_array_add(list, expr); + return list; +} + +sql_expr_t *sql_expr_list_find(sql_expr_list_t *list, const char *name) +{ + int i = 0; + for (i = 0; i < list->len; ++i) { + sql_expr_t *col = g_ptr_array_index(list, i); + if (sql_expr_is_id(col, name)) { + return col; + } + } + return NULL; +} + +sql_expr_t *sql_expr_list_find_fullname(sql_expr_list_t *list, const sql_expr_t *expr) +{ + if (sql_expr_is_dotted_name(expr, NULL, NULL)) { + const char *prefix = expr->left->token_text; + const char *suffix = expr->right->token_text; + int i; + for (i = 0; i < list->len; ++i) { + sql_expr_t *col = g_ptr_array_index(list, i); + if (sql_expr_is_dotted_name(col, prefix, suffix)) { + return col; + } + } + } + return NULL; +} + +int sql_expr_list_find_aggregate(sql_expr_list_t *list) +{ + int i, index = 0; + enum sql_func_type_t type; + for (i = 0; i < list->len; ++i) { + sql_expr_t *p = g_ptr_array_index(list, i); + if (p->op == TK_FUNCTION) { + type = sql_func_type(p->token_text); + if (type != FT_UNKNOWN) { + return 1; + } + } + } + + return 0; +} + +int sql_expr_list_find_aggregates(sql_expr_list_t *list, GROUP_AGGR *aggr_array) +{ + int i, index = 0; + enum sql_func_type_t type; + for (i = 0; i < list->len; ++i) { + sql_expr_t *p = g_ptr_array_index(list, i); + if (p->op == TK_FUNCTION) { + type = sql_func_type(p->token_text); + if (type != FT_UNKNOWN) { + if (index < MAX_AGGR_FUNS) { + aggr_array[index].pos = i; + aggr_array[index].fun_type = type; + index++; + } + } + } + } + return index; +} + +void sql_expr_list_free(sql_expr_list_t *list) +{ + if (list) + g_ptr_array_free(list, TRUE); +} + +enum sql_func_type_t sql_func_type(const char *s) +{ + if (strncasecmp(s, "count", 5) == 0) return FT_COUNT; + else if (strncasecmp(s, "sum", 3) == 0) return FT_SUM; + else if (strncasecmp(s, "avg", 3) == 0) return FT_AVG; + else if (strncasecmp(s, "max", 3) == 0) return FT_MAX; + else if (strncasecmp(s, "min", 3) == 0) return FT_MIN; + else return FT_UNKNOWN; +} + +sql_column_t *sql_column_new() +{ + return g_new0(struct sql_column_t, 1); +} + +void sql_column_free(void *p) +{ + if (!p) + return; + sql_column_t *col = (sql_column_t *)p; + if (col->expr) + sql_expr_free(col->expr); + if (col->alias) + g_free(col->alias); + if (col->type) + g_free(col->type); + g_free(col); +} + +sql_column_list_t *sql_column_list_append(sql_column_list_t *list, sql_column_t *col) +{ + if (col == NULL) + return list; + if (list == NULL) { + list = g_ptr_array_new_with_free_func(sql_column_free); + } + g_ptr_array_add(list, col); + return list; +} + +void sql_column_list_free(sql_column_list_t *list) +{ + if (list) + g_ptr_array_free(list, TRUE); +} + +sql_select_t *sql_select_new() +{ + sql_select_t *p = g_new0(sql_select_t, 1); + return p; +} + +void sql_select_free(sql_select_t *p) +{ + if (!p) + return; + if (p->columns) /* The fields of the result */ + sql_expr_list_free(p->columns); + if (p->from_src) /* The FROM clause */ + sql_src_list_free(p->from_src); + if (p->where_clause) /* The WHERE clause */ + sql_expr_free(p->where_clause); + if (p->groupby_clause) /* The GROUP BY clause */ + sql_expr_list_free(p->groupby_clause); + if (p->having_clause) /* The HAVING clause */ + sql_expr_free(p->having_clause); + if (p->orderby_clause) /* The ORDER BY clause */ + sql_expr_list_free(p->orderby_clause); + if (p->prior) /* Prior select in a compound select statement */ + sql_select_free(p->prior); + /* sql_select_t *pNext; Next select to the left in a compound */ + if (p->limit) /* LIMIT expression. NULL means not used. */ + sql_expr_free(p->limit); + if (p->offset) /* OFFSET expression. NULL means not used. */ + sql_expr_free(p->offset); + g_free(p); +} + +sql_delete_t *sql_delete_new() +{ + sql_delete_t *p = g_new0(sql_delete_t, 1); + return p; +} + +void sql_delete_free(sql_delete_t *p) +{ + if (!p) + return; + if (p->from_src) /* The FROM clause */ + sql_src_list_free(p->from_src); + if (p->where_clause) /* The WHERE clause */ + sql_expr_free(p->where_clause); + if (p->orderby_clause) + sql_expr_list_free(p->orderby_clause); /* The ORDER BY clause */ + if (p->limit) /* LIMIT expression. NULL means not used. */ + sql_expr_free(p->limit); + if (p->offset) /* OFFSET expression. NULL means not used. */ + sql_expr_free(p->offset); + g_free(p); +} + +sql_update_t *sql_update_new() +{ + sql_update_t *p = g_new0(sql_update_t, 1); + return p; +} + +void sql_update_free(sql_update_t *p) +{ + if (!p) + return; + if (p->table) + sql_src_list_free(p->table); + if (p->set_list) + sql_expr_list_free(p->set_list); + if (p->where_clause) /* The WHERE clause */ + sql_expr_free(p->where_clause); + if (p->orderby_clause) + sql_expr_list_free(p->orderby_clause); /* The ORDER BY clause */ + if (p->limit) /* LIMIT expression. NULL means not used. */ + sql_expr_free(p->limit); + if (p->offset) /* OFFSET expression. NULL means not used. */ + sql_expr_free(p->offset); + g_free(p); +} + +sql_insert_t *sql_insert_new() +{ + sql_insert_t *p = g_new0(sql_insert_t, 1); + return p; +} + + +void sql_insert_free(sql_insert_t *p) +{ + if (!p) + return; + if (p->table) + sql_src_list_free(p->table); + if (p->sel_val) + sql_select_free(p->sel_val); + if (p->columns) + sql_id_list_free(p->columns); + g_free(p); +} + +void sql_src_item_free(void *p) +{ + if (!p) + return; + struct sql_src_item_t *item = (struct sql_src_item_t *)p; + if (item->table_name) + g_free(item->table_name); + if (item->table_alias) + g_free(item->table_alias); + if (item->dbname) + g_free(item->dbname); + if (item->select) + sql_select_free(item->select); + if (item->on_clause) + sql_expr_free(item->on_clause); + if (item->pUsing) + sql_id_list_free(item->pUsing); + if (item->func_arg) + sql_expr_list_free(item->func_arg); + g_free(item); +} + +sql_src_list_t *sql_src_list_append(sql_src_list_t *p, sql_token_t *tname, + sql_token_t *dbname, sql_token_t *alias, sql_select_t *subquery, + sql_expr_t *on_clause, sql_id_list_t *using_clause) +{ + struct sql_src_item_t *item = g_new0(sql_src_item_t, 1); + if (item) { + item->table_name = tname ? sql_token_dup(*tname) : NULL; + item->table_alias = alias ? sql_token_dup(*alias) : NULL; + item->dbname = dbname ? sql_token_dup(*dbname) : NULL; + item->select = subquery; + item->on_clause = on_clause; + item->pUsing = using_clause; + } + if (!p) { + p = g_ptr_array_new_with_free_func(sql_src_item_free); + } + g_ptr_array_add(p, item); + return p; +} + +void sql_src_list_free(sql_src_list_t *p) +{ + if (p) + g_ptr_array_free(p, TRUE); +} + +sql_id_list_t *sql_id_list_append(sql_id_list_t *p, sql_token_t *id_name) +{ + if (!p) { + p = g_ptr_array_new_with_free_func(g_free); + } + if (id_name) + g_ptr_array_add(p, sql_token_dup(*id_name)); + return p; +} + +void sql_id_list_free(sql_id_list_t *p) +{ + if (p) + g_ptr_array_free(p, TRUE); +} + +char *sql_get_token_name(int op) +{ + static struct token_list_s{ + int code; + char *name; + } token_list[] = { + {TK_SEMI ,"TK_SEMI "}, + {TK_CREATE ,"TK_CREATE "}, + {TK_TABLE ,"TK_TABLE "}, + {TK_IF ,"TK_IF "}, + {TK_NOT ,"TK_NOT "}, + {TK_EXISTS ,"TK_EXISTS "}, + {TK_TEMP ,"TK_TEMP "}, + {TK_LP ,"TK_LP "}, + {TK_RP ,"TK_RP "}, + {TK_AS ,"TK_AS "}, + {TK_WITHOUT ,"TK_WITHOUT "}, + {TK_COMMA ,"TK_COMMA "}, + {TK_OR ,"TK_OR "}, + {TK_AND ,"TK_AND "}, + {TK_IS ,"TK_IS "}, + {TK_MATCH ,"TK_MATCH "}, + {TK_LIKE_KW ,"TK_LIKE_KW "}, + {TK_BETWEEN ,"TK_BETWEEN "}, + {TK_IN ,"TK_IN "}, + /* {TK_ISNULL ,"TK_ISNULL "},*/ + /* {TK_NOTNULL ,"TK_NOTNULL "},*/ + {TK_NE ,"TK_NE "}, + {TK_EQ ,"TK_EQ "}, + {TK_GT ,"TK_GT "}, + {TK_LE ,"TK_LE "}, + {TK_LT ,"TK_LT "}, + {TK_GE ,"TK_GE "}, + {TK_ESCAPE ,"TK_ESCAPE "}, + {TK_BITAND ,"TK_BITAND "}, + {TK_BITOR ,"TK_BITOR "}, + {TK_LSHIFT ,"TK_LSHIFT "}, + {TK_RSHIFT ,"TK_RSHIFT "}, + {TK_PLUS ,"TK_PLUS "}, + {TK_MINUS ,"TK_MINUS "}, + {TK_STAR ,"TK_STAR "}, + {TK_SLASH ,"TK_SLASH "}, + {TK_REM ,"TK_REM "}, + {TK_CONCAT ,"TK_CONCAT "}, + {TK_COLLATE ,"TK_COLLATE "}, + {TK_BITNOT ,"TK_BITNOT "}, + {TK_ID ,"TK_ID "}, + {TK_ABORT ,"TK_ABORT "}, + {TK_ACTION ,"TK_ACTION "}, + {TK_AFTER ,"TK_AFTER "}, + {TK_ANALYZE ,"TK_ANALYZE "}, + {TK_ASC ,"TK_ASC "}, + {TK_ATTACH ,"TK_ATTACH "}, + {TK_BEFORE ,"TK_BEFORE "}, + {TK_BEGIN ,"TK_BEGIN "}, + {TK_BY ,"TK_BY "}, + {TK_CASCADE ,"TK_CASCADE "}, + {TK_CAST ,"TK_CAST "}, + {TK_COLUMNKW ,"TK_COLUMNKW "}, + {TK_CONFLICT ,"TK_CONFLICT "}, + {TK_DATABASE ,"TK_DATABASE "}, + {TK_DESC ,"TK_DESC "}, + {TK_DETACH ,"TK_DETACH "}, + {TK_EACH ,"TK_EACH "}, + {TK_END ,"TK_END "}, + {TK_FAIL ,"TK_FAIL "}, + {TK_FOR ,"TK_FOR "}, + {TK_IGNORE ,"TK_IGNORE "}, + {TK_INITIALLY ,"TK_INITIALLY "}, + {TK_INSTEAD ,"TK_INSTEAD "}, + {TK_NO ,"TK_NO "}, + {TK_PLAN ,"TK_PLAN "}, + {TK_QUERY ,"TK_QUERY "}, + {TK_KEY ,"TK_KEY "}, + {TK_OF ,"TK_OF "}, + {TK_TO,"TK_TO"}, + {TK_OFFSET ,"TK_OFFSET "}, + {TK_PRAGMA ,"TK_PRAGMA "}, + {TK_RAISE ,"TK_RAISE "}, + {TK_RECURSIVE ,"TK_RECURSIVE "}, + {TK_RELEASE ,"TK_RELEASE "}, + {TK_REPLACE ,"TK_REPLACE "}, + {TK_RESTRICT ,"TK_RESTRICT "}, + {TK_ROW ,"TK_ROW "}, + {TK_TRANSACTION ,"TK_TRANSACTION"}, + {TK_START ,"TK_START"}, + {TK_COMMIT ,"TK_COMMIT"}, + {TK_ROLLBACK ,"TK_ROLLBACK "}, + {TK_SAVEPOINT ,"TK_SAVEPOINT "}, + {TK_TRIGGER ,"TK_TRIGGER "}, + {TK_VACUUM ,"TK_VACUUM "}, + {TK_VIEW ,"TK_VIEW "}, + {TK_VIRTUAL ,"TK_VIRTUAL "}, + {TK_WITH ,"TK_WITH "}, + {TK_RENAME ,"TK_RENAME "}, + {TK_ANY ,"TK_ANY "}, + {TK_STRING ,"TK_STRING "}, + {TK_JOIN_KW ,"TK_JOIN_KW "}, + {TK_INTEGER ,"TK_INTEGER "}, + {TK_FLOAT ,"TK_FLOAT "}, + {TK_CONSTRAINT ,"TK_CONSTRAINT"}, + {TK_DEFAULT ,"TK_DEFAULT "}, + {TK_CHECK ,"TK_CHECK "}, + {TK_AUTO_INCREMENT ,"TK_AUTO_INCREMENT "}, + {TK_PRIMARY ,"TK_PRIMARY "}, + {TK_UNIQUE ,"TK_UNIQUE "}, + {TK_FOREIGN ,"TK_FOREIGN "}, + {TK_DROP ,"TK_DROP "}, + {TK_SELECT ,"TK_SELECT "}, + {TK_VALUES ,"TK_VALUES "}, + {TK_DISTINCT ,"TK_DISTINCT "}, + {TK_DOT ,"TK_DOT "}, + {TK_FROM ,"TK_FROM "}, + {TK_JOIN ,"TK_JOIN "}, + {TK_ON ,"TK_ON "}, + {TK_USING ,"TK_USING "}, + {TK_ORDER ,"TK_ORDER "}, + {TK_GROUP ,"TK_GROUP "}, + {TK_HAVING ,"TK_HAVING "}, + {TK_LIMIT ,"TK_LIMIT "}, + {TK_DELETE ,"TK_DELETE "}, + {TK_WHERE ,"TK_WHERE "}, + {TK_UPDATE ,"TK_UPDATE "}, + {TK_SET ,"TK_SET "}, + {TK_INTO ,"TK_INTO "}, + {TK_INSERT ,"TK_INSERT "}, + {TK_NULL ,"TK_NULL "}, + {TK_BLOB ,"TK_BLOB "}, + {TK_CASE ,"TK_CASE "}, + {TK_WHEN ,"TK_WHEN "}, + {TK_THEN ,"TK_THEN "}, + {TK_ELSE ,"TK_ELSE "} + }; + int len = sizeof(token_list)/sizeof(struct token_list_s); + int i; + for (i = 0; i < len; ++i) + { + if (token_list[i].code == op) + return token_list[i].name; + } + return "NotFound"; +} + +/* print the syntax tree, used for debug */ +void sql_expr_print(sql_expr_t *p, int depth) +{ + if (p) { + int INDENT = 2; + static const char *spaces = " "; + printf("%.*s", depth *INDENT, spaces); + printf("%s ", sql_get_token_name(p->op)); + if (p->op != TK_INTEGER && p->token_text) + printf("%s\n", p->token_text); + else if (p->op == TK_INTEGER) + printf("%" PRIu64 "\n", p->num_value); + else + printf("\n"); + + if (p->left) + sql_expr_print(p->left, depth+1); + else if (p->right) + printf("%.*s[nul]\n", (depth+1)*INDENT, spaces); + else + ; + if (p->right) + sql_expr_print(p->right, depth+1); + else if (p->left) + printf("%.*s[nul]\n", (depth+1)*INDENT, spaces); + else + ; + } +} + +void sql_statement_free(void *clause, sql_stmt_type_t stmt_type) +{ + if (!clause) + return; + switch(stmt_type) { + case STMT_SELECT: + sql_select_free(clause); + break; + case STMT_UPDATE: + sql_update_free(clause); + break; + case STMT_INSERT: + sql_insert_free(clause); + break; + case STMT_DELETE: + sql_delete_free(clause); + break; + case STMT_SHOW: + break; + case STMT_SHOW_COLUMNS: + case STMT_SHOW_CREATE: + case STMT_EXPLAIN_TABLE: + sql_src_list_free(clause); + break; + case STMT_SET: + sql_expr_list_free(clause); + break; + case STMT_SET_TRANSACTION: + case STMT_SET_NAMES: + case STMT_USE: + case STMT_SAVEPOINT: + g_free(clause); + break; + case STMT_START: + case STMT_COMMIT: + case STMT_ROLLBACK: + case STMT_COMMON_DDL: + break; + case STMT_CALL: + sql_expr_free(clause); + break; + default: + g_warning(G_STRLOC ":not supported clause type, caution mem leak"); + } +} + +gboolean sql_is_quoted_string(const char *s) +{ + if (!s) + return FALSE; + int len = strlen(s); + if (len < 2) + return FALSE; + if (s[0] != s[len-1]) + return FALSE; + return s[0] == '\'' || s[0] == '"' || s[0] == '`'; +} + +int sql_join_type(sql_token_t kw) +{ + static struct { + char *name; + uint8_t code; + } _map[] = { + {"INNER", JT_INNER}, + {"CROSS", JT_CROSS}, + {"NATURAL", JT_NATURAL}, + {"LEFT", JT_LEFT }, + {"RIGHT", JT_RIGHT}, + {"OUTER", JT_OUTER}, + }; + int ret = JT_ERROR; + char *kw_str = sql_token_dup(kw); + int i = 0; + while (i < 6) { + if (strcasecmp(_map[i].name, kw_str) == 0) { + ret = _map[i].code; + break; + } + ++i; + } + g_free(kw_str); + return ret; +} + +/** + * !! support only ID, INTGER and STRING + */ +gboolean sql_expr_equals(const sql_expr_t *p1, const sql_expr_t *p2) +{ + if (p1 && p2 && p1->op == p2->op) { + if (p1->op == TK_ID || p1->op == TK_STRING) { + return strcmp(p1->token_text, p2->token_text) == 0; + } else if (p1->op == TK_INTEGER) { + return p1->num_value == p2->num_value; + } + } + return FALSE; +} diff --git a/lib/sql-expression.h b/lib/sql-expression.h new file mode 100644 index 0000000..753c023 --- /dev/null +++ b/lib/sql-expression.h @@ -0,0 +1,317 @@ +#ifndef SQL_EXPRESSION_H +#define SQL_EXPRESSION_H + +#include +#include +#include +#include + +enum sql_join_type_t { + JT_INNER =0x01, /* Any kind of inner or cross join */ + JT_CROSS =0x02, /* Explicit use of the CROSS keyword */ + JT_NATURAL =0x04, /* True for a "natural" join */ + JT_LEFT =0x08, /* Left outer join */ + JT_RIGHT =0x10, /* Right outer join */ + JT_OUTER =0x20, /* The "OUTER" keyword is present */ + JT_ERROR =0x40, /* unknown or unsupported join type */ +}; + +#define MAX_AGGR_FUNS 8 + +typedef struct GROUP_AGGR { + uint8_t type; + int pos; + unsigned int fun_type; +} GROUP_AGGR; + + +typedef struct sql_token_t sql_token_t; +typedef struct sql_expr_t sql_expr_t; +typedef GPtrArray sql_expr_list_t; +typedef GPtrArray sql_id_list_t; +typedef GPtrArray sql_src_list_t; +typedef struct sql_src_item_t sql_src_item_t; +typedef struct sql_expr_span_t sql_expr_span_t; + +typedef struct sql_select_t sql_select_t; +typedef struct sql_delete_t sql_delete_t; +typedef struct sql_update_t sql_update_t; +typedef struct sql_insert_t sql_insert_t; +typedef enum sql_stmt_type_t sql_stmt_type_t; +typedef struct sql_column_t sql_column_t; +typedef GPtrArray sql_column_list_t; + +enum sql_stmt_type_t { + STMT_UNKOWN, + STMT_SELECT, + STMT_INSERT, + STMT_UPDATE, + STMT_DELETE, + + STMT_COMMON_DDL, + + STMT_SHOW, + STMT_SET, + STMT_SET_NAMES, + STMT_SET_TRANSACTION, + STMT_ROLLBACK, + STMT_COMMIT, + STMT_CALL, + STMT_START, + STMT_EXPLAIN_TABLE, + STMT_USE, + STMT_SAVEPOINT, + STMT_SHOW_COLUMNS, + STMT_SHOW_CREATE, + STMT_SHOW_WARNINGS, +}; +struct sql_token_t { + const char *z; /* pointer to token text, not NUL-terminated */ + uint32_t n; /* length of token text in bytes */ +}; + +enum sql_var_scope_t { + SCOPE_GLOBAL, + SCOPE_SESSION, + SCOPE_USER, + + /* SET TRANSACTION ...without explicit scope keyword, + only affect next transaction, it's transient */ + SCOPE_TRANSIENT, + SCOPE_NONE +}; + +enum sql_trx_feature_t { + TF_READ_ONLY = 1, + TF_READ_WRITE, + + /* isolation levels */ + TF_SERIALIZABLE, + TF_REPEATABLE_READ, + TF_READ_COMMITTED, + TF_READ_UNCOMMITTED, +}; + +enum sql_clause_flag_t { + CF_READ = 0x01, + CF_WRITE = 0x02, + CF_FORCE_MASTER = 0x04, + CF_FORCE_SLAVE = 0x08, + CF_DDL = 0x10, + CF_LOCAL_QUERY = 0x20, + CF_DISTINCT_AGGR = 0x40, + CF_SUBQUERY = 0x80, +}; + +enum sql_sort_order_t { + SQL_SO_ASC, + SQL_SO_DESC, +}; + +enum sql_expr_flags_t { + EP_FUNCTION = 0x01, + EP_INTERVAL = 0x02, + EP_EXISTS = 0x04, + EP_BETWEEN = 0x08, + EP_ATOMIC = 0x10, + EP_LAST_INSERT_ID = 0x20, + EP_DISTINCT = 0x40, + EP_SHARD_COND = 0x80, + EP_JOIN_LINK = 0x0100, + EP_NOT = 0x0200, + EP_CASE_WHEN = 0x0400, + EP_ORDER_BY = 0x0800, + EP_AGGREGATE = 0x1000, +}; + +enum sql_func_type_t { + FT_UNKNOWN = 0, + FT_COUNT, + FT_SUM, + FT_AVG, + FT_MAX, + FT_MIN, +}; + +struct sql_expr_t { + uint16_t op; /* Operation performed by this node */ + char *token_text; /* Token value. Zero terminated and dequoted */ + uint64_t num_value; /* Non-negative integer value */ + sql_expr_t *left; + sql_expr_t *right; + + sql_expr_list_t *list; /* op = IN, EXISTS, SELECT, CASE, FUNCTION, BETWEEN */ + sql_select_t *select; /* EP_xIsSelect and op = IN, EXISTS, SELECT */ + + int height; /* Height of the tree headed by this node */ + char *alias; + enum sql_expr_flags_t flags; + enum sql_var_scope_t var_scope; /* variable scope: SESSION(default) or GLOBAL */ + const char *start; /* first char of expr in original sql */ + const char *end; /* one char past the end of expr in orig sql */ +}; + +enum select_flag_t{ + SF_DISTINCT = 0x01, + SF_ALL = 0x02, + SF_CALC_FOUND_ROWS = 0x04, + SF_MULTI_VALUE = 0x08, + SF_REWRITE_ORDERBY = 0x10, +}; + +struct sql_select_t { + uint8_t op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ + uint32_t flags; /* Various SF_* values */ + sql_expr_list_t *columns; /* The fields of the result */ + sql_src_list_t *from_src; /* The FROM clause */ + sql_expr_t *where_clause; + sql_expr_list_t *groupby_clause; + sql_expr_t *having_clause; + sql_column_list_t *orderby_clause; + sql_select_t *prior; /* Prior select in a compound select statement */ + sql_select_t *next; /* Next select to the left in a compound */ + sql_expr_t *limit; /* LIMIT expression. NULL means not used. */ + sql_expr_t *offset; /* OFFSET expression. NULL means not used. */ + int lock_read; +}; + +struct sql_delete_t { + sql_src_list_t *from_src; /* The FROM clause */ + sql_expr_t *where_clause; + sql_column_list_t *orderby_clause; + sql_expr_t *limit; /* LIMIT expression. NULL means not used. */ + sql_expr_t *offset; /* OFFSET expression. NULL means not used. */ +}; + +struct sql_update_t { + sql_src_list_t *table; + sql_expr_list_t *set_list; + sql_expr_t *where_clause; + sql_column_list_t *orderby_clause; + sql_expr_t *limit; /* LIMIT expression. NULL means not used. */ + sql_expr_t *offset; /* OFFSET expression. NULL means not used. */ + +}; + +struct sql_insert_t { + int is_replace; + sql_src_list_t *table; + sql_select_t *sel_val; /* [1] select... [2] values(...)*/ + sql_id_list_t *columns; +}; + +struct sql_column_t { + sql_expr_t *expr; + char *type; + enum sql_sort_order_t sort_order; + char *alias; +}; + +struct sql_src_item_t { + char *dbname; /* Name of database holding this table */ + char *table_name; /* Name of the table */ + char *table_alias; /* The "B" part of a "A AS B" phrase. zName is the "A" */ + sql_select_t *select; /* A SELECT statement used in place of a table name */ + uint8_t jointype; /* Type of join between this table and the previous */ + + sql_expr_t *on_clause; /* The ON clause of a join */ + sql_id_list_t *pUsing; /* The USING clause of a join */ + + sql_expr_list_t *func_arg; /* Arguments to table-valued-function */ +}; /* One entry for each identifier on the list */ + +typedef struct sql_set_transaction_t { + enum sql_var_scope_t scope; + enum sql_trx_feature_t rw_feature; + enum sql_trx_feature_t level; +} sql_set_transaction_t; + + +char *sql_token_dup(sql_token_t); + +void sql_string_dequote(char *str); + +sql_expr_t *sql_expr_new(int op, const sql_token_t *token); + +sql_expr_t *sql_expr_dup(const sql_expr_t *); + +void sql_expr_attach_subtrees(sql_expr_t *p, sql_expr_t *left, sql_expr_t *right); + +void sql_expr_free(void *p); + +void sql_expr_print(sql_expr_t *p, int depth); + +gboolean sql_expr_get_int(sql_expr_t *p, gint64 *value); + +gboolean sql_expr_is_boolean(sql_expr_t *p, gboolean *value); + +#define sql_expr_id(A) \ + (A ? ((A->op == TK_ID)?A->token_text:NULL) : NULL) + +gboolean sql_expr_is_id(const sql_expr_t *p, const char *name); + +gboolean sql_expr_is_function(const sql_expr_t *p, const char *func_name); + +gboolean sql_expr_is_dotted_name(const sql_expr_t *p, + const char *prefix, const char *suffix); + +gboolean sql_expr_is_field_name(const sql_expr_t *p); + +sql_expr_list_t *sql_expr_list_append(sql_expr_list_t *list, sql_expr_t *expr); + +sql_expr_t *sql_expr_list_find(sql_expr_list_t *list, const char *name); + +sql_expr_t *sql_expr_list_find_fullname(sql_expr_list_t *list, const sql_expr_t *expr); + +int sql_expr_list_find_aggregates(sql_expr_list_t *list, GROUP_AGGR *aggr_array); +int sql_expr_list_find_aggregate(sql_expr_list_t *list); + +void sql_expr_list_free(sql_expr_list_t *list); + +sql_src_list_t *sql_src_list_append(sql_src_list_t *, sql_token_t *tname, + sql_token_t *dbname, sql_token_t *alias, sql_select_t *subquery, + sql_expr_t *on_clause, sql_id_list_t *using_clause); + +void sql_src_list_free(sql_src_list_t *p); + +sql_id_list_t *sql_id_list_append(sql_id_list_t *p, sql_token_t *id_name); + +void sql_id_list_free(sql_id_list_t *p); + +sql_select_t *sql_select_new(); + +void sql_select_free(sql_select_t *); + +sql_delete_t *sql_delete_new(); + +void sql_delete_free(sql_delete_t *); + +sql_update_t *sql_update_new(); + +void sql_update_free(sql_update_t *); + +sql_insert_t *sql_insert_new(); + +void sql_insert_free(sql_insert_t *); + +char *sql_get_token_name(int op); + +sql_column_list_t *sql_column_list_append(sql_column_list_t *, sql_column_t *); + +void sql_column_list_free(sql_column_list_t *); + +sql_column_t *sql_column_new(); + +void sql_column_free(void *); + +int sql_join_type(sql_token_t kw); + +void sql_statement_free(void *clause, sql_stmt_type_t stmt_type); + +gboolean sql_is_quoted_string(const char *s); + +enum sql_func_type_t sql_func_type(const char *s); + +gboolean sql_expr_equals(const sql_expr_t *, const sql_expr_t *); + +#endif /* SQL_EXPRESSION_H */ diff --git a/lib/sql-filter-variables.c b/lib/sql-filter-variables.c new file mode 100644 index 0000000..6ecef47 --- /dev/null +++ b/lib/sql-filter-variables.c @@ -0,0 +1,284 @@ +#include "sql-filter-variables.h" +#include "cetus-util.h" + +#include +#include +#include +#include "cJSON.h" + +enum _value_type_t { + VAL_UNKNOWN = -1, + VAL_INT = 1, + VAL_STRING, + VAL_STRING_CSV, /* comma seperated string value */ +}; + +static enum _value_type_t value_type(const char *str) +{ + if (strcasecmp(str, "int") == 0) return VAL_INT; + else if (strcasecmp(str, "string") == 0) return VAL_STRING; + else if (strcasecmp(str, "string-csv") == 0) return VAL_STRING_CSV; + else return VAL_UNKNOWN; +} + +struct sql_variable_t { + char *name; + enum _value_type_t type; + GList *silent_values; /* GList */ + GList *allowed_values; /* GList */ + gboolean allow_all; + gboolean silence_all; +}; + +static void sql_variable_free(struct sql_variable_t *p) +{ + if (p->name) g_free(p->name); + if (p->silent_values) g_list_free_full(p->silent_values, g_free); + if (p->allowed_values) g_list_free_full(p->allowed_values, g_free); + g_free(p); +} + +static gboolean sql_variable_is_silent_value(struct sql_variable_t *p, const char *value) +{ + GList *l; + for (l = p->silent_values; l; l = l->next) { + if (strcasecmp(l->data, value) == 0) { + return TRUE; + } + } + return FALSE; +} + +static gboolean sql_variable_is_allowed_value(struct sql_variable_t *p, const char *value) +{ + GList *l; + for (l = p->allowed_values; l; l = l->next) { + if (strcasecmp(l->data, value) == 0) { + return TRUE; + } + } + return FALSE; +} + +static GHashTable *cetus_variables = NULL; + +void sql_filter_vars_destroy() +{ + if (cetus_variables) { + g_hash_table_destroy(cetus_variables); + } +} + +gboolean sql_filter_vars_load_rules(char *filename) +{ + char *buffer = NULL; + + if (read_file_to_buffer(filename, &buffer) == FALSE) { + g_free(buffer); + return FALSE; + } + + gboolean rc = sql_filter_vars_load_str_rules(buffer); + g_free(buffer); + + return rc; +} + +gboolean str_case_equal(gconstpointer v1, gconstpointer v2) +{ + if (!v1 || !v2) return FALSE; + return strcasecmp((const char*)v1, (const char*)v2)==0; +} + +guint str_case_hash(gconstpointer v) +{ + char* lower = g_ascii_strdown((const char*)v, -1); + guint hash = g_str_hash(lower); + g_free(lower); + return hash; +} + +gboolean sql_filter_vars_load_str_rules(const char *json_str) +{ + if (!cetus_variables) { + cetus_variables = g_hash_table_new_full(str_case_hash, str_case_equal, + NULL, (GDestroyNotify)sql_variable_free); + } + cJSON *root = cJSON_Parse(json_str); + if (!root) { + g_warning(G_STRLOC ":rule file parse error"); + return FALSE; + } + cJSON *var_node = cJSON_GetObjectItem(root, "variables"); + if (!var_node) { + g_warning("cannot find \"variables\" json node"); + return FALSE; + } + cJSON *cur = var_node->child; + for (; cur; cur = cur->next) { + cJSON *name = cJSON_GetObjectItem(cur, "name"); + cJSON *type = cJSON_GetObjectItem(cur, "type"); + if (!name || !type) { + return FALSE; + } + struct sql_variable_t *var = g_new0(struct sql_variable_t, 1); + var->name = g_strdup(name->valuestring); + var->type = value_type(type->valuestring); + cJSON *silent_array = cJSON_GetObjectItem(cur, "silent_values"); + if (silent_array) { + cJSON *silent = silent_array->child; + for (; silent; silent = silent->next) { + if (strcmp(silent->valuestring, "*") == 0) { + var->silence_all = TRUE; + break; + } + var->silent_values = g_list_append(var->silent_values, + g_strdup(silent->valuestring)); + } + } + cJSON *allowed_array = cJSON_GetObjectItem(cur, "allowed_values"); + if (allowed_array) { + cJSON *allowed = allowed_array->child; + for (; allowed; allowed = allowed->next) { + if (strcmp(allowed->valuestring, "*") == 0) { + var->allow_all = TRUE; + break; + } + var->allowed_values = g_list_append(var->allowed_values, + g_strdup(allowed->valuestring)); + } + } + /* if duplicated, replace and free the old (key & value) */ + g_hash_table_replace(cetus_variables, var->name, var); + } + cJSON_Delete(root); + return TRUE; +} + +gboolean sql_filter_vars_is_silent(const char *name, const char *val) +{ + if (!name) { + return FALSE; + } + struct sql_variable_t *var = g_hash_table_lookup(cetus_variables, name); + if (!var) { + return FALSE; + } + if (var->silence_all) { + return TRUE; + } + if (!val) { + return FALSE; + } + switch (var->type) { + case VAL_STRING: + return sql_variable_is_silent_value(var, val); + case VAL_STRING_CSV: { + gchar **values = g_strsplit_set(val, ", ", -1); + int i = 0; + for (i = 0; values[i] != NULL; ++i) { + if (!sql_variable_is_silent_value(var, values[i])) { + g_strfreev(values); + return FALSE; + } + } + g_strfreev(values); + return TRUE; + } + default: + g_warning(G_STRLOC ":not implemented"); + break; + } + return FALSE; +} + +gboolean sql_filter_vars_is_allowed(const char *name, const char *val) +{ + if (!name) { + return FALSE; + } + struct sql_variable_t *var = g_hash_table_lookup(cetus_variables, name); + if (!var) { + return FALSE; + } + if (var->allow_all) { + return TRUE; + } + if (!val) { + return FALSE; + } + switch (var->type) { + case VAL_STRING: + return sql_variable_is_allowed_value(var, val); + case VAL_STRING_CSV: { + gchar **values = g_strsplit_set(val, ", ", -1); + int i = 0; + for (i = 0; values[i] != NULL; ++i) { + if (!sql_variable_is_allowed_value(var, values[i])) { + g_strfreev(values); + return FALSE; + } + } + g_strfreev(values); + return TRUE; + } + default: + g_warning(G_STRLOC "not implemented"); + break; + } + return FALSE; +} + +void sql_filter_vars_load_default_rules() +{ + static const char *default_var_rule = "{" + " \"variables\": [" + " {" + " \"name\": \"sql_mode\"," + " \"type\": \"string-csv\"," + " \"allowed_values\": [" + " \"STRICT_TRANS_TABLES\"," + " \"NO_AUTO_CREATE_USER\"," + " \"NO_ENGINE_SUBSTITUTION\"" + " ]" + " }," + " {" + " \"name\": \"autocommit\"," + " \"type\": \"string\"," + " \"allowed_values\": [\"*\"]" + " }," + " {" + " \"name\": \"character_set_client\"," + " \"type\": \"string\"," + " \"allowed_values\": [\"latin1\",\"ascii\",\"gb2312\",\"gbk\",\"utf8\",\"utf8mb4\",\"binary\",\"big5\"]" + " }," + " {" + " \"name\": \"character_set_connection\"," + " \"type\": \"string\"," + " \"allowed_values\": [\"latin1\",\"ascii\",\"gb2312\",\"gbk\",\"utf8\",\"utf8mb4\",\"binary\",\"big5\"]" + " }," + " {" + " \"name\": \"character_set_results\"," + " \"type\": \"string\"," + " \"allowed_values\": [\"latin1\",\"ascii\",\"gb2312\",\"gbk\",\"utf8\",\"utf8mb4\",\"binary\",\"big5\",\"NULL\"]" + " }" + " ]" + "}"; + gboolean rc = sql_filter_vars_load_str_rules(default_var_rule); + g_assert(rc); +} + +void sql_filter_vars_shard_load_default_rules() +{ + static const char *default_var_rule = "{" + " \"variables\": [" + " {" + " \"name\": \"autocommit\"," + " \"type\": \"string\"," + " \"allowed_values\": [\"*\"]" + " }" + " ]" + "}"; + gboolean rc = sql_filter_vars_load_str_rules(default_var_rule); + g_assert(rc); +} diff --git a/lib/sql-filter-variables.h b/lib/sql-filter-variables.h new file mode 100644 index 0000000..aab1336 --- /dev/null +++ b/lib/sql-filter-variables.h @@ -0,0 +1,20 @@ +#ifndef _SQL_FILTER_VARIABLES_H_ +#define _SQL_FILTER_VARIABLES_H_ + +#include + +gboolean sql_filter_vars_load_rules(char *filename); + +gboolean sql_filter_vars_load_str_rules(const char *json_str); + +void sql_filter_vars_load_default_rules(); + +void sql_filter_vars_shard_load_default_rules(); + +void sql_filter_vars_destroy(); + +gboolean sql_filter_vars_is_silent(const char *name, const char *val); + +gboolean sql_filter_vars_is_allowed(const char *name, const char *val); + +#endif /*_SQL_FILTER_VARIABLES_H_*/ diff --git a/lib/sql-operation.c b/lib/sql-operation.c new file mode 100644 index 0000000..cac3063 --- /dev/null +++ b/lib/sql-operation.c @@ -0,0 +1,183 @@ +#include "sql-operation.h" + +#include +#include +#include +#include + +#include "sql-expression.h" +#include "sql-filter-variables.h" +#include "myparser.y.h" + +void sql_select(sql_context_t *st, sql_select_t *select) +{ + st->rw_flag |= CF_READ; + sql_context_add_stmt(st, STMT_SELECT, select); +} + +void sql_delete(sql_context_t *st, sql_delete_t *del) +{ + if (st->rc != PARSE_OK) { + g_warning("Delete Parse Error"); + } + st->rw_flag |= CF_WRITE; + sql_context_add_stmt(st, STMT_DELETE, del); +} + +void sql_update(sql_context_t *st, sql_update_t *update) +{ + if (st->rc != PARSE_OK) { + g_warning("Update Parse Error"); + } + st->rw_flag |= CF_WRITE; + sql_context_add_stmt(st, STMT_UPDATE, update); +} + + +void sql_insert(sql_context_t *st, sql_insert_t *insert) +{ + if (st->rc != PARSE_OK) { + g_warning("Insert Parse Error"); + } + st->rw_flag |= CF_WRITE; + sql_context_add_stmt(st, STMT_INSERT, insert); +} + +void sql_start_transaction(sql_context_t *st) +{ + if (st->rc != PARSE_OK) { + g_warning("Start transaction parse Error"); + } + st->rw_flag |= CF_WRITE; + sql_context_add_stmt(st, STMT_START, NULL); +} + +void sql_commit_transaction(sql_context_t *st) +{ + if (st->rc != PARSE_OK) { + g_warning("COMMIT transaction parse Error"); + } + st->rw_flag |= CF_WRITE; + sql_context_add_stmt(st, STMT_COMMIT, NULL); +} + +void sql_rollback_transaction(sql_context_t *st) +{ + if (st->rc != PARSE_OK) { + g_warning("ROLLBACK transaction parse Error"); + } + st->rw_flag |= CF_WRITE; + sql_context_add_stmt(st, STMT_ROLLBACK, NULL); +} + +void sql_savepoint(sql_context_t *st, int tk, char *name) +{ + if (st->rc != PARSE_OK) { + g_warning("SAVE POINT parse Error"); + } + st->rw_flag |= CF_WRITE; + sql_context_add_stmt(st, STMT_SAVEPOINT, name); +} + +static gboolean string_array_contains(const char **sa, int size, const char *str) +{ + int i = 0; + for (;i < size; ++i) { + if (strcasecmp(sa[i], str) == 0) + return TRUE; + } + return FALSE; +} + +void sql_set_variable(sql_context_t *ps, sql_expr_list_t *exps) +{ + if (ps->property) { + sql_context_set_error(ps, PARSE_NOT_SUPPORT, + "Commanding comment is not allowed in SET clause"); + goto out; + } + g_assert(exps); + + int i = 0; + for (i = 0; i < exps->len; ++i) { + sql_expr_t *p = g_ptr_array_index(exps, i); + if (!p || p->op != TK_EQ || !sql_expr_is_id(p->left, NULL)) { + sql_context_set_error(ps, PARSE_SYNTAX_ERR, + "syntax error in SET"); + goto out; + } + if (p->left && p->left->var_scope == SCOPE_GLOBAL) { + sql_context_set_error(ps, PARSE_NOT_SUPPORT, + "Only session scope SET is supported now"); + goto out; + } + const char *var_name = p->left->token_text; + const char *value = p->right->token_text; + if (strcasecmp(var_name, "sql_mode") == 0) { + gboolean supported = sql_filter_vars_is_allowed(var_name, value); + if (!supported) { + sql_context_set_error(ps, PARSE_NOT_SUPPORT, + "This sql_mode is not supported"); + goto out; + } + } else if (!sql_filter_vars_is_allowed(var_name, value)) { + char msg[100]; + snprintf(msg, 100, "SET of %s is not supported", var_name); + sql_context_set_error(ps, PARSE_NOT_SUPPORT, msg); + goto out; + } + } +out: + sql_context_add_stmt(ps, STMT_SET, exps); +} + +void sql_set_names(sql_context_t *ps, char *val) +{ + if (ps->property) { + sql_context_set_error(ps, PARSE_NOT_SUPPORT, + "Commanding comment is not allowed in SET clause"); + g_free(val); + return; + } + const char *charsets[] = {"latin1", "ascii", "gb2312", "gbk", "utf8", "utf8mb4", "big5"}; + int cs_size = sizeof(charsets) / sizeof(char *); + if (!string_array_contains(charsets, cs_size, val)) { + char msg[64] = {0}; + snprintf(msg, 64, "Unknown character set: %s", val); + sql_context_set_error(ps, PARSE_NOT_SUPPORT, msg); + g_free(val); + return; + } + sql_context_add_stmt(ps, STMT_SET_NAMES, val); +} + +void sql_set_transaction(sql_context_t *ps, int scope, int rw_feature, int level) +{ + if (ps->property) { + sql_context_set_error(ps, PARSE_NOT_SUPPORT, + "Commanding comment is not allowed in SET clause"); + return; + } + if (scope == SCOPE_GLOBAL) { + sql_context_set_error(ps, PARSE_NOT_SUPPORT, + "GLOBAL scope SET TRANSACTION is not supported now"); + return; + } + sql_set_transaction_t *set_tran = g_new0(sql_set_transaction_t, 1); + set_tran->scope = scope; + set_tran->rw_feature = rw_feature; + set_tran->level = level; + sql_context_add_stmt(ps, STMT_SET_TRANSACTION, set_tran); +} + +void sql_use_database(sql_context_t *ps, char *val) +{ + ps->rw_flag |= CF_READ; + sql_context_add_stmt(ps, STMT_USE, val); +} + +void sql_explain_table(sql_context_t *ps, sql_src_list_t *table) +{ + ps->rw_flag |= CF_READ; + sql_context_add_stmt(ps, STMT_EXPLAIN_TABLE, table); +} diff --git a/lib/sql-operation.h b/lib/sql-operation.h new file mode 100644 index 0000000..368706f --- /dev/null +++ b/lib/sql-operation.h @@ -0,0 +1,35 @@ +#ifndef SQL_OPERATION_H +#define SQL_OPERATION_H + +#include "sql-context.h" +#include "sql-expression.h" + +void sql_set_variable(sql_context_t *, sql_expr_list_t *); + +void sql_set_names(sql_context_t *, char *); + +void sql_set_transaction(sql_context_t *, int scope, int rwfeature, int level); + +void sql_select(sql_context_t *st, sql_select_t *select); + +void sql_delete(sql_context_t *st, sql_delete_t *del); + +void sql_update(sql_context_t *st, sql_update_t *p); + +void sql_insert(sql_context_t *st, sql_insert_t *p); + +void sql_statement_free(void *stmt, sql_stmt_type_t stmt_type); + +void sql_start_transaction(sql_context_t *st); + +void sql_commit_transaction(sql_context_t *st); + +void sql_rollback_transaction(sql_context_t *st); + +void sql_savepoint(sql_context_t *st, int, char *); + +void sql_explain_table(sql_context_t *st, sql_src_list_t *table); + +void sql_use_database(sql_context_t *ps, char *val); + +#endif /* SQL_OPERATION_H */ diff --git a/lib/sql-property.c b/lib/sql-property.c new file mode 100644 index 0000000..07ef2f2 --- /dev/null +++ b/lib/sql-property.c @@ -0,0 +1,147 @@ +#include "sql-property.h" + +#include + +enum property_parse_state_t{ + // PARSE_STATE_INIT, + PARSE_STATE_KEY = 0, + PARSE_STATE_VALUE, + PARSE_STATE_EQ_SIGN, + PARSE_STATE_ERROR, +}; + +enum { + TYPE_INT, + TYPE_STRING, +}; + +#define MAX_VALUE_LEN 50 + +gboolean sql_property_is_valid(sql_property_t *p) +{ + if (p->table && p->group)/* mutual exclusive */ + return FALSE; + return TRUE; +} + +void sql_property_free(sql_property_t *p) +{ + if (p->group) + g_free(p->group); + if (p->table) + g_free(p->table); + if (p->key) + g_free(p->key); + g_free(p); +} + +void sql_property_parser_reset(sql_property_parser_t *parser) +{ + memset(parser, 0, sizeof(*parser)); +} + +static int string_to_code(const char *str) +{ + struct code_map_t { + char *name; + int code; + } map[] = { + {"READWRITE", MODE_READWRITE}, + {"READONLY", MODE_READONLY}, + {"SCOPE_LOCAL", P_SCOPE_LOCAL}, + {"SCOPE_GLOBAL", P_SCOPE_GLOBAL}, + {"SINGLE_NODE", TRX_SINGLE_NODE}, + }; + int i; + for (i = 0; i < sizeof(map) / sizeof(*map); ++i) { + if (strcasecmp(map[i].name, str) == 0) + return map[i].code; + } + return ERROR_VALUE; +} + +static gboolean parser_find_key(sql_property_parser_t *parser, + const char *token, int len) +{ + static const struct property_desc_t { + const char *name; + size_t offset; + int type; + value_parse_func get_value; + } desc[] = { + {"mode", offsetof(struct sql_property_t, mode), TYPE_INT, string_to_code }, + {"scope", offsetof(struct sql_property_t, scope), TYPE_INT, string_to_code }, + {"transaction", offsetof(struct sql_property_t, transaction), TYPE_INT, string_to_code }, + {"group", offsetof(struct sql_property_t, group), TYPE_STRING, NULL }, + {"table", offsetof(struct sql_property_t, table), TYPE_STRING, NULL }, + {"key", offsetof(struct sql_property_t, key), TYPE_STRING, NULL }, + }; + int i = 0; + for (i = 0; i < sizeof(desc)/sizeof(*desc); ++i) { + if (strcasecmp(token, desc[i].name) == 0) { + parser->key_offset = desc[i].offset; + parser->key_type = desc[i].type; + parser->get_value = desc[i].get_value; + parser->state = PARSE_STATE_EQ_SIGN; + return TRUE; + } + } + parser->state = PARSE_STATE_ERROR; + return FALSE; +} + +static gboolean parser_set_value(sql_property_parser_t *parser, + void *object, const char *token, int len) +{ + if (parser->key_type == TYPE_INT) { + int *p = (int *)((unsigned char *) object + parser->key_offset); + if (token[0] == '"' && token[len-1] == '"') {/* might be quoted */ + char buf[MAX_VALUE_LEN] = {0}; + memcpy(buf, token+1, len-2); + *p = parser->get_value(buf); + } else { + *p = parser->get_value(token); + } + parser->state = PARSE_STATE_KEY; + return TRUE; + } else if (parser->key_type == TYPE_STRING) { + char **p = (char **)((unsigned char *) object + parser->key_offset); + if (token[0] == '"' && token[len-1] == '"') {/* might be quoted */ + *p = g_malloc0(len-1); + memcpy(*p, token+1, len-2); + } else { + *p = strndup(token, len); + } + parser->state = PARSE_STATE_KEY; + return TRUE; + } + parser->state = PARSE_STATE_ERROR; + return FALSE; +} + +gboolean sql_property_parser_parse(sql_property_parser_t *parser, + const char *token, int len, sql_property_t *property) +{ + switch (parser->state) { + + case PARSE_STATE_KEY: + return parser_find_key(parser, token, len); + + case PARSE_STATE_EQ_SIGN: + if (len == 1 && token[0] == '=') { + parser->state = PARSE_STATE_VALUE; + return TRUE; + } else { + return FALSE; + } + + case PARSE_STATE_VALUE: + if (len > MAX_VALUE_LEN) { + return FALSE; + } + return parser_set_value(parser, property, token, len); + + default: + return FALSE; + } +} diff --git a/lib/sql-property.h b/lib/sql-property.h new file mode 100644 index 0000000..0bbb1cf --- /dev/null +++ b/lib/sql-property.h @@ -0,0 +1,46 @@ +#ifndef SQL_PROPERTY_H +#define SQL_PROPERTY_H + +#include + +#define MAX_PROPERTY_LEN 64 + +enum { + ERROR_VALUE = -1, + MODE_READWRITE = 1, + MODE_READONLY, + P_SCOPE_LOCAL, + P_SCOPE_GLOBAL, + TRX_SINGLE_NODE, +}; + +typedef struct sql_property_t { + int mode; + int scope; + int transaction; + char *group; + char *table; + char *key; +} sql_property_t; + +void sql_property_free(sql_property_t *); +gboolean sql_property_is_valid(sql_property_t *); + +typedef int(*value_parse_func)(const char *); + +typedef struct sql_property_parser_t { + int key_offset; + value_parse_func get_value; + int key_type; + int state; + + gboolean is_parsing; + +} sql_property_parser_t; + +void sql_property_parser_reset(sql_property_parser_t *parser); + +gboolean sql_property_parser_parse(sql_property_parser_t *parser, + const char *token, int len, sql_property_t *property); + +#endif /* SQL_PROPERTY_H */ diff --git a/mysql-chassis.pc.cmake b/mysql-chassis.pc.cmake new file mode 100644 index 0000000..b20b0dd --- /dev/null +++ b/mysql-chassis.pc.cmake @@ -0,0 +1,12 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=@CMAKE_INSTALL_PREFIX@ +libdir=@CMAKE_INSTALL_PREFIX@/lib +includedir=@CMAKE_INSTALL_PREFIX@/include + +Name: mysql-chassis +Version: @PACKAGE_VERSION_STRING@ +Description: the Chassis of the MySQL Proxy +URL: http://forge.mysql.com/wiki/MySQL_Proxy +Requires: glib-2.0 >= 2.16 +Libs: -L${libdir} -lmysql-chassis -lmysql-chassis-timing -lmysql-chassis-glibext +Cflags: -I${includedir} diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt new file mode 100644 index 0000000..e22ac48 --- /dev/null +++ b/plugins/CMakeLists.txt @@ -0,0 +1,24 @@ +# $%BEGINLICENSE%$ +# Copyright (c) 2009, Oracle and/or its affiliates. All rights reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2 of the +# License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA +# 02110-1301 USA +# +# $%ENDLICENSE%$ +ADD_SUBDIRECTORY(proxy) +ADD_SUBDIRECTORY(admin) +ADD_SUBDIRECTORY(shard) +## needs readline +# ADD_SUBDIRECTORY(cli) diff --git a/plugins/admin/CMakeLists.txt b/plugins/admin/CMakeLists.txt new file mode 100644 index 0000000..c6f53b8 --- /dev/null +++ b/plugins/admin/CMakeLists.txt @@ -0,0 +1,36 @@ +# $%BEGINLICENSE%$ +# Copyright (c) 2009, Oracle and/or its affiliates. All rights reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2 of the +# License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA +# 02110-1301 USA +# +# $%ENDLICENSE%$ +INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/src/) +INCLUDE_DIRECTORIES(${PROJECT_BINARY_DIR}) # for config.h +INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}) + +INCLUDE_DIRECTORIES(${GLIB_INCLUDE_DIRS}) +INCLUDE_DIRECTORIES(${MYSQL_INCLUDE_DIRS}) +INCLUDE_DIRECTORIES(${EVENT_INCLUDE_DIRS}) + +LINK_DIRECTORIES(${GLIB_LIBRARY_DIRS}) +LINK_DIRECTORIES(${LIBINTL_LIBRARY_DIRS}) +LINK_DIRECTORIES(${MYSQL_LIBRARY_DIRS}) + +SET(_plugin_name admin) +ADD_LIBRARY(${_plugin_name} SHARED "${_plugin_name}-plugin.c") +TARGET_LINK_LIBRARIES(${_plugin_name} mysql-chassis-proxy) +CHASSIS_PLUGIN_INSTALL(${_plugin_name}) + diff --git a/plugins/admin/admin-plugin.c b/plugins/admin/admin-plugin.c new file mode 100644 index 0000000..1bc3fb3 --- /dev/null +++ b/plugins/admin/admin-plugin.c @@ -0,0 +1,2531 @@ +#include +#include +#include +#include +#include +#include + +#include "cetus-users.h" +#include "cetus-util.h" +#include "cetus-variable.h" +#include "character-set.h" +#include "chassis-event.h" +#include "chassis-options.h" +#include "cetus-monitor.h" +#include "glib-ext.h" +#include "network-mysqld-packet.h" +#include "network-mysqld-proto.h" +#include "network-mysqld.h" +#include "server-session.h" +#include "sys-pedantic.h" + +#ifndef PLUGIN_VERSION +#ifdef CHASSIS_BUILD_TAG +#define PLUGIN_VERSION CHASSIS_BUILD_TAG +#else +#define PLUGIN_VERSION PACKAGE_VERSION +#endif +#endif + +struct chassis_plugin_config { + gchar *address; /**< listening address of the admin interface */ + + gchar *admin_username; /**< login username */ + gchar *admin_password; /**< login password */ + + gchar *allow_ip; /**< allow ip addr list */ + GHashTable *allow_ip_table; + + gchar *deny_ip; /**< deny ip addr list */ + GHashTable *deny_ip_table; + + gboolean has_shard_plugin; /**< another plugin name is shard or proxy, TRUE is shard, FALSE is proxy */ + + network_mysqld_con *listen_con; +}; + +static struct event* g_sampling_timer = NULL; + +/* + * tokenize input, alloc and return nth token + * n -> [0,..) + */ +static char *str_nth_token(const char *input, int n) +{ + char *t = NULL; + char **tokens = g_strsplit(input, " ", -1); + if (g_strv_length(tokens) > n) { + t = g_strdup(tokens[n]); + } + g_strfreev(tokens); + return t; +} + +/* get config->has_shard_plugin */ +static gboolean has_shard_plugin(GPtrArray *modules) +{ + int i; + for (i = 0; i < modules->len; i++) { + chassis_plugin *plugin = modules->pdata[i]; + if (strcmp(plugin->name, "shard") == 0) { + return TRUE; + } + } + return FALSE; +} + +NETWORK_MYSQLD_PLUGIN_PROTO(server_con_init) { + network_mysqld_auth_challenge *challenge; + GString *packet; + + challenge = network_mysqld_auth_challenge_new(); + challenge->server_version_str = g_strdup_printf("%s admin", PACKAGE_STRING); + challenge->server_version = 50099; + challenge->charset = charset_get_number("latin1"); + challenge->capabilities = CETUS_DEFAULT_FLAGS; + challenge->server_status = SERVER_STATUS_AUTOCOMMIT; + challenge->thread_id = 1; + + /* generate a random challenge */ + network_mysqld_auth_challenge_set_challenge(challenge); + + packet = g_string_new(NULL); + network_mysqld_proto_append_auth_challenge(packet, challenge); + con->client->challenge = challenge; + + network_mysqld_queue_append(con->client, con->client->send_queue, S(packet)); + + g_string_free(packet, TRUE); + + con->state = ST_SEND_HANDSHAKE; + + g_assert(con->plugin_con_state == NULL); + + return NETWORK_SOCKET_SUCCESS; +} + +NETWORK_MYSQLD_PLUGIN_PROTO(server_read_auth) { + network_packet packet; + network_socket *recv_sock, *send_sock; + network_mysqld_auth_response *auth; + GString *excepted_response; + GString *hashed_pwd; + + recv_sock = con->client; + send_sock = con->client; + + packet.data = g_queue_peek_head(recv_sock->recv_queue->chunks); + packet.offset = 0; + + /* decode the packet */ + network_mysqld_proto_skip_network_header(&packet); + + auth = network_mysqld_auth_response_new(con->client->challenge->capabilities); + if (network_mysqld_proto_get_auth_response(&packet, auth)) { + network_mysqld_auth_response_free(auth); + return NETWORK_SOCKET_ERROR; + } + if (!(auth->client_capabilities & CLIENT_PROTOCOL_41)) { + /* should use packet-id 0 */ + network_mysqld_queue_append(con->client, con->client->send_queue, + C("\xff\xd7\x07" "4.0 protocol is not supported")); + network_mysqld_auth_response_free(auth); + return NETWORK_SOCKET_ERROR; + } + + con->client->response = auth; + + /* Check client addr in admin-allow-ip and admin-deny-ip */ + gboolean check_ip; + char *ip_err_msg = NULL; + if (con->config->allow_ip_table || con->config->deny_ip_table) { + char *client_addr = con->client->src->name->str; + char **client_addr_arr = g_strsplit(client_addr, ":", -1); + char *client_ip = client_addr_arr[0]; + if (g_hash_table_size(con->config->allow_ip_table) != 0 && + (g_hash_table_lookup(con->config->allow_ip_table, client_ip) || g_hash_table_lookup(con->config->allow_ip_table, "*"))) { + check_ip = FALSE; + } else if (g_hash_table_size(con->config->deny_ip_table) != 0 && + (g_hash_table_lookup(con->config->deny_ip_table, client_ip) || g_hash_table_lookup(con->config->deny_ip_table, "*"))) { + check_ip = TRUE; + ip_err_msg = g_strdup_printf("Access denied for user '%s'@'%s'", + con->config->admin_username, client_ip); + } else { + check_ip = FALSE; + } + g_strfreev(client_addr_arr); + } else { + check_ip = FALSE; + } + + if (check_ip) { + network_mysqld_con_send_error_full(send_sock, L(ip_err_msg), 1045, "28000"); + g_free(ip_err_msg); + con->state = ST_SEND_ERROR; + } else { + /* check if the password matches */ + excepted_response = g_string_new(NULL); + hashed_pwd = g_string_new(NULL); + + if (!strleq(S(con->client->response->username), + con->config->admin_username, + strlen(con->config->admin_username))) + { + network_mysqld_con_send_error_full(send_sock, + C("unknown user"), 1045, "28000"); + + /* close the connection after we have sent this packet */ + con->state = ST_SEND_ERROR; + } else if (network_mysqld_proto_password_hash(hashed_pwd, + con->config->admin_password, + strlen(con->config->admin_password))) + { + } else if (network_mysqld_proto_password_scramble(excepted_response, + S(recv_sock->challenge->auth_plugin_data), + S(hashed_pwd))) { + network_mysqld_con_send_error_full(send_sock, + C("scrambling failed"), 1045, "28000"); + + /* close the connection after we have sent this packet */ + con->state = ST_SEND_ERROR; + } else if (!g_string_equal(excepted_response, auth->auth_plugin_data)) { + network_mysqld_con_send_error_full(send_sock, + C("password doesn't match"), 1045, "28000"); + + /* close the connection after we have sent this packet */ + con->state = ST_SEND_ERROR; + } else { + network_mysqld_con_send_ok(send_sock); + + con->state = ST_SEND_AUTH_RESULT; + } + + g_string_free(hashed_pwd, TRUE); + g_string_free(excepted_response, TRUE); + } + + g_string_free(g_queue_pop_tail(recv_sock->recv_queue->chunks), TRUE); + + if (recv_sock->recv_queue->chunks->length > 0) { + g_warning("%s: client-recv-queue-len = %d", + G_STRLOC, recv_sock->recv_queue->chunks->length); + } + + return NETWORK_SOCKET_SUCCESS; +} + +static const char *get_conn_xa_state_name(network_mysqld_con_dist_tran_state_t state) { + switch (state) { + case NEXT_ST_XA_START: return "XS"; + case NEXT_ST_XA_QUERY: return "XQ"; + case NEXT_ST_XA_END: return "XE"; + case NEXT_ST_XA_PREPARE: return "XP"; + case NEXT_ST_XA_COMMIT: return "XC"; + case NEXT_ST_XA_ROLLBACK: return "XR"; + case NEXT_ST_XA_CANDIDATE_OVER: return "XCO"; + case NEXT_ST_XA_OVER: return "XO"; + default: + break; + } + + return "NX"; +} + +static char *states[] = { + "unknown", + "up", + "down", + "maintaining", + "deleted", +}; + +static char *types[] = { + "unknown", + "rw", + "ro", +}; + + +static int admin_send_backends_info(network_mysqld_con *admin_con, const char *sql) +{ + chassis *chas = admin_con->srv; + chassis_private *priv = chas->priv; + + chassis_plugin_config *config = admin_con->config; + + GPtrArray *fields = g_ptr_array_new_with_free_func( + (GDestroyNotify)network_mysqld_proto_fielddef_free); + + MYSQL_FIELD *field; + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("backend_ndx"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("address"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("state"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("type"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("slave delay"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("uuid"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("idle_conns"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("used_conns"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("total_conns"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + if (config->has_shard_plugin) { + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("group"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + } + + GPtrArray *rows = g_ptr_array_new_with_free_func( + (GDestroyNotify)network_mysqld_mysql_field_row_free); + + network_backends_t *bs = priv->backends; + int len = bs->backends->len; + int i; + char buffer[32]; + for (i = 0; i < len; i++) { + network_backend_t *backend = bs->backends->pdata[i]; + GPtrArray *row = g_ptr_array_new_with_free_func(g_free); + + snprintf(buffer, sizeof(buffer), "%d", i+1); + + g_ptr_array_add(row, g_strdup(buffer)); + + g_ptr_array_add(row, g_strdup(backend->addr->name->str)); + g_ptr_array_add(row, g_strdup(states[(int) (backend->state)])); + g_ptr_array_add(row, g_strdup(types[(int) (backend->type)])); + + snprintf(buffer, sizeof(buffer), "%d", backend->slave_delay_msec); + g_ptr_array_add(row, backend->type == BACKEND_TYPE_RO ? g_strdup(buffer) : NULL); + + g_ptr_array_add(row, backend->uuid->len ? g_strdup(backend->uuid->str) : NULL); + + snprintf(buffer, sizeof(buffer), "%d", backend->pool->cur_idle_connections); + g_ptr_array_add(row, g_strdup(buffer)); + + snprintf(buffer, sizeof(buffer), "%d", backend->connected_clients); + g_ptr_array_add(row, g_strdup(buffer)); + + snprintf(buffer, sizeof(buffer), "%d", + backend->pool->cur_idle_connections + backend->connected_clients); + g_ptr_array_add(row, g_strdup(buffer)); + + g_ptr_array_add(row, backend->server_group->len ? + g_strdup(backend->server_group->str) : NULL); + + g_ptr_array_add(rows, row); + + } + + network_mysqld_con_send_resultset(admin_con->client, fields, rows); + + /* Free data */ + g_ptr_array_free(rows, TRUE); + g_ptr_array_free(fields, TRUE); + return PROXY_SEND_RESULT; +} + +static void g_table_free_all(gpointer q) { + GHashTable *table = q; + g_hash_table_destroy(table); +} + +typedef struct used_conns { + int num; +} used_conns_t; + +static int admin_send_backend_detail_info(network_mysqld_con *admin_con, const char *sql) +{ + chassis *chas = admin_con->srv; + chassis_private *priv = chas->priv; + + int i, j, len; + + char buffer[32]; + GPtrArray *fields; + GPtrArray *rows; + GPtrArray *row; + MYSQL_FIELD *field; + + GHashTable *back_user_conn_hash_table = g_hash_table_new_full(g_str_hash, + g_str_equal, g_free, g_table_free_all); + + network_backends_t *bs = priv->backends; + len = bs->backends->len; + + for (i = 0; i < len; i++) { + network_backend_t *backend = bs->backends->pdata[i]; + GHashTable *table = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free); + g_hash_table_insert(back_user_conn_hash_table, g_strdup(backend->addr->name->str), table); + } + + len = priv->cons->len; + + for (i = 0; i < len; i++) { + network_mysqld_con *con = priv->cons->pdata[i]; + + if (!con->client || !con->client->response) { + continue; + } + + if (chas->sharding_mode) { + if (con->servers == NULL) { + continue; + } + + for (j = 0; j < con->servers->len; j++) { + server_session_t *pmd = g_ptr_array_index(con->servers, j); + + GHashTable *table = g_hash_table_lookup(back_user_conn_hash_table, + pmd->backend->addr->name->str); + if (table == NULL) { + g_warning("%s: table is null for backend:%s", G_STRLOC, + pmd->backend->addr->name->str); + continue; + } + + used_conns_t *total_used = g_hash_table_lookup(table, + con->client->response->username->str); + if (total_used == NULL) { + total_used = g_new0(used_conns_t, 1); + g_hash_table_insert(table, + g_strdup(con->client->response->username->str), total_used); + } + total_used->num++; + } + + } else { + + if (con->servers != NULL) { + for (j = 0; j < con->servers->len; j++) { + network_socket *sock = g_ptr_array_index(con->servers, j); + GHashTable *table = g_hash_table_lookup(back_user_conn_hash_table, + sock->dst->name->str); + if (table == NULL) { + g_warning("%s: table is null for backend:%s", G_STRLOC, + sock->dst->name->str); + continue; + } + + used_conns_t *total_used = g_hash_table_lookup(table, + con->client->response->username->str); + if (total_used == NULL) { + total_used = g_new0(used_conns_t, 1); + g_hash_table_insert(table, + g_strdup(con->client->response->username->str), total_used); + } + total_used->num++; + } + } else { + if (con->server == NULL) { + continue; + } + + GHashTable *table = g_hash_table_lookup(back_user_conn_hash_table, + con->server->dst->name->str); + if (table == NULL) { + g_warning("%s: table is null for backend:%s", G_STRLOC, + con->server->dst->name->str); + continue; + } + used_conns_t *total_used = g_hash_table_lookup(table, + con->client->response->username->str); + if (total_used == NULL) { + total_used = g_new0(used_conns_t, 1); + g_hash_table_insert(table, + g_strdup(con->client->response->username->str), total_used); + } + + total_used->num++; + } + } + } + + + fields = g_ptr_array_new_with_free_func((void *) network_mysqld_proto_fielddef_free); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("backend_ndx"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("username"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("idle_conns"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("used_conns"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("total_conns"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + rows = g_ptr_array_new_with_free_func((void *) network_mysqld_mysql_field_row_free); + + len = bs->backends->len; + for (i = 0; i < len; i++) { + network_backend_t *backend = bs->backends->pdata[i]; + + GHashTable *table = g_hash_table_lookup(back_user_conn_hash_table, + backend->addr->name->str); + if (table == NULL) { + g_warning("%s: table is null for backend:%s", G_STRLOC, + backend->addr->name->str); + continue; + } + + GHashTable *users = backend->pool->users; + if (users != NULL) { + GHashTableIter iter; + GString *key; + GQueue *queue; + g_hash_table_iter_init(&iter, users); + /* count all users' pooled connections */ + while (g_hash_table_iter_next(&iter, (void **)&key, (void **)&queue)) { + row = g_ptr_array_new_with_free_func(g_free); + + snprintf(buffer, sizeof(buffer),"%d", i+1); + g_ptr_array_add(row, g_strdup(buffer)); + g_ptr_array_add(row, g_strdup(key->str)); + snprintf(buffer, sizeof(buffer), "%d", queue->length); + g_ptr_array_add(row, g_strdup(buffer)); + + used_conns_t *total_used = g_hash_table_lookup(table, key->str); + if (total_used) { + snprintf(buffer, sizeof(buffer), "%d", total_used->num); + } else { + snprintf(buffer, sizeof(buffer), "%d", 0); + } + g_ptr_array_add(row, g_strdup(buffer)); + + if (total_used) { + snprintf(buffer, sizeof(buffer), "%d", queue->length + total_used->num); + } else { + snprintf(buffer, sizeof(buffer), "%d", queue->length); + } + g_ptr_array_add(row, g_strdup(buffer)); + + g_ptr_array_add(rows, row); + } + } + } + + network_mysqld_con_send_resultset(admin_con->client, fields, rows); + + /* Free data */ + g_ptr_array_free(rows, TRUE); + g_ptr_array_free(fields, TRUE); + + g_hash_table_destroy(back_user_conn_hash_table); + + return PROXY_SEND_RESULT; +} + +static int admin_show_connectionlist(network_mysqld_con *admin_con, const char *sql) +{ + char *arg = str_nth_token(sql, 2); + int number = 65536; + if (arg) { + number = atoi(arg); + g_free(arg); + } + + chassis *chas = admin_con->srv; + chassis_private *priv = chas->priv; + + chassis_plugin_config *config = admin_con->config; + + int i, len; + char buffer[32]; + GPtrArray *fields; + GPtrArray *rows; + GPtrArray *row; + MYSQL_FIELD *field; + + fields = g_ptr_array_new_with_free_func((void *) network_mysqld_proto_fielddef_free); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("User"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("Host"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("db"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("Command"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("Time"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("Trans"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("PS"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("State"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + if (config->has_shard_plugin) { + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("Xa"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("Xid"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + } + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("Server"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("Info"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + rows = g_ptr_array_new_with_free_func((void *) network_mysqld_mysql_field_row_free); + + struct timeval now; + gettimeofday(&(now), NULL); + + len = priv->cons->len; + int count = 0; + + for (i = 0; i < len; i++) { + network_mysqld_con *con = priv->cons->pdata[i]; + + if (!con->client) { + continue; + } + + if (count >= number) { + break; + } + + count++; + + row = g_ptr_array_new_with_free_func(g_free); + if (con->client->response != NULL) { + g_ptr_array_add(row, g_strdup(con->client->response->username->str)); + } else { + g_ptr_array_add(row, NULL); + } + g_ptr_array_add(row, g_strdup(con->client->src->name->str)); + + g_ptr_array_add(row, g_strdup(con->client->default_db->str)); + + if (con->state <= ST_READ_QUERY) { + g_ptr_array_add(row, g_strdup("Sleep")); + g_ptr_array_add(row, g_strdup("0")); + } else { + g_ptr_array_add(row, g_strdup("Query")); + int diff = (now.tv_sec - con->req_recv_time.tv_sec) * 1000; + diff += (now.tv_usec - con->req_recv_time.tv_usec) / 1000; + snprintf(buffer, sizeof(buffer), "%d", diff); + g_ptr_array_add(row, g_strdup(buffer)); + } + g_ptr_array_add(row, g_strdup(con->is_in_transaction ? "Y" : "N")); + g_ptr_array_add(row, g_strdup(con->is_prepared ? "Y" : "N")); + + g_ptr_array_add(row, g_strdup(network_mysqld_con_st_name(con->state))); + + if (config->has_shard_plugin) { + g_ptr_array_add(row, g_strdup(get_conn_xa_state_name(con->dist_tran_state))); + if (con->dist_tran) { + snprintf(buffer, sizeof(buffer), "%lld", con->xa_id); + g_ptr_array_add(row, g_strdup(buffer)); + } else { + g_ptr_array_add(row, NULL); + } + } + + if (con->servers != NULL) { + int j; + GString *servers = g_string_new(NULL); + for (j = 0; j < con->servers->len; j++) { + server_session_t *pmd = g_ptr_array_index(con->servers, j); + if (pmd && pmd->server) { + if (pmd->server->src) { + g_string_append_len(servers, S(pmd->server->src->name)); + char *delim = "->"; + g_string_append_len(servers, delim, strlen(delim)); + } + g_string_append_len(servers, S(pmd->server->dst->name)); + g_string_append_c(servers, ' '); + } + } + g_ptr_array_add(row, g_strdup(servers->str)); + g_string_free(servers, TRUE); + + } else { + if (con->server != NULL) { + GString *server = g_string_new(NULL); + if (con->server->src) { + g_string_append_len(server, S(con->server->src->name)); + char *delim = "->"; + g_string_append_len(server, delim, strlen(delim)); + } + g_string_append_len(server, S(con->server->dst->name)); + g_ptr_array_add(row, g_strdup(server->str)); + g_string_free(server, TRUE); + } else { + g_ptr_array_add(row, NULL); + } + } + + if (con->orig_sql->len) { + if (con->state == ST_READ_QUERY) { + g_ptr_array_add(row, NULL); + } else { + g_ptr_array_add(row, g_strdup(con->orig_sql->str)); + } + } else { + g_ptr_array_add(row, NULL); + } + + g_ptr_array_add(rows, row); + } + + network_mysqld_con_send_resultset(admin_con->client, fields, rows); + + g_ptr_array_free(rows, TRUE); + g_ptr_array_free(fields, TRUE); + return PROXY_SEND_RESULT; +} + +static GList *network_mysqld_admin_plugin_allow_ip_get(chassis_plugin_config *config) { + if (config && config->allow_ip_table) { + return g_hash_table_get_keys(config->allow_ip_table); + } + return NULL; +} + +static GList *network_mysqld_admin_plugin_deny_ip_get(chassis_plugin_config *config) { + if (config && config->deny_ip_table) { + return g_hash_table_get_keys(config->deny_ip_table); + } + return NULL; +} + +static gboolean +network_mysqld_admin_plugin_allow_ip_add(chassis_plugin_config *config, char *addr) +{ + if (!config || !addr) return FALSE; + if (!config->allow_ip_table) { + config->allow_ip_table = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + } + gboolean success = FALSE; + if (!g_hash_table_lookup(config->allow_ip_table, addr)) { + g_hash_table_insert(config->allow_ip_table, g_strdup(addr), (void *) TRUE); + success = TRUE; + } + return success; +} + +static gboolean +network_mysqld_admin_plugin_deny_ip_add(chassis_plugin_config *config, char *addr) +{ + if (!config || !addr) return FALSE; + if (!config->deny_ip_table) { + config->deny_ip_table = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + } + gboolean success = FALSE; + if (!g_hash_table_lookup(config->deny_ip_table, addr)) { + g_hash_table_insert(config->deny_ip_table, g_strdup(addr), (void *) TRUE); + success = TRUE; + } + return success; +} + +static gboolean +network_mysqld_admin_plugin_allow_ip_del(chassis_plugin_config *config, char *addr) +{ + if (!config || !addr || !config->allow_ip_table) return FALSE; + return g_hash_table_remove(config->allow_ip_table, addr); +} + +static gboolean +network_mysqld_admin_plugin_deny_ip_del(chassis_plugin_config *config, char *addr) +{ + if (!config || !addr || !config->deny_ip_table) return FALSE; + return g_hash_table_remove(config->deny_ip_table, addr); +} + +static int admin_show_allow_ip(network_mysqld_con *con, const char *sql) { + char *module_name = str_nth_token(sql, 2); + if (!module_name) { + return PROXY_NO_DECISION; + } + if (strcmp(module_name, "admin") != 0 + && strcmp(module_name, "proxy") != 0 + && strcmp(module_name, "shard") != 0) { + g_free(module_name); + return PROXY_NO_DECISION; + } + chassis *chas = con->srv; + GPtrArray *fields = g_ptr_array_new_with_free_func((void *) network_mysqld_proto_fielddef_free); + MYSQL_FIELD *field; + GPtrArray *rows = g_ptr_array_new_with_free_func((void *) network_mysqld_mysql_field_row_free); + GPtrArray *row; + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("Plugin"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("Address"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + int i; + for (i = 0; i < chas->modules->len; i++) { + chassis_plugin *plugin = chas->modules->pdata[i]; + if (strcmp(plugin->name, module_name) == 0) { + GList *allow_ip_list = plugin->allow_ip_get(plugin->config); + if (allow_ip_list) { + GList *cur_p = allow_ip_list; + while (cur_p) { + row = g_ptr_array_new_with_free_func(g_free); + g_ptr_array_add(row, g_strdup(module_name)); + g_ptr_array_add(row, g_strdup((char *) cur_p->data)); + g_ptr_array_add(rows, row); + cur_p = cur_p->next; + } + } + break; + } + } + + network_mysqld_con_send_resultset(con->client, fields, rows); + + /* Free data */ + g_ptr_array_free(rows, TRUE); + g_ptr_array_free(fields, TRUE); + g_free(module_name); + return PROXY_SEND_RESULT; +} + +static int admin_show_deny_ip(network_mysqld_con *con, const char *sql) { + char *module_name = str_nth_token(sql, 2); + if (!module_name) { + return PROXY_NO_DECISION; + } + if (strcmp(module_name, "admin") != 0 + && strcmp(module_name, "proxy") != 0 + && strcmp(module_name, "shard") != 0) { + g_free(module_name); + return PROXY_NO_DECISION; + } + chassis *chas = con->srv; + GPtrArray *fields = g_ptr_array_new_with_free_func((void *) network_mysqld_proto_fielddef_free); + MYSQL_FIELD *field; + GPtrArray *rows = g_ptr_array_new_with_free_func((void *) network_mysqld_mysql_field_row_free); + GPtrArray *row; + + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("Plugin"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("Address"); + field->type = MYSQL_TYPE_STRING; + g_ptr_array_add(fields, field); + + int i; + for (i = 0; i < chas->modules->len; i++) { + chassis_plugin *plugin = chas->modules->pdata[i]; + if (strcmp(plugin->name, module_name) == 0) { + GList *deny_ip_list = plugin->deny_ip_get(plugin->config); + if (deny_ip_list) { + GList *cur_p = deny_ip_list; + while (cur_p) { + row = g_ptr_array_new_with_free_func(g_free); + g_ptr_array_add(row, g_strdup(module_name)); + g_ptr_array_add(row, g_strdup((char *) cur_p->data)); + g_ptr_array_add(rows, row); + cur_p = cur_p->next; + } + } + break; + } + } + + network_mysqld_con_send_resultset(con->client, fields, rows); + + /* Free data */ + g_ptr_array_free(rows, TRUE); + g_ptr_array_free(fields, TRUE); + g_free(module_name); + return PROXY_SEND_RESULT; +} + +static int admin_add_allow_ip(network_mysqld_con *con, const char *sql) +{ + char *module_name = str_nth_token(sql, 2); + if (!module_name) + return PROXY_NO_DECISION; + char *addr = str_nth_token(sql, 3); + if (!addr) + return PROXY_NO_DECISION; + if (strcmp(module_name, "admin") != 0 + && strcmp(module_name, "proxy") != 0 + && strcmp(module_name, "shard") != 0) { + g_free(module_name); + g_free(addr); + return PROXY_SEND_RESULT; + } + + chassis *chas = con->srv; + int i; + gboolean success = FALSE; + for (i = 0; i < chas->modules->len; i++) { + chassis_plugin *plugin = chas->modules->pdata[i]; + if (strcmp(plugin->name, module_name) == 0) { + success = plugin->allow_ip_add(plugin->config, addr); + break; + } + } + network_mysqld_con_send_ok_full(con->client, success, 0, SERVER_STATUS_AUTOCOMMIT, 0); + g_free(module_name); + g_free(addr); + + return PROXY_SEND_RESULT; +} + +static int admin_add_deny_ip(network_mysqld_con *con, const char *sql) +{ + char *module_name = str_nth_token(sql, 2); + if (!module_name) + return PROXY_NO_DECISION; + char *addr = str_nth_token(sql, 3); + if (!addr) + return PROXY_NO_DECISION; + if (strcmp(module_name, "admin") != 0 + && strcmp(module_name, "proxy") != 0 + && strcmp(module_name, "shard") != 0) { + g_free(module_name); + g_free(addr); + return PROXY_SEND_RESULT; + } + + chassis *chas = con->srv; + int i; + gboolean success = FALSE; + for (i = 0; i < chas->modules->len; i++) { + chassis_plugin *plugin = chas->modules->pdata[i]; + if (strcmp(plugin->name, module_name) == 0) { + success = plugin->deny_ip_add(plugin->config, addr); + break; + } + } + network_mysqld_con_send_ok_full(con->client, success, 0, SERVER_STATUS_AUTOCOMMIT, 0); + g_free(module_name); + g_free(addr); + + return PROXY_SEND_RESULT; +} + +static int admin_delete_allow_ip(network_mysqld_con *con, const char *sql) +{ + char *module_name = str_nth_token(sql, 2); + if (!module_name) + return PROXY_NO_DECISION; + char *addr = str_nth_token(sql, 3); + if (!addr) + return PROXY_NO_DECISION; + if (strcmp(module_name, "admin") != 0 + && strcmp(module_name, "proxy") != 0 + && strcmp(module_name, "shard") != 0) { + g_free(module_name); + g_free(addr); + return PROXY_SEND_RESULT; + } + chassis *chas = con->srv; + int i; + gboolean success = FALSE; + for (i = 0; i < chas->modules->len; i++) { + chassis_plugin *plugin = chas->modules->pdata[i]; + if (strcmp(plugin->name, module_name) == 0) { + success = plugin->allow_ip_del(plugin->config, addr); + break; + } + } + network_mysqld_con_send_ok_full(con->client, success, 0, SERVER_STATUS_AUTOCOMMIT, 0); + g_free(module_name); + g_free(addr); + + return PROXY_SEND_RESULT; +} + +static int admin_delete_deny_ip(network_mysqld_con *con, const char *sql) +{ + char *module_name = str_nth_token(sql, 2); + if (!module_name) + return PROXY_NO_DECISION; + char *addr = str_nth_token(sql, 3); + if (!addr) + return PROXY_NO_DECISION; + if (strcmp(module_name, "admin") != 0 + && strcmp(module_name, "proxy") != 0 + && strcmp(module_name, "shard") != 0) { + g_free(module_name); + g_free(addr); + return PROXY_SEND_RESULT; + } + chassis *chas = con->srv; + int i; + gboolean success = FALSE; + for (i = 0; i < chas->modules->len; i++) { + chassis_plugin *plugin = chas->modules->pdata[i]; + if (strcmp(plugin->name, module_name) == 0) { + success = plugin->deny_ip_del(plugin->config, addr); + break; + } + } + network_mysqld_con_send_ok_full(con->client, success, 0, SERVER_STATUS_AUTOCOMMIT, 0); + g_free(module_name); + g_free(addr); + + return PROXY_SEND_RESULT; +} + +#define MAKE_FIELD_DEF_1_COL(fields, col_name) \ + do {\ + MYSQL_FIELD *field = network_mysqld_proto_fielddef_new();\ + field->name = g_strdup((col_name)); \ + field->type = FIELD_TYPE_VAR_STRING;\ + g_ptr_array_add(fields, field); \ + }while(0) + +#define MAKE_FIELD_DEF_2_COL(fields, col1_name, col2_name) \ + do {\ + MYSQL_FIELD *field = network_mysqld_proto_fielddef_new();\ + field->name = g_strdup((col1_name)); \ + field->type = FIELD_TYPE_VAR_STRING;\ + g_ptr_array_add(fields, field); \ + field = network_mysqld_proto_fielddef_new(); \ + field->name = g_strdup((col2_name)); \ + field->type = FIELD_TYPE_VAR_STRING;\ + g_ptr_array_add(fields, field); \ + }while(0) + +#define APPEND_ROW_1_COL(rows, row_data) \ + do {\ + GPtrArray *row = g_ptr_array_new();\ + g_ptr_array_add(row, (row_data)); \ + g_ptr_array_add(rows, row);\ + }while(0) + +#define APPEND_ROW_2_COL(rows, col1, col2) \ + do {\ + GPtrArray *row = g_ptr_array_new();\ + g_ptr_array_add(row, (col1)); \ + g_ptr_array_add(row, (col2)); \ + g_ptr_array_add(rows, row);\ + }while(0) + + +static void strip_extra_spaces(char *str) { + int i,x; + for(i=x=0; str[i]; ++i) + if (!isspace(str[i]) || (i > 0 && !isspace(str[i-1]))) + str[x++] = str[i]; + str[x] = '\0'; +} + +static void str_replace(char *p, const char *x, char y) +{ + int i,j; + for (i=j=0; p[i]; ++i) { + if (p[i] == x[0] && p[i+1] == x[1]) { + ++i; + p[j++] = y; + } else { + p[j++] = p[i]; + } + } + p[j] = '\0'; +} + +static void normalize_equal_sign(char *p) +{ + str_replace(p, "= ", '='); + str_replace(p, " =", '='); +} + +/* + * tolower, but leave "quoted strings" unmodified + */ +static void lower_identifiers(char *str) +{ + gboolean in_string = FALSE; + int i; + for (i = 0; str[i]; ++i) { + if (in_string) + continue; + if (str[i] == '\'' || str[i] == '"') + in_string = !in_string; + if (isalpha(str[i])) + str[i] = tolower(str[i]); + } +} + +/* only match % wildcard, case insensitive */ +static gboolean sql_pattern_like(const char *pattern, const char *string) +{ + if (!pattern || pattern[0] == '\0') + return TRUE; + char *glob = g_strdup(pattern); + int i; + for (i = 0; glob[i]; ++i) { + if (glob[i] == '%') glob[i] = '*'; + glob[i] = tolower(glob[i]); + } + char *lower_str = g_ascii_strdown(string, -1); + gboolean rc = g_pattern_match_simple(glob, lower_str); + g_free(glob); + g_free(lower_str); + return rc; +} + +/* returned list must be freed */ +static GList *admin_get_all_options(chassis *chas) +{ + GList* options = g_list_copy(chas->options->options); /* shallow copy */ + return options; +} + +static int admin_show_variables(network_mysqld_con *con, const char *sql) +{ + char **tokens = g_strsplit(sql, " ", -1); + int token_count = g_strv_length(tokens); + char *pattern = NULL; + if (token_count == 2) { + pattern = "%"; + } else if (token_count == 4) { + pattern = tokens[3]; + cetus_string_dequote(pattern); + } else { + network_mysqld_con_send_error(con->client, C("error syntax")); + g_strfreev(tokens); + return PROXY_SEND_RESULT; + } + GList *options = admin_get_all_options(con->srv); + + GPtrArray *fields = network_mysqld_proto_fielddefs_new(); + MAKE_FIELD_DEF_2_COL(fields, "Variable_name", "Value"); + + GPtrArray *rows = g_ptr_array_new_with_free_func( + (void *)network_mysqld_mysql_field_row_free); + + GList *freelist = NULL; + GList *l = NULL; + for (l = options; l; l = l->next) { + chassis_option_t *opt = l->data; + /* just support these for now */ + if (opt->arg != OPTION_ARG_INT && opt->arg != OPTION_ARG_INT64 + &&opt->arg != OPTION_ARG_DOUBLE && opt->arg != OPTION_ARG_STRING + &&opt->arg != OPTION_ARG_NONE) + continue; + if (sql_pattern_like(pattern, opt->long_name)) { + char *value = chassis_option_get_value_str(opt); + freelist = g_list_append(freelist, value); + APPEND_ROW_2_COL(rows, (char *)opt->long_name, value); + } + } + network_mysqld_con_send_resultset(con->client, fields, rows); + + network_mysqld_proto_fielddefs_free(fields); + g_ptr_array_free(rows, TRUE); + g_list_free_full(freelist, g_free); + g_list_free(options); + g_strfreev(tokens); + return PROXY_SEND_RESULT; +} + +static int admin_show_status(network_mysqld_con *con, const char *sql) +{ + char **tokens = g_strsplit(sql, " ", -1); + int token_count = g_strv_length(tokens); + char *pattern = NULL; + if (token_count == 2) { + pattern = "%"; + } else if (token_count == 4) { + pattern = tokens[3]; + cetus_string_dequote(pattern); + } else { + network_mysqld_con_send_error(con->client, C("error syntax")); + g_strfreev(tokens); + return PROXY_SEND_RESULT; + } + cetus_variable_t *variables = con->srv->priv->stats_variables; + + GPtrArray *fields = network_mysqld_proto_fielddefs_new(); + MAKE_FIELD_DEF_2_COL(fields, "Variable_name", "Value"); + + GPtrArray *rows = g_ptr_array_new_with_free_func( + (void *)network_mysqld_mysql_field_row_free); + + GList *freelist = NULL; + int i = 0; + for (i = 0; variables[i].name; ++i) { + if (sql_pattern_like(pattern, variables[i].name)) { + char *value = cetus_variable_get_value_str(&variables[i]); + freelist = g_list_append(freelist, value); + APPEND_ROW_2_COL(rows, variables[i].name, value); + } + } + network_mysqld_con_send_resultset(con->client, fields, rows); + + network_mysqld_proto_fielddefs_free(fields); + g_ptr_array_free(rows, TRUE); + g_list_free_full(freelist, g_free); + g_strfreev(tokens); + return PROXY_SEND_RESULT; +} + +static int admin_set_reduce_conns(network_mysqld_con *con, const char *sql) +{ + char *mode = str_nth_token(sql, 2); + if (mode) { + if (strcasecmp(mode, "true") == 0) { + con->srv->is_reduce_conns = 1; + } else if (strcasecmp(mode, "false") == 0) { + con->srv->is_reduce_conns = 0; + } + g_free(mode); + } + network_mysqld_con_send_ok(con->client); + return PROXY_SEND_RESULT; +} + +static int admin_reduce_memory(network_mysqld_con *con, const char *sql) +{ + struct mallinfo m; + + m = mallinfo(); + + g_message("%s:Total allocated space (bytes): %d", G_STRLOC, m.uordblks); + g_message("%s:Total free space (bytes): %d", G_STRLOC, m.fordblks); + g_message("%s:Top-most, releasable space (bytes): %d", G_STRLOC, m.keepcost); + + if (m.fordblks > m.uordblks) { + malloc_trim(0); + m = mallinfo(); + g_message("%s:After trim, total allocated space (bytes): %d", G_STRLOC, m.uordblks); + g_message("%s:After trim, total free space (bytes): %d", G_STRLOC, m.fordblks); + g_message("%s:After trim, top-most, releasable space (bytes): %d", G_STRLOC, m.keepcost); + } + + network_mysqld_con_send_ok_full(con->client, 1, 0, SERVER_STATUS_AUTOCOMMIT, 0); + return PROXY_SEND_RESULT; +} + +static int admin_set_maintain(network_mysqld_con *con, const char *sql) +{ + gboolean error = FALSE; + char *mode = str_nth_token(sql, 2); + if (mode) { + if (strcasecmp(mode, "true") == 0) { + con->srv->maintain_close_mode = 1; + } else if (strcasecmp(mode, "false") == 0) { + con->srv->maintain_close_mode = 0; + } else { + error = TRUE; + } + g_free(mode); + } else { + error = TRUE; + } + if (error) + return PROXY_NO_DECISION; + network_mysqld_con_send_ok(con->client); + return PROXY_SEND_RESULT; +} + +static int admin_reload_shard(network_mysqld_con *con, const char *sql) +{ + network_mysqld_con_send_ok_full(con->client, 1, 0, SERVER_STATUS_AUTOCOMMIT, 0); + return PROXY_SEND_RESULT; +} + +static int admin_send_version(network_mysqld_con *con, const char *sql) +{ + GPtrArray *fields = network_mysqld_proto_fielddefs_new(); + + MAKE_FIELD_DEF_1_COL(fields, "cetus version"); + + GPtrArray *rows = g_ptr_array_new_with_free_func( + (void *)network_mysqld_mysql_field_row_free); + + APPEND_ROW_1_COL(rows, PLUGIN_VERSION); + + network_mysqld_con_send_resultset(con->client, fields, rows); + + network_mysqld_proto_fielddefs_free(fields); + g_ptr_array_free(rows, TRUE); + return PROXY_SEND_RESULT; +} + +static int admin_send_connection_stat(network_mysqld_con *con, const char *sql) +{ + int backend_ndx = -1; + char user[128] = {0}; + sscanf(sql, + "select conn_num from backends where backend_ndx=%d and user=%*['\"]%64[^'\"]%*['\"]", + &backend_ndx, user); + if (backend_ndx == -1 || user[0] == '\0') + return PROXY_NO_DECISION; + + GPtrArray *fields = network_mysqld_proto_fielddefs_new(); + + MAKE_FIELD_DEF_1_COL(fields, "connection_num"); + + GPtrArray *rows = g_ptr_array_new_with_free_func( + (void *)network_mysqld_mysql_field_row_free); + + char *numstr = NULL; + chassis_private *g = con->srv->priv; + backend_ndx -= 1; /* index in sql start from 1, not 0 */ + if (backend_ndx >= 0 && backend_ndx < network_backends_count(g->backends)) { + network_backend_t *backend = network_backends_get(g->backends, backend_ndx); + GString *user_name = g_string_new(user); + + /* TODO: if robbed, conns is not for user_name */ + GQueue *conns = network_connection_pool_get_conns(backend->pool, user_name, NULL); + if (conns) { + numstr = g_strdup_printf("%d", conns->length); + } + g_string_free(user_name, TRUE); + } + if (numstr) { + APPEND_ROW_1_COL(rows, numstr); + } else { + APPEND_ROW_1_COL(rows, "0"); + } + + network_mysqld_con_send_resultset(con->client, fields, rows); + + network_mysqld_proto_fielddefs_free(fields); + g_ptr_array_free(rows, TRUE); + if (numstr) + g_free(numstr); + + return PROXY_SEND_RESULT; +} + +static void bytes_to_hex_str(char *pin, int len, char *pout) +{ + const char *hex = "0123456789ABCDEF"; + int i = 0; + for(; i < len; ++i) { + *pout++ = hex[(*pin>>4)&0xF]; + *pout++ = hex[(*pin++)&0xF]; + } + *pout = 0; +} + +static int admin_send_user_password(network_mysqld_con *con, const char *sql) +{ + char from_table[128] = {0}; + char user[128] = {0}; + char *where_start = strcasestr(sql, "where"); + if (where_start) { + sscanf(sql, "select * from %s where user=%*['\"]%64[^'\"]%*['\"]", from_table, user); + if (from_table[0] == '\0' || user[0] == '\0') + return PROXY_NO_DECISION; + } else { + sscanf(sql, "select * from %s", from_table); + if (from_table[0] == '\0') + return PROXY_NO_DECISION; + } + + chassis_private *g = con->srv->priv; + enum cetus_pwd_type pwd_type; + if (strcmp(from_table, "user_pwd") == 0) { + pwd_type = CETUS_SERVER_PWD; + } else if (strcmp(from_table, "app_user_pwd") == 0) { + pwd_type = CETUS_CLIENT_PWD; + } else { + g_critical("error target db"); + return PROXY_NO_DECISION; + } + + GPtrArray *fields = network_mysqld_proto_fielddefs_new(); + + MAKE_FIELD_DEF_2_COL(fields, "user", "password(sha1)"); + + GPtrArray *rows = g_ptr_array_new_with_free_func( + (void *)network_mysqld_mysql_field_row_free); + + if (user[0]) { /* one user */ + GString *hashpwd = g_string_new(0); + cetus_users_get_hashed_pwd(g->users, user, pwd_type, hashpwd); + char *pwdhex = NULL; + if (hashpwd->len > 0) { + pwdhex = g_malloc0(hashpwd->len *2 + 10); + bytes_to_hex_str(hashpwd->str, hashpwd->len, pwdhex); + APPEND_ROW_2_COL(rows, user, pwdhex); + } + network_mysqld_con_send_resultset(con->client, fields, rows); + network_mysqld_proto_fielddefs_free(fields); + g_ptr_array_free(rows, TRUE); + g_string_free(hashpwd, TRUE); + if (pwdhex) + g_free(pwdhex); + } else { /* all users */ + GList *strings_to_free = NULL; + GHashTableIter iter; + char *username = NULL; + GString *hashpwd = g_string_new(0); + char *hack = NULL; /* don't use value directly */ + g_hash_table_iter_init(&iter, g->users->records); + while (g_hash_table_iter_next(&iter, (gpointer *)&username, (gpointer *)&hack)) { + cetus_users_get_hashed_pwd(g->users, username, pwd_type, hashpwd); + char *pwdhex = NULL; + if (hashpwd->len > 0) { + pwdhex = g_malloc0(hashpwd->len *2 + 10); + bytes_to_hex_str(hashpwd->str, hashpwd->len, pwdhex); + strings_to_free = g_list_append(strings_to_free, pwdhex); + } + APPEND_ROW_2_COL(rows, username, pwdhex); + } + network_mysqld_con_send_resultset(con->client, fields, rows); + network_mysqld_proto_fielddefs_free(fields); + g_ptr_array_free(rows, TRUE); + g_string_free(hashpwd, TRUE); + if (strings_to_free) + g_list_free_full(strings_to_free, g_free); + } + return PROXY_SEND_RESULT; +} + +/* update or insert */ +static int admin_update_user_password(network_mysqld_con *con, const char *sql) +{ + char from_table[128] = {0}; + char user[128] = {0}; + char new_pwd[128] = {0}; + sscanf(sql, "update %64s set password=%*['\"]%64[^'\"]%*['\"] where user=%*['\"]%64[^'\"]%*['\"]", + from_table, new_pwd, user); + if (new_pwd[0] == '\0' || from_table[0] == '\0' || user[0] == '\0') + return PROXY_NO_DECISION; + + chassis_private *g = con->srv->priv; + enum cetus_pwd_type pwd_type = CETUS_UNKNOWN_PWD; + if (strcmp(from_table, "user_pwd") == 0) { + pwd_type = CETUS_SERVER_PWD; + } else if (strcmp(from_table, "app_user_pwd") == 0) { + pwd_type = CETUS_CLIENT_PWD; + } + gboolean affected = cetus_users_update_record(g->users, user, new_pwd, pwd_type); + if (affected) + cetus_users_write_json(g->users); + network_mysqld_con_send_ok_full(con->client, affected?1:0, 0, + SERVER_STATUS_AUTOCOMMIT, 0); + return PROXY_SEND_RESULT; +} + +static int admin_delete_user_password(network_mysqld_con *con, const char *sql) +{ + char from_table[128] = {0}; + char user[128] = {0}; + sscanf(sql, "delete from %64s where user=%*['\"]%64[^'\"]%*['\"]", from_table, user); + if (from_table[0] == '\0' || user[0] == '\0') + return PROXY_NO_DECISION; + + chassis_private *g = con->srv->priv; + gboolean affected = cetus_users_delete_record(g->users, user); + if (affected) + cetus_users_write_json(g->users); + network_mysqld_con_send_ok_full(con->client, affected?1:0, 0, + SERVER_STATUS_AUTOCOMMIT, 0); + return PROXY_SEND_RESULT; +} + +static backend_type_t backend_type(const char *str) +{ + backend_type_t type = BACKEND_TYPE_UNKNOWN; + if (strcasecmp(str, "ro") == 0) + type = BACKEND_TYPE_RO; + else if (strcasecmp(str, "rw") == 0) + type = BACKEND_TYPE_RW; + return type; +} + +static backend_state_t backend_state(const char *str) +{ + backend_state_t state = BACKEND_STATE_UNKNOWN; + if (strcasecmp(str, "up") == 0) + state = BACKEND_STATE_UP; + else if (strcasecmp(str, "down") == 0) + state = BACKEND_STATE_DOWN; + else if (strcasecmp(str, "maintaining") == 0) + state = BACKEND_STATE_MAINTAINING; + return state; +} + +static int admin_insert_backend(network_mysqld_con *con, const char *sql) +{ + /* TODO: to which group */ + char address[128] = {0}; + char type_str[64] = {0}; + char state_str[64] = {0}; + sscanf(sql, "insert into backends values (%*['\"]%64[0-9:.@a-zA-Z]%*['\"], %*['\"]%32[rowRWO]%*['\"], %*['\"]%32[a-zA-Z]%*['\"])", + address, type_str, state_str); + if (address[0] == '\0' || type_str[0] == '\0' || state_str[0] == '\0') + return PROXY_NO_DECISION; + + chassis_private *g = con->srv->priv; + int affected = network_backends_add(g->backends, address, + backend_type(type_str), + backend_state(state_str), + con->srv); + network_mysqld_con_send_ok_full(con->client, affected == 0?1:0, 0, + SERVER_STATUS_AUTOCOMMIT, 0); + return PROXY_SEND_RESULT; +} + +static int admin_add_backend(network_mysqld_con *con, const char *sql) +{ + char address[128] = {0}; + backend_type_t type = BACKEND_TYPE_UNKNOWN; + if (strncasecmp(sql, "add master", 10) == 0) { + sscanf(sql, "add master %*['\"]%64[0-9:.@a-zA-Z]%*['\"]", address); + type = BACKEND_TYPE_RW; + } else if (strncasecmp(sql, "add slave", 9) == 0) { + sscanf(sql, "add slave %*['\"]%64[0-9:.@a-zA-Z]%*['\"]", address); + type = BACKEND_TYPE_RO; + } + if (address[0] == '\0' || type == BACKEND_TYPE_UNKNOWN) + return PROXY_NO_DECISION; + + chassis_private *g = con->srv->priv; + int affected = network_backends_add(g->backends, address, type, BACKEND_STATE_UNKNOWN, con->srv); + network_mysqld_con_send_ok_full(con->client, affected == 0?1:0, 0, + SERVER_STATUS_AUTOCOMMIT, 0); + return PROXY_SEND_RESULT; +} + +static int admin_update_backend(network_mysqld_con *con, const char *sql) +{ + char *set_start = strcasestr(sql, "set"); + char *where_start = strcasestr(sql, "where"); + + char *set_cols_str = NULL; + if (where_start) { + set_cols_str = g_strndup(set_start+4, where_start-set_start-4); + } else { + set_cols_str = g_strdup(set_start+4); + } + char **set_cols_array = g_strsplit(set_cols_str, ",", -1); + int i = 0; + char type_str[64] = {0}; + char state_str[64] = {0}; + char *col; + for (; (col = set_cols_array[i]) != NULL; ++i) { + g_strstrip(col); + if (strncasecmp(col, "type", 4) == 0) { + sscanf(col, "type=%*['\"]%32[a-zA-Z]%*['\"]", type_str); + } else if (strncasecmp(col, "state", 5) == 0) { + sscanf(col, "state=%*['\"]%32[a-zA-Z]%*['\"]", state_str); + } + } + g_free(set_cols_str); + g_strfreev(set_cols_array); + + if (type_str[0] == '\0' && state_str[0] == '\0') + return PROXY_NO_DECISION; + + chassis_private *g = con->srv->priv; + int affected_rows = 0; + if (where_start) { + int backend_ndx = -1; + sscanf(where_start, "where backend_ndx=%d", &backend_ndx); + backend_ndx -= 1; /* index in sql start from 1, not 0 */ + if (backend_ndx < 0) { + char address[128] = {0}; + sscanf(where_start, "where address=%*['\"]%64[0-9:.]%*['\"]", address); + backend_ndx = network_backends_find_address(g->backends, address); + } + network_backend_t *bk = network_backends_get(g->backends, backend_ndx); + if (!bk) { + network_mysqld_con_send_error(con->client, C("no such backend")); + return PROXY_SEND_RESULT; + } + int type = type_str[0] != '\0' ? backend_type(type_str) : bk->type; + int state = state_str[0] != '\0' ? backend_state(state_str) : bk->state; + network_backends_modify(g->backends, backend_ndx, type, state); + affected_rows = 1; + } else { + for (i = 0; i < network_backends_count(g->backends); ++i) { + network_backend_t *bk = network_backends_get(g->backends, i); + if (bk) { + int type = type_str[0] != '\0' ? backend_type(type_str) : bk->type; + int state = state_str[0] != '\0' ? backend_state(state_str) : bk->state; + network_backends_modify(g->backends, i, type, state); + affected_rows += 1; + } + } + } + network_mysqld_con_send_ok_full(con->client, affected_rows, 0, + SERVER_STATUS_AUTOCOMMIT, 0); + return PROXY_SEND_RESULT; +} + +static int admin_delete_backend(network_mysqld_con *con, const char *sql) +{ + char *where_start = strcasestr(sql, "where"); + + chassis_private *g = con->srv->priv; + int backend_ndx = -1; + if (where_start) {/* delete from backends where xx=xx */ + sscanf(where_start, "where backend_ndx=%d", &backend_ndx); + backend_ndx -= 1; /* index in sql start from 1, not 0 */ + if (backend_ndx < 0) { + char address[128] = {0}; + sscanf(where_start, "where address=%*['\"]%64[0-9:.]%*['\"]", address); + backend_ndx = network_backends_find_address(g->backends, address); + } + } else {/* remove backend xxx */ + sscanf(sql, "remove backend %d", &backend_ndx); + backend_ndx -= 1; /* index in sql start from 1, not 0 */ + } + if (backend_ndx >= 0 && backend_ndx < network_backends_count(g->backends)) { + network_backends_remove(g->backends, backend_ndx);/* TODO: just change state? */ + network_mysqld_con_send_ok_full(con->client, 1, 0, + SERVER_STATUS_AUTOCOMMIT, 0); + return PROXY_SEND_RESULT; + } else { + network_mysqld_con_send_ok_full(con->client, 0, 0, + SERVER_STATUS_AUTOCOMMIT, 0); + return PROXY_SEND_RESULT; + } +} + +static int admin_supported_stats(network_mysqld_con *con) +{ + GPtrArray *fields = network_mysqld_proto_fielddefs_new(); + MAKE_FIELD_DEF_1_COL(fields, "name"); + GPtrArray *rows = g_ptr_array_new_with_free_func( + (void *)network_mysqld_mysql_field_row_free); + APPEND_ROW_1_COL(rows, "client_query"); + APPEND_ROW_1_COL(rows, "proxyed_query"); + APPEND_ROW_1_COL(rows, "reset"); + APPEND_ROW_1_COL(rows, "query_time_table"); + APPEND_ROW_1_COL(rows, "server_query_details"); + APPEND_ROW_1_COL(rows, "query_wait_table"); + network_mysqld_con_send_resultset(con->client, fields, rows); + network_mysqld_proto_fielddefs_free(fields); + g_ptr_array_free(rows, TRUE); + return PROXY_SEND_RESULT; +} + +static int admin_get_stats(network_mysqld_con *con, const char *sql) +{ + const char *p = sql + 9; + if (*p == '\0') { /* just "stats get", no argument */ + return admin_supported_stats(con); + } else { + ++p; /* stats get [xxx], point to xxx */ + } + + GPtrArray *fields = network_mysqld_proto_fielddefs_new(); + MAKE_FIELD_DEF_2_COL(fields, "name", "value"); + GPtrArray *rows = g_ptr_array_new_with_free_func( + (void *)network_mysqld_mysql_field_row_free); + chassis *chas = con->srv; + query_stats_t *stats = &(chas->query_stats); + char buf1[32] = {0}; + char buf2[32] = {0}; + int i; + if (strcasecmp(p, "client_query") == 0) { + snprintf(buf1, 32, "%lu", stats->client_query.ro); + snprintf(buf2, 32, "%lu", stats->client_query.rw); + APPEND_ROW_2_COL(rows, "client_query.ro", buf1); + APPEND_ROW_2_COL(rows, "client_query.rw", buf2); + } else if (strcasecmp(p, "proxyed_query") == 0) { + snprintf(buf1, 32, "%lu", stats->proxyed_query.ro); + snprintf(buf2, 32, "%lu", stats->proxyed_query.rw); + APPEND_ROW_2_COL(rows, "proxyed_query.ro", buf1); + APPEND_ROW_2_COL(rows, "proxyed_query.rw", buf2); + } else if (strcasecmp(p, "query_time_table") == 0) { + for (i = 0; i < MAX_QUERY_TIME && stats->query_time_table[i]; ++i) { + GPtrArray *row = g_ptr_array_new_with_free_func(g_free); + g_ptr_array_add(row, g_strdup_printf("query_time_table.%d", i+1)); + g_ptr_array_add(row, g_strdup_printf("%lu", stats->query_time_table[i])); + g_ptr_array_add(rows, row); + } + } else if (strcasecmp(p, "query_wait_table") == 0) { + for (i = 0; i < MAX_QUERY_TIME && stats->query_wait_table[i]; ++i) { + GPtrArray *row = g_ptr_array_new_with_free_func(g_free); + g_ptr_array_add(row, g_strdup_printf("query_wait_table.%d", i+1)); + g_ptr_array_add(row, g_strdup_printf("%lu", stats->query_wait_table[i])); + g_ptr_array_add(rows, row); + } + } else if (strcasecmp(p, "server_query_details") == 0) { + for (i = 0; i < MAX_SERVER_NUM && i < network_backends_count(chas->priv->backends); ++i) { + GPtrArray *row = g_ptr_array_new_with_free_func(g_free); + g_ptr_array_add(row, g_strdup_printf("server_query_details.%d.ro", i+1)); + g_ptr_array_add(row, g_strdup_printf("%lu", stats->server_query_details[i].ro)); + g_ptr_array_add(rows, row); + row = g_ptr_array_new_with_free_func(g_free); + g_ptr_array_add(row, g_strdup_printf("server_query_details.%d.rw", i+1)); + g_ptr_array_add(row, g_strdup_printf("%lu", stats->server_query_details[i].rw)); + g_ptr_array_add(rows, row); + } + } else if (strcasecmp(p, "reset") == 0) { + APPEND_ROW_2_COL(rows, "reset", "0"); + } else { + APPEND_ROW_2_COL(rows, (char *)p, (char *)p); + } + network_mysqld_con_send_resultset(con->client, fields, rows); + + network_mysqld_proto_fielddefs_free(fields); + g_ptr_array_free(rows, TRUE); + return PROXY_SEND_RESULT; +} + +static int admin_supported_config(network_mysqld_con *con) +{ + GPtrArray *fields = network_mysqld_proto_fielddefs_new(); + MAKE_FIELD_DEF_1_COL(fields, "name"); + GPtrArray *rows = g_ptr_array_new_with_free_func( + (void *)network_mysqld_mysql_field_row_free); + APPEND_ROW_1_COL(rows, "common"); + APPEND_ROW_1_COL(rows, "pool"); + network_mysqld_con_send_resultset(con->client, fields, rows); + network_mysqld_proto_fielddefs_free(fields); + g_ptr_array_free(rows, TRUE); + return PROXY_SEND_RESULT; +} + +static int admin_get_config(network_mysqld_con *con, const char *sql) +{ + const char *p = sql + 10; + if (*p == '\0') { /* just "config get", no argument */ + return admin_supported_config(con); + } else { + ++p; /* config get [xxx], point to xxx */ + } + + GPtrArray *fields = network_mysqld_proto_fielddefs_new(); + MAKE_FIELD_DEF_2_COL(fields, "name", "value"); + GPtrArray *rows = g_ptr_array_new_with_free_func( + (void *)network_mysqld_mysql_field_row_free); + chassis *chas = con->srv; + char buf1[32] = {0}, buf2[32] = {0}; + char buf3[32] = {0}, buf4[32] = {0}; + if (strcasecmp(p, "common") == 0) { + snprintf(buf1, 32, "%d", chas->check_slave_delay); + snprintf(buf2, 32, "%f", chas->slave_delay_down_threshold_sec); + snprintf(buf3, 32, "%f", chas->slave_delay_recover_threshold_sec); + snprintf(buf4, 32, "%d", chas->long_query_time); + APPEND_ROW_2_COL(rows, "common.check_slave_delay", buf1); + APPEND_ROW_2_COL(rows, "common.slave_delay_down_threshold_sec", buf2); + APPEND_ROW_2_COL(rows, "common.slave_delay_recover_threshold_sec", buf3); + APPEND_ROW_2_COL(rows, "common.long_query_time", buf4); + } else if (strcasecmp(p, "pool") == 0) { + snprintf(buf1, 32, "%d", chas->mid_idle_connections); + snprintf(buf2, 32, "%d", chas->max_idle_connections); + snprintf(buf3, 32, "%d", chas->max_resp_len); + snprintf(buf4, 32, "%d", chas->master_preferred); + APPEND_ROW_2_COL(rows, "pool.default_pool_size", buf1); + APPEND_ROW_2_COL(rows, "pool.max_pool_size", buf2); + APPEND_ROW_2_COL(rows, "pool.max_resp_len", buf3); + APPEND_ROW_2_COL(rows, "pool.master_preferred", buf4); + } else { + APPEND_ROW_2_COL(rows, (char *)p, (char *)p); + } + network_mysqld_con_send_resultset(con->client, fields, rows); + + network_mysqld_proto_fielddefs_free(fields); + g_ptr_array_free(rows, TRUE); + return PROXY_SEND_RESULT; +} + +static int admin_set_config(network_mysqld_con *con, const char *sql) +{ + char key[128] = {0}; + int val = -1; + sscanf(sql, "config set %64[a-zA-Z0-9_.]=%d", key, &val); + if (key[0] == '\0' || val == -1) { + network_mysqld_con_send_ok_full(con->client, 0, 0, + SERVER_STATUS_AUTOCOMMIT, 0); + return PROXY_SEND_RESULT; + } + chassis *chas = con->srv; + int affected_rows = 1; + if (strcasecmp(key, "common.check_slave_delay") == 0) { + chas->check_slave_delay = val; + } else if (strcasecmp(key, "common.slave_delay_down_threshold_sec") == 0) { + chas->slave_delay_down_threshold_sec = val; + } else if (strcasecmp(key, "common.slave_delay_recover_threshold_sec") == 0) { + chas->slave_delay_recover_threshold_sec = val; + } else if (strcasecmp(key, "common.long_query_time") == 0) { + chas->long_query_time = val; + } else if (strcasecmp(key, "pool.default_pool_size") == 0) { + chas->mid_idle_connections = val; + } else if (strcasecmp(key, "pool.max_pool_size") == 0) { + chas->max_idle_connections = val; + } else if (strcasecmp(key, "pool.max_resp_len") == 0) { + chas->max_resp_len = val; + } else if (strcasecmp(key, "pool.master_preferred") == 0) { + chas->master_preferred = val; + } else { + affected_rows = 0; + } + network_mysqld_con_send_ok_full(con->client, affected_rows, 0, + SERVER_STATUS_AUTOCOMMIT, 0); + return PROXY_SEND_RESULT; +} + +static int admin_reset_stats(network_mysqld_con *con, const char *sql) +{ + query_stats_t *stats = &con->srv->query_stats; + memset(stats, 0, sizeof(*stats)); + network_mysqld_con_send_ok_full(con->client, 1, 0, + SERVER_STATUS_AUTOCOMMIT, 0); + return PROXY_SEND_RESULT; +} + +static int admin_save_settings(network_mysqld_con *con, const char *sql) +{ + /* TODO:*/ + network_mysqld_con_send_ok_full(con->client, 1, 0, + SERVER_STATUS_AUTOCOMMIT, 0); + return PROXY_SEND_RESULT; +} + +static void calc_qps_average(char *buf, int len); +static void calc_tps_average(char *buf, int len); + +static void get_module_names(chassis *chas, GString *plugin_names) +{ + int i; + for (i = 0; i < chas->modules->len; ++i) { + chassis_plugin *p = g_ptr_array_index(chas->modules, i); + g_string_append(plugin_names, p->name); + g_string_append_c(plugin_names, ' '); + } +} + +static int admin_send_status(network_mysqld_con *con, const char *sql) +{ + chassis_private *g = con->srv->priv; + chassis_plugin_config *config = con->config; + + GPtrArray *fields = network_mysqld_proto_fielddefs_new(); + MAKE_FIELD_DEF_2_COL(fields, "Status", "Value"); + + GPtrArray *rows = g_ptr_array_new_with_free_func( + (void *)network_mysqld_mysql_field_row_free); + APPEND_ROW_2_COL(rows, "Cetus version", PLUGIN_VERSION); + char start_time[32]; + chassis_epoch_to_string(&con->srv->startup_time, C(start_time)); + APPEND_ROW_2_COL(rows, "Startup time", start_time); + GString *plugin_names = g_string_new(0); + get_module_names(con->srv, plugin_names); + APPEND_ROW_2_COL(rows, "Loaded modules", plugin_names->str); + const int bsize = 32; + static char buf1[32], buf2[32], buf3[32]; + snprintf(buf1, bsize, "%d", network_backends_idle_conns(g->backends)); + APPEND_ROW_2_COL(rows, "Idle backend connections", buf1); + snprintf(buf2, bsize, "%d", network_backends_used_conns(g->backends)); + APPEND_ROW_2_COL(rows, "Used backend connections", buf2); + snprintf(buf3, bsize, "%d", g->cons->len); + APPEND_ROW_2_COL(rows, "Client connections", buf3); + + query_stats_t *stats = &(con->srv->query_stats); + char qcount[32]; + snprintf(qcount, 32, "%ld", stats->client_query.ro+stats->client_query.rw); + APPEND_ROW_2_COL(rows, "Query count", qcount); + + if (config->has_shard_plugin) { + char xacount[32]; + snprintf(xacount, 32, "%ld", stats->xa_count); + APPEND_ROW_2_COL(rows, "XA count", xacount); + } + + char qps[64]; + calc_qps_average(C(qps)); + APPEND_ROW_2_COL(rows, "QPS (1min, 5min, 15min)", qps); + char tps[64]; + calc_tps_average(C(tps)); + APPEND_ROW_2_COL(rows, "TPS (1min, 5min, 15min)", tps); + network_mysqld_con_send_resultset(con->client, fields, rows); + + network_mysqld_proto_fielddefs_free(fields); + g_ptr_array_free(rows, TRUE); + g_string_free(plugin_names, TRUE); + return PROXY_SEND_RESULT; +} + +static int admin_send_group_info(network_mysqld_con *con, const char *sql) +{ + GPtrArray *fields = network_mysqld_proto_fielddefs_new(); + + MYSQL_FIELD *field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("group"); + field->type = FIELD_TYPE_VAR_STRING; + g_ptr_array_add(fields, field); + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup(("master")); + field->type = FIELD_TYPE_VAR_STRING; + g_ptr_array_add(fields, field); + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup(("slaves")); + field->type = FIELD_TYPE_VAR_STRING; + g_ptr_array_add(fields, field); + + GPtrArray *rows = g_ptr_array_new_with_free_func( + (void *)network_mysqld_mysql_field_row_free); + + GList *free_list = NULL; + network_backends_t *bs = con->srv->priv->backends; + int i; + for (i = 0; i < bs->groups->len; ++i) { + network_group_t *gp = g_ptr_array_index(bs->groups, i); + GPtrArray *row = g_ptr_array_new(); + g_ptr_array_add(row, gp->name->str); + if (gp->master) { + g_ptr_array_add(row, gp->master->addr->name->str); + } else { + g_ptr_array_add(row, ""); + } + GString *slaves = g_string_new(0); + network_group_get_slave_names(gp, slaves); + g_ptr_array_add(row, slaves->str); + free_list = g_list_append(free_list, slaves); + g_ptr_array_add(rows, row); + } + network_mysqld_con_send_resultset(con->client, fields, rows); + + network_mysqld_proto_fielddefs_free(fields); + g_ptr_array_free(rows, TRUE); + g_list_free_full(free_list, g_string_true_free); + return PROXY_SEND_RESULT; + +} + +static int admin_help(network_mysqld_con *con, const char *sql); + +typedef int(*sql_handler_func)(network_mysqld_con *, const char *); + +struct sql_handler_entry_t { + const char *prefix; + sql_handler_func func; + const char *pattern; + const char *desc; +}; + +static struct sql_handler_entry_t sql_handler_shard_map[] = { + {"select conn_details from backend", admin_send_backend_detail_info, + "select conn_details from backend", "display the idle conns"}, + {"select * from backends", admin_send_backends_info, + "select * from backends", "list the backends and their state"}, + {"select * from groups", admin_send_group_info, + "select * from groups"}, + {"show connectionlist", admin_show_connectionlist, + "show connectionlist []", "show connections"}, + {"show allow_ip ", admin_show_allow_ip, + "show allow_ip ", "show allow_ip rules of module, currently admin|proxy|shard"}, + {"show deny_ip ", admin_show_deny_ip, + "show deny_ip ", "show deny_ip rules of module, currently admin|proxy|shard"}, + {"add allow_ip ", admin_add_allow_ip, + "add allow_ip
", "add address to white list of module"}, + {"add deny_ip ", admin_add_deny_ip, + "add deny_ip
", "add address to white list of module"}, + {"delete allow_ip ", admin_delete_allow_ip, + "delete allow_ip
", "delete address from white list of module"}, + {"delete deny_ip ", admin_delete_deny_ip, + "delete deny_ip
", "delete address from white list of module"}, + {"set reduce_conns ", admin_set_reduce_conns, + "set reduce_conns (true|false)", "reduce idle connections if set to true"}, + {"reduce memory", admin_reduce_memory, + "reduce memory", "reduce memory occupied by system"}, + {"set maintain ", admin_set_maintain, + "set maintain (true|false)", "close all client connections if set to true"}, + {"reload shard", admin_reload_shard, + "reload shard", "reload sharding config from remote db"}, + {"show status", admin_show_status, + "show status [like '%pattern%']", "show select/update/insert/delete statistics"}, + {"show variables", admin_show_variables, + "show variables [like '%pattern%']"}, + {"select version", admin_send_version, + "select version", "cetus version"}, + {"select conn_num from backends where", admin_send_connection_stat, + "select conn_num from backends where backend_ndx= and user='')"}, + {"select * from user_pwd", admin_send_user_password, + "select * from user_pwd [where user='']"}, + {"select * from app_user_pwd", admin_send_user_password, + "select * from app_user_pwd [where user='']"}, + {"update user_pwd set password", admin_update_user_password, + "update user_pwd set password='xx' where user=''"}, + {"update app_user_pwd set password", admin_update_user_password, + "update app_user_pwd set password='xx' where user=''"}, + {"delete from user_pwd where", admin_delete_user_password, + "delete from user_pwd where user=''"}, + {"delete from app_user_pwd where", admin_delete_user_password, + "delete from app_user_pwd where user=''"}, + {"insert into backends values", admin_insert_backend, + "insert into backends values ('', '(ro|rw)', '')", + "add mysql instance to backends list"}, + {"update backends set", admin_update_backend, + "update backends set (type|state)=x where (backend_ndx=|address=<'ip:port'>)", + "update mysql instance type or state"}, + {"delete from backends", admin_delete_backend, + "delete from backends where (backend_ndx=|address=<'ip:port'>)"}, + {"remove backend ", admin_delete_backend, /* TODO: unify */ + "remove backend where (backend_ndx=|address=<'ip:port'>)"}, + {"add master", admin_add_backend, "add master <'ip:port@group'>"}, + {"add slave", admin_add_backend, "add slave <'ip:port@group'>"}, + {"stats get", admin_get_stats, "stats get []", "show query statistics"}, + {"config get", admin_get_config, "config get []", "show config"}, + {"config set ", admin_set_config, "config set ="}, + {"stats reset", admin_reset_stats, "stats reset", "reset query statistics"}, + {"save settings ", admin_save_settings, "save settings", "not implemented"}, + {"select * from help", admin_help, "select * from help", "show this help"}, + {"select help", admin_help, "select help", "show this help"}, + {"cetus", admin_send_status, "cetus", "Show overall status of Cetus"}, + {NULL, NULL, NULL, NULL} +}; + +static struct sql_handler_entry_t sql_handler_rw_map[] = { + {"select conn_details from backend", admin_send_backend_detail_info, + "select conn_details from backend", "display the idle conns"}, + {"select * from backends", admin_send_backends_info, + "select * from backends", "list the backends and their state"}, + {"show connectionlist", admin_show_connectionlist, + "show connectionlist []", "show connections"}, + {"show allow_ip ", admin_show_allow_ip, + "show allow_ip ", "show allow_ip rules of module, currently admin|proxy|shard"}, + {"show deny_ip ", admin_show_deny_ip, + "show deny_ip ", "show deny_ip rules of module, currently admin|proxy|shard"}, + {"add allow_ip ", admin_add_allow_ip, + "add allow_ip
", "add address to white list of module"}, + {"add deny_ip ", admin_add_deny_ip, + "add deny_ip
", "add address to white list of module"}, + {"delete allow_ip ", admin_delete_allow_ip, + "delete allow_ip
", "delete address from white list of module"}, + {"delete deny_ip ", admin_delete_deny_ip, + "delete deny_ip
", "delete address from white list of module"}, + {"set reduce_conns ", admin_set_reduce_conns, + "set reduce_conns (true|false)", "reduce idle connections if set to true"}, + {"reduce memory", admin_reduce_memory, + "reduce memory", "reduce memory occupied by system"}, + {"set maintain ", admin_set_maintain, + "set maintain (true|false)", "close all client connections if set to true"}, + {"show status", admin_show_status, + "show status [like '%pattern%']", "show select/update/insert/delete statistics"}, + {"show variables", admin_show_variables, + "show variables [like '%pattern%']"}, + {"select version", admin_send_version, + "select version", "cetus version"}, + {"select conn_num from backends where", admin_send_connection_stat, + "select conn_num from backends where backend_ndx= and user='')"}, + {"select * from user_pwd", admin_send_user_password, + "select * from user_pwd [where user='']"}, + {"select * from app_user_pwd", admin_send_user_password, + "select * from app_user_pwd [where user='']"}, + {"update user_pwd set password", admin_update_user_password, + "update user_pwd set password='xx' where user=''"}, + {"update app_user_pwd set password", admin_update_user_password, + "update app_user_pwd set password='xx' where user=''"}, + {"delete from user_pwd where", admin_delete_user_password, + "delete from user_pwd where user=''"}, + {"delete from app_user_pwd where", admin_delete_user_password, + "delete from app_user_pwd where user=''"}, + {"insert into backends values", admin_insert_backend, + "insert into backends values ('', '(ro|rw)', '')", + "add mysql instance to backends list"}, + {"update backends set", admin_update_backend, + "update backends set (type|state)=x where (backend_ndx=|address=<'ip:port'>)", + "update mysql instance type or state"}, + {"delete from backends", admin_delete_backend, + "delete from backends where (backend_ndx=|address=<'ip:port'>)"}, + {"remove backend ", admin_delete_backend, /* TODO: unify */ + "remove backend where (backend_ndx=|address=<'ip:port'>)"}, + {"add master", admin_add_backend, "add master <'ip:port'>"}, + {"add slave", admin_add_backend, "add slave <'ip:port'>"}, + {"stats get", admin_get_stats, "stats get []", "show query statistics"}, + {"config get", admin_get_config, "config get []", "show config"}, + {"config set ", admin_set_config, "config set ="}, + {"stats reset", admin_reset_stats, "stats reset", "reset query statistics"}, + {"save settings ", admin_save_settings, "save settings", "not implemented"}, + {"select * from help", admin_help, "select * from help", "show this help"}, + {"select help", admin_help, "select help", "show this help"}, + {"cetus", admin_send_status, "cetus", "Show overall status of Cetus"}, + {NULL, NULL, NULL, NULL} +}; + +static int admin_help(network_mysqld_con *con, const char *sql) +{ + GPtrArray *fields = network_mysqld_proto_fielddefs_new(); + MAKE_FIELD_DEF_2_COL(fields, "Command", "Description"); + GPtrArray *rows = g_ptr_array_new_with_free_func( + (void *)network_mysqld_mysql_field_row_free); + + chassis_plugin_config *config = con->config; + struct sql_handler_entry_t *sql_handler_map = + config->has_shard_plugin ? sql_handler_shard_map : sql_handler_rw_map; + int i; + for (i = 0; sql_handler_map[i].prefix; ++i) { + struct sql_handler_entry_t *e = &(sql_handler_map[i]); + char *pattern = e->pattern ? (char *)e->pattern : ""; + char *desc = e->desc ? (char *)e->desc : ""; + APPEND_ROW_2_COL(rows, pattern, desc); + } + + network_mysqld_con_send_resultset(con->client, fields, rows); + network_mysqld_proto_fielddefs_free(fields); + g_ptr_array_free(rows, TRUE); + return PROXY_SEND_RESULT; +} + +static network_mysqld_stmt_ret +admin_process_query(network_mysqld_con *con) +{ + char command = -1; + network_socket *recv_sock = con->client; + GList *chunk = recv_sock->recv_queue->chunks->head; + GString *packet = chunk->data; + + if (packet->len < NET_HEADER_SIZE) { + /* packet too short */ + return PROXY_SEND_QUERY; + } + + command = packet->str[NET_HEADER_SIZE + 0]; + + if (COM_QUERY == command) { + /* we need some more data after the COM_QUERY */ + if (packet->len < NET_HEADER_SIZE + 2) return PROXY_SEND_QUERY; + + /* LOAD DATA INFILE is nasty */ + if (packet->len - NET_HEADER_SIZE - 1 >= sizeof("LOAD ") - 1 && + !g_ascii_strncasecmp(packet->str + NET_HEADER_SIZE + 1, C("LOAD "))) + { + return PROXY_SEND_QUERY; + } + } + + g_string_assign_len(con->orig_sql, packet->str + (NET_HEADER_SIZE + 1), + packet->len - (NET_HEADER_SIZE + 1)); + + g_strstrip(con->orig_sql->str); /* strip leading and trailing spaces */ + strip_extra_spaces(con->orig_sql->str); /* replace multiple spaces with one */ + normalize_equal_sign(con->orig_sql->str); /* remove spaces on the side of = */ + lower_identifiers(con->orig_sql->str); + con->orig_sql->len = strlen(con->orig_sql->str); + + const char *sql = con->orig_sql->str; + + chassis_plugin_config *config = con->config; + struct sql_handler_entry_t *sql_handler_map = + config->has_shard_plugin ? sql_handler_shard_map : sql_handler_rw_map; + int i; + for (i = 0; sql_handler_map[i].prefix; ++i) { + if (strcasestr(sql, sql_handler_map[i].prefix)) { + return sql_handler_map[i].func(con, sql); + } + } + + network_mysqld_con_send_error(con->client, C("request error, \"select * from help\" for usage")); + return PROXY_SEND_RESULT; +} + +/** + * gets called after a query has been read + */ +NETWORK_MYSQLD_PLUGIN_PROTO(server_read_query) { + network_socket *recv_sock; + network_mysqld_stmt_ret ret; + + gettimeofday(&(con->req_recv_time), NULL); + + recv_sock = con->client; + + if (recv_sock->recv_queue->chunks->length != 1) { + g_message("%s: client-recv-queue-len = %d", G_STRLOC, + recv_sock->recv_queue->chunks->length); + } + + ret = admin_process_query(con); + + switch (ret) { + case PROXY_NO_DECISION: + network_mysqld_con_send_error(con->client, + C("request error, \"select * from help\" for usage")); + con->state = ST_SEND_QUERY_RESULT; + break; + case PROXY_SEND_RESULT: + con->state = ST_SEND_QUERY_RESULT; + break; + default: + network_mysqld_con_send_error(con->client, + C("need a resultset + proxy.PROXY_SEND_RESULT, got something else")); + + con->state = ST_SEND_ERROR; + break; + } + + g_string_free(g_queue_pop_tail(recv_sock->recv_queue->chunks), TRUE); + + return NETWORK_SOCKET_SUCCESS; +} + +/** + * cleanup the admin specific data on the current connection + * + * @return NETWORK_SOCKET_SUCCESS + */ +NETWORK_MYSQLD_PLUGIN_PROTO(admin_disconnect_client) { + /* private state is not used in admin-plugin */ + con->plugin_con_state = NULL; + + return NETWORK_SOCKET_SUCCESS; +} + + +static int network_mysqld_server_connection_init(network_mysqld_con *con) { + con->plugins.con_init = server_con_init; + + con->plugins.con_read_auth = server_read_auth; + + con->plugins.con_read_query = server_read_query; + + con->plugins.con_cleanup = admin_disconnect_client; + + return 0; +} + +static chassis_plugin_config *network_mysqld_admin_plugin_new(void) { + chassis_plugin_config *config; + + config = g_new0(chassis_plugin_config, 1); + + return config; +} + +static void network_mysqld_admin_plugin_free(chassis *chas, chassis_plugin_config *config) { + if (config->listen_con) { + /* the socket will be freed by network_mysqld_free() */ + } + + if (config->address) { + chassis_config_unregister_service(chas->config_manager, config->address); + g_free(config->address); + } + if (g_sampling_timer) { + evtimer_del(g_sampling_timer); + g_free(g_sampling_timer); + g_sampling_timer = NULL; + } + + if (config->admin_username) g_free(config->admin_username); + if (config->admin_password) g_free(config->admin_password); + if (config->allow_ip) g_free(config->allow_ip); + if (config->allow_ip_table) g_hash_table_destroy(config->allow_ip_table); + if (config->deny_ip) g_free(config->deny_ip); + if (config->deny_ip_table) g_hash_table_destroy(config->deny_ip_table); + g_free(config); +} + +/** + * add the proxy specific options to the cmdline interface + */ +static GList * +network_mysqld_admin_plugin_get_options(chassis_plugin_config *config) +{ + chassis_options_t opts = {0}; + + chassis_options_add(&opts, "admin-address", + 0, 0, OPTION_ARG_STRING, &(config->address), + "listening address:port of the admin-server (default: :4041)", + ""); + + chassis_options_add(&opts, "admin-username", + 0, 0, OPTION_ARG_STRING, &(config->admin_username), + "username to allow to log in", ""); + + chassis_options_add(&opts, "admin-password", + 0, 0, OPTION_ARG_STRING, &(config->admin_password), + "password to allow to log in", ""); + + chassis_options_add(&opts, "admin-allow-ip", + 0, 0, OPTION_ARG_STRING, &(config->allow_ip), + "ip address allowed to connect to admin", ""); + chassis_options_add(&opts, "admin-deny-ip", + 0, 0, OPTION_ARG_STRING, &(config->deny_ip), + "ip address denyed to connect to admin", ""); + + return opts.options; +} + +/* ring buffer from: https://github.com/AndersKaloer/Ring-Buffer */ +#define RING_BUFFER_SIZE 128 /* must be power of 2, !! index [0, 126] !!*/ +#define RING_BUFFER_MASK (RING_BUFFER_SIZE-1) +typedef struct ring_buffer_t { + int head; + int tail; + guint64 buffer[RING_BUFFER_SIZE]; +} ring_buffer_t; + +static void ring_buffer_add(ring_buffer_t *buffer, guint64 data) { + if (((buffer->head - buffer->tail) & RING_BUFFER_MASK) == RING_BUFFER_MASK) + buffer->tail = ((buffer->tail + 1) & RING_BUFFER_MASK); + buffer->buffer[buffer->head] = data; + buffer->head = ((buffer->head + 1) & RING_BUFFER_MASK); +} + +static guint64 ring_buffer_get(ring_buffer_t *buffer, int index) { + if (index >= ((buffer->head - buffer->tail) & RING_BUFFER_MASK)) + return 0; + int data_index = ((buffer->tail + index) & RING_BUFFER_MASK); + return buffer->buffer[data_index]; +} + +static ring_buffer_t g_sql_count = {126, 0}; +static ring_buffer_t g_trx_count = {126, 0}; + +static void calc_qps_average(char *buf, int len) +{ + const int MOST_RECENT = 126; + guint64 c_now = ring_buffer_get(&g_sql_count, MOST_RECENT); + guint64 c_1min = ring_buffer_get(&g_sql_count, MOST_RECENT - 6); + guint64 c_5min = ring_buffer_get(&g_sql_count, MOST_RECENT - 6*5); + guint64 c_15min = ring_buffer_get(&g_sql_count, MOST_RECENT - 6*15); + snprintf(buf, len, "%.2f, %.2f, %.2f", + (c_now-c_1min)/60.0, (c_now-c_5min)/300.0, (c_now-c_15min)/900.0); +} + +static void calc_tps_average(char *buf, int len) +{ + const int MOST_RECENT = 126; + guint64 c_now = ring_buffer_get(&g_trx_count, MOST_RECENT); + guint64 c_1min = ring_buffer_get(&g_trx_count, MOST_RECENT - 6); + guint64 c_5min = ring_buffer_get(&g_trx_count, MOST_RECENT - 6*5); + guint64 c_15min = ring_buffer_get(&g_trx_count, MOST_RECENT - 6*15); + snprintf(buf, len, "%.2f, %.2f, %.2f", + (c_now-c_1min)/60.0, (c_now-c_5min)/300.0, (c_now-c_15min)/900.0); +} + +struct _timer_func_arg_t { + chassis *chas; + struct event *ev; +}; + +/* sample interval is 10-sec, 127 samples takes about 21-min */ +static void sql_stats_sampling_func(int fd, short what, void *arg) +{ + chassis *chas = arg; + + query_stats_t *stats = &(chas->query_stats); + ring_buffer_add(&g_sql_count, stats->client_query.ro + stats->client_query.rw); + ring_buffer_add(&g_trx_count, stats->xa_count); + + static struct timeval ten_sec = {10, 0}; + /* EV_PERSIST not work for libevent1.4, re-activate timer each time */ + chassis_event_add_with_timeout(chas, g_sampling_timer, &ten_sec); +} + +/** + * init the plugin with the parsed config + */ +static int +network_mysqld_admin_plugin_apply_config(chassis *chas, + chassis_plugin_config *config) +{ + network_mysqld_con *con; + network_socket *listen_sock; + + if (!config->address) { + config->address = g_strdup(":4041"); + } else { + chas->proxy_address = config->address; + g_message("set proxy address for chassis:%s", config->address); + } + + if (!config->admin_username) { + g_critical("%s: --admin-username needs to be set", + G_STRLOC); + return -1; + } + if (!config->admin_password) { + g_critical("%s: --admin-password needs to be set", + G_STRLOC); + return -1; + } + if (!g_strcmp0(config->admin_password, "")) { + g_critical("%s: --admin-password cannot be empty", + G_STRLOC); + return -1; + } + g_message("%s:admin-server listening on port", G_STRLOC); + GHashTable *allow_ip_table = NULL; + if (config->allow_ip) { + allow_ip_table = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + char **ip_arr = g_strsplit(config->allow_ip, ",", -1); + int i; + for (i = 0; ip_arr[i]; i++) { + g_hash_table_insert(allow_ip_table, g_strdup(ip_arr[i]), (void *) TRUE); + } + g_strfreev(ip_arr); + } + config->allow_ip_table = allow_ip_table; + + GHashTable *deny_ip_table = NULL; + if (config->deny_ip) { + deny_ip_table = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + char **ip_arr = g_strsplit(config->deny_ip, ",", -1); + int i; + for (i = 0; ip_arr[i]; i++) { + g_hash_table_insert(deny_ip_table, g_strdup(ip_arr[i]), (void *) TRUE); + } + g_strfreev(ip_arr); + } + config->deny_ip_table = deny_ip_table; + + g_message("%s:admin-server listening on port", G_STRLOC); + + /** + * create a connection handle for the listen socket + */ + con = network_mysqld_con_new(); + network_mysqld_add_connection(chas, con, TRUE); + con->config = config; + + config->listen_con = con; + + listen_sock = network_socket_new(); + con->server = listen_sock; + + /* + * set the plugin hooks as we want to apply them to the new + * connections too later + */ + network_mysqld_server_connection_init(con); + + g_message("%s:admin-server listening on port:%s", G_STRLOC, config->address); + /* FIXME: network_socket_set_address() */ + if (0 != network_address_set_address(listen_sock->dst, + config->address)) + { + return -1; + } + + g_message("%s:admin-server listening on port", G_STRLOC); + /* FIXME: network_socket_bind() */ + if (0 != network_socket_bind(listen_sock)) { + return -1; + } + g_message("admin-server listening on port %s", config->address); + + /* set config->has_shard_plugin */ + config->has_shard_plugin = has_shard_plugin(chas->modules); + + /** + * call network_mysqld_con_accept() with this connection when we are done + */ + event_set(&(listen_sock->event), listen_sock->fd, + EV_READ|EV_PERSIST, network_mysqld_con_accept, con); + chassis_event_add(chas, &(listen_sock->event)); + + chassis_config_register_service(chas->config_manager, config->address, "admin"); + + /* EV_PERSIST not work for libevent 1.4 */ + g_sampling_timer = g_new0(struct event, 1); + evtimer_set(g_sampling_timer, sql_stats_sampling_func, chas); + struct timeval ten_sec = {10, 0}; + chassis_event_add_with_timeout(chas, g_sampling_timer, &ten_sec); + return 0; +} + +G_MODULE_EXPORT int plugin_init(chassis_plugin *p) { + p->magic = CHASSIS_PLUGIN_MAGIC; + p->name = g_strdup("admin"); + p->version = g_strdup(PLUGIN_VERSION); + + p->init = network_mysqld_admin_plugin_new; + p->get_options = network_mysqld_admin_plugin_get_options; + p->apply_config = network_mysqld_admin_plugin_apply_config; + p->destroy = network_mysqld_admin_plugin_free; + + /* For allow_ip configs */ + p->allow_ip_get = network_mysqld_admin_plugin_allow_ip_get; + p->allow_ip_add = network_mysqld_admin_plugin_allow_ip_add; + p->allow_ip_del = network_mysqld_admin_plugin_allow_ip_del; + + /* For deny_ip configs */ + p->deny_ip_get = network_mysqld_admin_plugin_deny_ip_get; + p->deny_ip_add = network_mysqld_admin_plugin_deny_ip_add; + p->deny_ip_del = network_mysqld_admin_plugin_deny_ip_del; + + return 0; +} diff --git a/plugins/proxy/CMakeLists.txt b/plugins/proxy/CMakeLists.txt new file mode 100644 index 0000000..a2fc58b --- /dev/null +++ b/plugins/proxy/CMakeLists.txt @@ -0,0 +1,44 @@ +# $%BEGINLICENSE%$ +# Copyright (c) 2009, Oracle and/or its affiliates. All rights reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2 of the +# License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA +# 02110-1301 USA +# +# $%ENDLICENSE%$ +INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/src/) +INCLUDE_DIRECTORIES(${PROJECT_BINARY_DIR}) # for config.h + +include_directories(${PROJECT_SOURCE_DIR}/lib) +include_directories(${PROJECT_BINARY_DIR}/lib) +INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}) + +INCLUDE_DIRECTORIES(${GLIB_INCLUDE_DIRS}) +INCLUDE_DIRECTORIES(${MYSQL_INCLUDE_DIRS}) +INCLUDE_DIRECTORIES(${EVENT_INCLUDE_DIRS}) + +LINK_DIRECTORIES(${GLIB_LIBRARY_DIRS}) +LINK_DIRECTORIES(${LIBINTL_LIBRARY_DIRS}) +LINK_DIRECTORIES(${MYSQL_LIBRARY_DIRS}) + +SET(_plugin_name proxy) +ADD_LIBRARY(${_plugin_name} SHARED "${_plugin_name}-plugin.c") +TARGET_LINK_LIBRARIES(${_plugin_name} mysql-chassis-proxy) + +if(SIMPLE_PARSER) + target_compile_definitions(${_plugin_name} PRIVATE SIMPLE_PARSER=1) +endif(SIMPLE_PARSER) + +CHASSIS_PLUGIN_INSTALL(${_plugin_name}) + diff --git a/plugins/proxy/proxy-plugin.c b/plugins/proxy/proxy-plugin.c new file mode 100644 index 0000000..69ccbc1 --- /dev/null +++ b/plugins/proxy/proxy-plugin.c @@ -0,0 +1,2481 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_SYS_FILIO_H +#include +#endif + +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include + +#ifndef HAVE_SOCKLEN_T +typedef int socklen_t; +#endif + + +#include /** for ER_UNKNOWN_ERROR */ + +#include "network-mysqld.h" +#include "network-mysqld-proto.h" +#include "network-mysqld-packet.h" + +#include "network-conn-pool.h" +#include "network-conn-pool-wrap.h" + +#include "sys-pedantic.h" +#include "network-injection.h" +#include "network-backend.h" +#include "sql-context.h" +#include "sql-filter-variables.h" +#include "glib-ext.h" +#include "chassis-timings.h" +#include "chassis-event.h" +#include "character-set.h" +#include "cetus-util.h" +#include "cetus-users.h" +#include "plugin-common.h" +#include "chassis-options.h" + +#ifndef PLUGIN_VERSION +#ifdef CHASSIS_BUILD_TAG +#define PLUGIN_VERSION CHASSIS_BUILD_TAG +#else +#define PLUGIN_VERSION PACKAGE_VERSION +#endif +#endif + +typedef enum { + PROXY_QUEUE_ADD_PREPEND, + PROXY_QUEUE_ADD_APPEND +} proxy_queue_add_t; + +typedef enum { + INJ_ID_COM_DEFAULT = 1, + INJ_ID_CHANGE_DB, + INJ_ID_COM_QUERY, + INJ_ID_COM_STMT_PREPARE, + INJ_ID_CHAR_SET_CLT, + INJ_ID_CHAR_SET_CONN, + INJ_ID_CHAR_SET_RESULTS, + INJ_ID_SET_NAMES, + INJ_ID_CHANGE_MULTI_STMT, + INJ_ID_CHANGE_SQL_MODE, + INJ_ID_CHANGE_USER, + INJ_ID_RESET_CONNECTION, +} proxy_inj_id_t; + +struct chassis_plugin_config { + /**< listening address of the proxy */ + gchar *address; + + /**< read-write backends */ + gchar **backend_addresses; + + /**< read-only backends */ + gchar **read_only_backend_addresses; + + network_mysqld_con *listen_con; + + /* exposed in the config as double */ + gdouble connect_timeout_dbl; + /* exposed in the config as double */ + gdouble read_timeout_dbl; + /* exposed in the config as double */ + gdouble write_timeout_dbl; + + gchar *allow_ip; + GHashTable *allow_ip_table; + + gchar *deny_ip; + GHashTable *deny_ip_table; + + int read_master_percentage; +}; + +static gboolean proxy_get_backend_ndx(network_mysqld_con *con, + int type, gboolean force_slave); + +/** + * handle event-timeouts on the different states + * + * @note con->state points to the current state + * + */ +NETWORK_MYSQLD_PLUGIN_PROTO(proxy_timeout) { + proxy_plugin_con_t *st = con->plugin_con_state; + if (st == NULL) + return NETWORK_SOCKET_ERROR; + + guint32 diff = time(0) - con->client->last_visit_time; + g_debug("%s, con:%p:call proxy_timeout", G_STRLOC, con); + switch (con->state) { + case ST_READ_QUERY: + if (diff < 8 * HOURS) { + if (con->server && !con->client->is_server_conn_reserved) { + if (network_pool_add_conn(con, 0) != 0) { + g_debug("%s, con:%p:conn to pool failed", G_STRLOC, con); + } else { + g_debug("%s, con:%p:conn returned to pool", G_STRLOC, con); + } + } + } else { + con->prev_state = con->state; + con->state = ST_ERROR; + } + break; + case ST_READ_QUERY_RESULT: + if (diff < 8 * HOURS) { + if (con->server && !con->client->is_server_conn_reserved) { + con->server_to_be_closed = 1; + g_critical("%s, con:%p read query result timeout, sql:%s", + G_STRLOC, con, con->orig_sql->str); + + network_mysqld_con_send_error_full(con->client, + C("Read query result timeout"), + ER_ABORTING_CONNECTION, "29001"); + con->prev_state = con->state; + con->state = ST_SEND_ERROR; + } + } else { + con->prev_state = con->state; + con->state = ST_SEND_ERROR; + } + break; + default: + if (diff >= 8 * HOURS) { + con->prev_state = con->state; + con->state = ST_SEND_ERROR; + } + break; + } + return NETWORK_SOCKET_SUCCESS; +} + +static int store_server_ndx_in_prepared_resp(proxy_resultset_t *res, int index) +{ + network_packet packet; + GString s; + int err = 0; + GString *tmp; + tmp = res->result_queue->head->data; + s.str = tmp->str + 4; /* skip the network-header */ + s.len = tmp->len - 4; + packet.data = &s; + packet.offset = 0; + + err = network_mysqld_proto_change_stmt_id_from_ok_packet(&packet, index); + if (err) { + return -1; + } + + return 0; +} + +static network_mysqld_stmt_ret +proxy_c_read_query_result(network_mysqld_con *con) +{ + network_socket *send_sock = con->client; + network_socket *recv_sock = con->server; + injection *inj = NULL; + proxy_plugin_con_t *st = con->plugin_con_state; + network_mysqld_stmt_ret ret = PROXY_NO_DECISION; + + if (0 == st->injected.queries->length) return PROXY_NO_DECISION; + + inj = g_queue_pop_head(st->injected.queries); + + inj->result_queue = con->server->recv_queue->chunks; + + /* fields, rows */ + proxy_resultset_t *res; + + res = proxy_resultset_new(); + + if (inj->resultset_is_needed && !inj->qstat.binary_encoded) { + res->result_queue = inj->result_queue; + } + res->qstat = inj->qstat; + res->rows = inj->rows; + res->bytes = inj->bytes; + + gboolean is_continue = FALSE; + + g_debug("%s: check inj id:%d", G_STRLOC, inj->id); + + switch(inj->id) + { + case INJ_ID_COM_DEFAULT: + case INJ_ID_COM_QUERY: + case INJ_ID_COM_STMT_PREPARE: + is_continue = TRUE; + break; + case INJ_ID_RESET_CONNECTION: + ret = PROXY_IGNORE_RESULT; + break; + case INJ_ID_CHANGE_USER: + if (con->is_changed_user_failed) { + g_warning("%s: change user failed for user '%s'@'%s'", G_STRLOC, + con->client->response->username->str, con->client->src->name->str); + network_mysqld_con_send_error_full(con->client, + C("Access denied for serving requests"), ER_ACCESS_DENIED_ERROR, "29001"); + + network_queue_clear(recv_sock->recv_queue); + network_queue_clear(con->client->recv_queue); + network_mysqld_queue_reset(con->client); + ret = PROXY_NO_DECISION; + } else { + g_string_assign_len(con->server->response->username, S(con->client->response->username)); + ret = PROXY_IGNORE_RESULT; + } + break; + case INJ_ID_CHANGE_DB: + if (res->qstat.query_status == MYSQLD_PACKET_ERR) { + /* could not change db */ + ret = PROXY_NO_DECISION; + } else { + ret = PROXY_IGNORE_RESULT; + } + break; + default: + ret = PROXY_IGNORE_RESULT; + break; + } + + if (is_continue) { + if (res->qstat.query_status) { + if (con->is_in_transaction) { + con->is_changed_user_when_quit = 0; + } + } else { + g_debug("%s: check is_in_transaction here:%p", G_STRLOC, con); + if (inj->id != INJ_ID_COM_STMT_PREPARE) { + if (res->qstat.server_status & SERVER_STATUS_IN_TRANS) { + con->is_in_transaction = 1; + g_debug("%s: set is_in_transaction true", G_STRLOC); + } else { + con->is_in_transaction = 0; + g_debug("%s: set is_in_transaction false", G_STRLOC); + } + + if (!con->is_in_transaction) { + if (!con->is_auto_commit) { + con->is_in_transaction = 1; + con->client->is_server_conn_reserved = 1; + g_debug("%s: set is_in_transaction true:%p", G_STRLOC, con); + } else { + if (con->is_calc_found_rows) { + con->client->is_server_conn_reserved = 1; + g_debug("%s: set is_server_conn_reserved true for con:%p", + G_STRLOC, con); + } else { + if (!con->is_prepared && !con->is_in_sess_context && + !con->last_warning_met) + { + con->client->is_server_conn_reserved = 0; + g_debug("%s: set is_server_conn_reserved false", G_STRLOC); + } else { + con->client->is_server_conn_reserved = 1; + g_debug("%s: set is_server_conn_reserved true", G_STRLOC); + } + } + } + } else { + con->client->is_server_conn_reserved = 1; + g_debug("%s: is_in_transaction true:%p", G_STRLOC, con); + } + } + } + + g_debug("%s: con multiple_server_mode:%d", G_STRLOC, con->multiple_server_mode); + if (con->multiple_server_mode) { + if (inj->id == INJ_ID_COM_STMT_PREPARE) { + if (st->backend_ndx >= 0 && st->backend_ndx_array != NULL) { + int index = st->backend_ndx_array[st->backend_ndx] - 1; + store_server_ndx_in_prepared_resp(res, index); + } + } + } + } + + switch (ret) { + case PROXY_NO_DECISION: + if (!st->injected.sent_resultset) { + /** + * make sure we send only one result-set per client-query + */ + if (!con->is_changed_user_failed) { + GString *packet; + while ((packet = g_queue_pop_head(recv_sock->recv_queue->chunks)) != NULL) + { + network_mysqld_queue_append_raw(send_sock, + send_sock->send_queue, packet); + } + } + st->injected.sent_resultset++; + break; + } + + st->injected.sent_resultset++; + + /* fall through */ + case PROXY_IGNORE_RESULT: + /* trash the packets for the injection query */ + + if (!con->resultset_is_needed) { + break; + } + network_queue_clear(recv_sock->recv_queue); + break; + default: + network_queue_clear(send_sock->send_queue); + break; + } + + proxy_resultset_free(res); + injection_free(inj); + + return ret; +} + +NETWORK_MYSQLD_PLUGIN_PROTO(proxy_read_auth) { + return do_read_auth(con, con->config->allow_ip_table, con->config->deny_ip_table); +} + + +static int network_mysqld_con_handle_insert_id_response(network_mysqld_con *con, + const char *name, int last_packet_id) +{ + char buffer[16]; + GPtrArray *fields; + GPtrArray *rows; + + fields = g_ptr_array_new_with_free_func((void *) network_mysqld_proto_fielddef_free); + + MYSQL_FIELD *field; + field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup(name); + field->type = MYSQL_TYPE_LONGLONG; + g_ptr_array_add(fields, field); + + rows = g_ptr_array_new_with_free_func((void *) network_mysqld_mysql_field_row_free); + + GPtrArray *row = g_ptr_array_new(); + snprintf(buffer, sizeof(buffer), "%d", last_packet_id); + g_ptr_array_add(row, buffer); + g_ptr_array_add(rows, row); + + network_mysqld_con_send_resultset(con->client, fields, rows); + + g_ptr_array_free(rows, TRUE); + g_ptr_array_free(fields, TRUE); + return 0; +} + +static int +process_non_trans_prepare_stmt(network_mysqld_con *con) +{ + con->is_prepared = 1; + gboolean visit_slave = FALSE; + proxy_plugin_con_t *st = con->plugin_con_state; + sql_context_t *context = st->sql_context; + + gboolean is_orig_ro_server = FALSE; + if (con->server != NULL) { + if (st->backend && st->backend->type == BACKEND_TYPE_RO) { + is_orig_ro_server = TRUE; + } + } + + g_debug("%s: call process_non_trans_prepare_stmt", G_STRLOC); + /* TODO:prepare does not support select for update until now */ + if (!con->srv->master_preferred && context->stmt_type == STMT_SELECT) { + visit_slave = TRUE; + g_debug("%s: set read only true for ps", G_STRLOC); + if (con->prepare_stmt_count == 0 && !is_orig_ro_server) { + g_debug("%s: try to get from slave", G_STRLOC); + /* use ro server */ + int type = BACKEND_TYPE_RO; + if (!proxy_get_backend_ndx(con, type, FALSE)) + { + visit_slave = FALSE; + if (con->server == NULL) { + con->slave_conn_shortaged = 1; + } + } + } else { + if (con->server) { + proxy_plugin_con_t *st = con->plugin_con_state; + if (st->backend->state != BACKEND_STATE_UP && + st->backend->state != BACKEND_STATE_UNKNOWN) + { + visit_slave = FALSE; + g_debug("%s: slave down,move to master", G_STRLOC); + } + } else { + g_warning("%s: original server null", G_STRLOC); + } + } + } + + if (visit_slave == FALSE) { + if (is_orig_ro_server) { + int type = BACKEND_TYPE_RW; + if (!proxy_get_backend_ndx(con, type, FALSE)) { + con->master_conn_shortaged = 1; + g_debug("%s:PROXY_NO_CONNECTION", G_STRLOC); + /* no master connection */ + return PROXY_NO_CONNECTION; + } + } + } + + return PROXY_NO_DECISION; +} + +static int +process_other_set_command(network_mysqld_con *con, const char *key, const char *s, + mysqld_query_attr_t *query_attr) +{ + g_debug("%s: vist process_other_set_command", G_STRLOC); + network_socket *sock = con->client; + size_t s_len = strlen(s); + + if (strcasecmp(key, "character_set_client") == 0) { + g_string_assign_len(sock->charset_client, s, s_len); + query_attr->charset_client_set = 1; + } else if (strcasecmp(key, "character_set_connection") == 0) { + g_string_assign_len(sock->charset_connection, s, s_len); + query_attr->charset_connection_set = 1; + } else if (strcasecmp(key, "character_set_results") == 0) { + g_string_assign_len(sock->charset_results, s, s_len); + query_attr->charset_results_set = 1; + } else if (strcasecmp(key, "sql_mode") == 0) { + g_string_assign_len(sock->sql_mode, s, s_len); + query_attr->sql_mode_set = 1; + } + return 0; +} + +static int +process_set_names(network_mysqld_con *con, char *s, + mysqld_query_attr_t *query_attr) +{ + network_socket *sock = con->client; + size_t s_len = strlen(s); + g_string_assign_len(sock->charset, s, s_len); + g_string_assign_len(sock->charset_client, s, s_len); + g_string_assign_len(sock->charset_connection, s, s_len); + g_string_assign_len(sock->charset_results, s, s_len); + + query_attr->charset_client_set = 1; + query_attr->charset_connection_set = 1; + query_attr->charset_results_set = 1; + query_attr->charset_set = 1; + sock->charset_code = charset_get_number(s); + return 0; +} + +static int process_trans_query(network_mysqld_con *con) +{ + proxy_plugin_con_t *st = con->plugin_con_state; + sql_context_t *context = st->sql_context; + g_debug("%s: visit process_trans_query here:%d", G_STRLOC, context->stmt_type); + switch (context->stmt_type) { + case STMT_SET: + if (sql_context_is_autocommit_off(context)) { + con->is_auto_commit = 0; + g_debug("%s: autocommit off", G_STRLOC); + } else if (sql_context_is_autocommit_on(context)) { + con->is_auto_commit = 1; + con->is_auto_commit_trans_buffered = 0; + g_debug("%s: autocommit on", G_STRLOC); + } + break; + default: + break; + } + + return PROXY_NO_DECISION; +} + +static int process_non_trans_query(network_mysqld_con *con, + sql_context_t *context, mysqld_query_attr_t *query_attr) +{ + gboolean is_orig_ro_server = FALSE; + proxy_plugin_con_t *st = con->plugin_con_state; + + if (con->server != NULL) { + if (st->backend && st->backend->type == BACKEND_TYPE_RO) { + is_orig_ro_server = TRUE; + } + } + + switch (context->stmt_type) { + case STMT_SELECT: { + sql_select_t *select = (sql_select_t *)context->sql_statement; + + con->is_calc_found_rows = (select->flags & SF_CALC_FOUND_ROWS) ? 1 : 0; + g_debug(G_STRLOC ": is_calc_found_rows: %d", con->is_calc_found_rows); + + const char *last_insert_id_name = NULL; + gboolean is_insert_id = FALSE; + sql_expr_list_t *cols = select->columns; + int i; + for (i = 0; cols && i < cols->len; ++i) { + sql_expr_t *col = g_ptr_array_index(cols, i); + if (sql_expr_is_function(col, "LAST_INSERT_ID")) { + is_insert_id = TRUE; + last_insert_id_name = "LAST_INSERT_ID()"; + } else if (sql_expr_is_id(col, "LAST_INSERT_ID")) { + is_insert_id = TRUE; + last_insert_id_name = "@@LAST_INSERT_ID"; + } + } + if (is_insert_id == TRUE) { + g_debug("%s: buffered last insert id:%d", G_STRLOC, + (int) con->last_insert_id); + network_mysqld_con_handle_insert_id_response(con, + last_insert_id_name, con->last_insert_id); + return PROXY_SEND_RESULT; + } + break; + } + case STMT_SET_NAMES:{ + char *charset_name = (char *)context->sql_statement; + process_set_names(con, charset_name, query_attr); + break; + } + case STMT_SET: { + sql_expr_list_t *set_list = context->sql_statement; + if (set_list && set_list->len > 0) { + sql_expr_t *expr = g_ptr_array_index(set_list, 0); + if (expr && expr->op == TK_EQ) { + sql_expr_t *left = expr->left; + sql_expr_t *right = expr->right; + if (!left || !right) + break; + + if (sql_filter_vars_is_silent(left->token_text, right->token_text)) { + network_mysqld_con_send_ok(con->client); + g_message("silent variable: %s\n", left->token_text); + return PROXY_SEND_RESULT; + } + + /* set autocommit = x */ + if (sql_context_is_autocommit_off(context)) { + con->is_auto_commit = 0; + con->is_in_transaction = 1; + con->is_changed_user_when_quit = 0; + con->is_auto_commit_trans_buffered = 1; + g_debug("%s: autocommit off, now in transaction", G_STRLOC); + } else if (sql_context_is_autocommit_on(context)) { + con->is_auto_commit = 1; + con->is_auto_commit_trans_buffered = 0; + g_debug("%s: autocommit on", G_STRLOC); + } else { + /* set charsetxxx = xxx */ + if (left->op == TK_ID && right->token_text) { + process_other_set_command(con, left->token_text, + right->token_text, query_attr); + } + } + } + } + break; + } + case STMT_UPDATE: + case STMT_INSERT: + case STMT_DELETE: + break; + default: + if (con->is_auto_commit) { + if (context->stmt_type == STMT_USE) { + char *dbname = (char *)context->sql_statement; + g_string_assign(con->client->default_db, dbname); + g_debug("%s:set default db:%s for con:%p", + G_STRLOC, con->client->default_db->str, con); + } + } + }/* end switch */ + + if (con->srv->master_preferred || context->rw_flag & CF_WRITE + || !con->is_auto_commit) + { + /* rw operation */ + con->srv->query_stats.client_query.rw++; + if (is_orig_ro_server) { + gboolean success = proxy_get_backend_ndx(con, BACKEND_TYPE_RW, FALSE); + if (!success) { + con->master_conn_shortaged = 1; + g_debug("%s:PROXY_NO_CONNECTION", G_STRLOC); + } + } + } else {/* ro operation */ + con->srv->query_stats.client_query.ro++; + con->is_read_ro_server_allowed = 1; + if (con->srv->query_cache_enabled) { + if (sql_context_is_cacheable(st->sql_context)) { + if (try_to_get_resp_from_query_cache(con)) { + return PROXY_SEND_RESULT; + } + } + } + + if (con->config->read_master_percentage != 100) { + if (!is_orig_ro_server) { + gboolean success = proxy_get_backend_ndx(con, BACKEND_TYPE_RO, FALSE); + if (!success && con->server == NULL) { + con->slave_conn_shortaged = 1; + g_debug("%s:PROXY_NO_CONNECTION", G_STRLOC); + } + } + } else { + if (is_orig_ro_server) { + gboolean success = proxy_get_backend_ndx(con, BACKEND_TYPE_RW, FALSE); + if (!success) { + con->master_conn_shortaged = 1; + g_debug("%s:PROXY_NO_CONNECTION", G_STRLOC); + } + } + } + } + + return PROXY_NO_DECISION; +} + +static void +proxy_inject_packet(network_mysqld_con *con, int type, int resp_type, + GString *payload, gboolean resultset_is_needed) +{ + proxy_plugin_con_t *st = con->plugin_con_state; + GQueue *q = st->injected.queries; + injection *inj = injection_new(resp_type, payload); + inj->resultset_is_needed = resultset_is_needed; + + switch (type) { + case PROXY_QUEUE_ADD_APPEND: + network_injection_queue_append(q, inj); + break; + case PROXY_QUEUE_ADD_PREPEND: + network_injection_queue_prepend(q, inj); + break; + } +} + +static int change_stmt_id(network_mysqld_con *con, uint32_t stmt_id) { + proxy_plugin_con_t *st = con->plugin_con_state; + int index = (stmt_id & 0xffff0000) >> 16; + if (con->servers != NULL) { + if (index >= con->servers->len) { + g_warning("%s:index:%d, stmt id:%d is too big, servers len:%d", + G_STRLOC, index, (int) stmt_id, con->servers->len); + return -1; + } + con->server = g_ptr_array_index(con->servers, index); + st->backend_ndx = st->backend_ndx_array[index] - 1; + g_debug("change conn:%p, server:%p stmt_id:%d, fd:%d, new back ndx:%d", + con, con->server, (int) stmt_id, con->server->fd, st->backend_ndx); + + if (index > 0) { + network_packet packet; + injection *inj; + inj = g_queue_peek_head(st->injected.queries); + if (inj != NULL) { + packet.data = inj->query; + packet.offset = 0; + network_mysqld_proto_change_stmt_id_from_clt_stmt(&packet); + } + } + } + + return 0; +} + +static int change_server_by_rw(network_mysqld_con *con, int backend_ndx) +{ + if (backend_ndx >= 0) { + proxy_plugin_con_t *st = con->plugin_con_state; + int index = st->backend_ndx_array[backend_ndx] - 1; + g_debug("conn:%p, change_server_by_rw,ndx:%d, index:%d, st ndx:%d", + con, backend_ndx, index, st->backend_ndx); + if (con->servers != NULL) { + con->server = g_ptr_array_index(con->servers, index); + st->backend_ndx = backend_ndx; + } + return 0; + } else { + g_critical("%s: get backend ndx failed: %d", + G_STRLOC, backend_ndx); + return -1; + } +} + + +static int +adjust_sql_mode(network_mysqld_con *con, mysqld_query_attr_t *query_attr) +{ + char *clt_sql_mode, *srv_sql_mode; + + if (con->client->sql_mode == NULL) { + clt_sql_mode = ""; + } else { + clt_sql_mode = con->client->sql_mode->str; + } + + if (con->server->sql_mode == NULL) { + srv_sql_mode = ""; + } else { + srv_sql_mode = con->server->sql_mode->str; + } + + if (!query_attr->sql_mode_set) { + if (strcasecmp(clt_sql_mode, srv_sql_mode) != 0) { + if (strcmp(clt_sql_mode, "") != 0) { + GString *packet = g_string_new(NULL); + g_string_append_c(packet, (char) COM_QUERY); + g_string_append(packet, "SET sql_mode='"); + g_string_append_len(packet, con->client->sql_mode->str, + con->client->sql_mode->len); + g_string_append(packet, "'"); + proxy_inject_packet(con, PROXY_QUEUE_ADD_PREPEND, + INJ_ID_CHANGE_SQL_MODE, packet, TRUE); + } else { + GString *packet = g_string_new(NULL); + g_string_append_c(packet, (char) COM_QUERY); + g_string_append(packet, "SET sql_mode=''"); + proxy_inject_packet(con, PROXY_QUEUE_ADD_PREPEND, + INJ_ID_CHANGE_SQL_MODE, packet, TRUE); + } + + g_string_assign_len(con->server->sql_mode, + con->client->sql_mode->str, + con->client->sql_mode->len); + } + } else { + g_string_assign_len(con->server->sql_mode, + con->client->sql_mode->str, + con->client->sql_mode->len); + } + + return 0; +} + +static int +adjust_charset(network_mysqld_con *con, mysqld_query_attr_t *query_attr) +{ + char *charset_str = NULL; + + if (!query_attr->charset_set) { + if (!g_string_equal(con->client->charset, con->server->charset)) { + GString *charset = con->client->charset; + if (charset->len > 0) { + query_attr->charset_reset = 1; + charset_str = charset->str; + g_string_assign_len(con->server->charset_client, + charset->str, charset->len); + g_string_assign_len(con->server->charset_connection, + charset->str, charset->len); + g_string_assign_len(con->server->charset_results, + charset->str, charset->len); + } + g_string_assign_len(con->server->charset, charset->str, + charset->len); + } + } else { + GString *charset = con->client->charset; + g_string_assign_len(con->server->charset, charset->str, + charset->len); + } + + if (!query_attr->charset_client_set) { + if (!g_string_equal(con->client->charset_client, con->server->charset_client)) { + if (con->client->charset_client->len > 0) { + GString *packet = g_string_new(NULL); + g_string_append_c(packet, (char) COM_QUERY); + g_string_append(packet, "SET character_set_client = "); + g_string_append(packet, con->client->charset_client->str); + proxy_inject_packet(con, PROXY_QUEUE_ADD_PREPEND, + INJ_ID_CHAR_SET_CLT, packet, TRUE); + } + GString *charset_client = con->client->charset_client; + g_string_assign_len(con->server->charset_client, + charset_client->str, + charset_client->len); + } + } else { + GString *charset_client = con->client->charset_client; + g_string_assign_len(con->server->charset_client, + charset_client->str, + charset_client->len); + } + + if (!query_attr->charset_connection_set) { + if (!g_string_equal(con->client->charset_connection, + con->server->charset_connection)) { + if (con->client->charset_connection->len > 0) { + GString *packet = g_string_new(NULL); + g_string_append_c(packet, (char) COM_QUERY); + g_string_append(packet, "SET character_set_connection = "); + g_string_append(packet, con->client->charset_connection->str); + proxy_inject_packet(con, PROXY_QUEUE_ADD_PREPEND, + INJ_ID_CHAR_SET_CONN, packet, TRUE); + } + GString *charset_conn = con->client->charset_connection; + g_string_assign_len(con->server->charset_connection, + charset_conn->str, + charset_conn->len); + } + } else { + GString *charset_conn = con->client->charset_connection; + g_string_assign_len(con->server->charset_connection, + charset_conn->str, + charset_conn->len); + } + + if (!query_attr->charset_results_set) { + if (!g_string_equal(con->client->charset_results, con->server->charset_results)) { + if (con->client->charset_results->len > 0) { + GString *packet = g_string_new(NULL); + g_string_append_c(packet, (char) COM_QUERY); + g_string_append(packet, "SET character_set_results = "); + g_string_append(packet, con->client->charset_results->str); + proxy_inject_packet(con, PROXY_QUEUE_ADD_PREPEND, + INJ_ID_CHAR_SET_RESULTS, packet, TRUE); + } else { + GString *packet = g_string_new(NULL); + g_string_append_c(packet, (char) COM_QUERY); + g_string_append(packet, "SET character_set_results = NULL"); + proxy_inject_packet(con, PROXY_QUEUE_ADD_PREPEND, + INJ_ID_CHAR_SET_RESULTS, packet, TRUE); + } + + GString *charset_results = con->client->charset_results; + g_string_assign_len(con->server->charset_results, + charset_results->str, + charset_results->len); + } + } else { + GString *charset_results = con->client->charset_results; + g_string_assign_len(con->server->charset_results, + charset_results->str, + charset_results->len); + } + + if (query_attr->charset_reset) { + GString *packet = g_string_new(NULL); + g_string_append_c(packet, (char) COM_QUERY); + g_string_append(packet, "SET NAMES "); + + if (strcmp(con->client->charset->str, "") == 0) { + g_warning("%s: client charset is empty:%s", + G_STRLOC, con->client->src->name->str); + g_string_append(packet, "''"); + } else { + g_string_append(packet, charset_str); + } + + proxy_inject_packet(con, PROXY_QUEUE_ADD_PREPEND, + INJ_ID_SET_NAMES, packet, TRUE); + } + + return 0; +} + +static int +adjust_default_db(network_mysqld_con *con, enum enum_server_command cmd) +{ + GString *clt_default_db = con->client->default_db; + GString *srv_default_db = con->server->default_db; + + g_debug(G_STRLOC " default client db:%s", clt_default_db ? clt_default_db->str:"null"); + g_debug(G_STRLOC " default server db:%s", srv_default_db ? srv_default_db->str:"null"); + + if (clt_default_db->len > 0) { + if (!g_string_equal(clt_default_db, srv_default_db)) { + GString *packet = g_string_new(NULL); + g_string_append_c(packet, (char) COM_INIT_DB); + g_string_append_len(packet, clt_default_db->str, + clt_default_db->len); + proxy_inject_packet(con, PROXY_QUEUE_ADD_PREPEND, + INJ_ID_CHANGE_DB, packet, TRUE); + g_debug("%s: adjust default db, COM_INIT_DB:%d", G_STRLOC, COM_INIT_DB); + } + } + return 0; +} + +static int +reset_connection(network_mysqld_con *con) +{ + GString *packet = g_string_new(NULL); + g_string_append_c(packet, (char) COM_RESET_CONNECTION); + + proxy_inject_packet(con, PROXY_QUEUE_ADD_PREPEND, + INJ_ID_RESET_CONNECTION, packet, TRUE); + + con->server->is_in_sess_context = 0; + + return 0; +} + + +static int +adjust_user(network_mysqld_con *con) +{ + g_debug("%s: user:%s try to robs conn from user:%s, server:%p for con:%p", G_STRLOC, + con->client->response->username->str, con->server->response->username->str, + con->server, con); + + GString *hashed_password = g_string_new(NULL); + const char *user = con->client->response->username->str; + cetus_users_get_hashed_server_pwd(con->srv->priv->users, user, hashed_password); + if (hashed_password->len == 0) { + g_warning("%s: user:%s hashed password is null", G_STRLOC, user); + g_string_free(hashed_password, TRUE); + return -1; + } else { + mysqld_change_user_packet_t chuser = {0}; + chuser.username = con->client->response->username; + chuser.auth_plugin_data = con->server->challenge->auth_plugin_data; + chuser.hashed_pwd = hashed_password; + chuser.database = con->client->default_db; + chuser.charset = con->client->charset_code; + + GString *payload = g_string_new(NULL); + mysqld_proto_append_change_user_packet(payload, &chuser); + + proxy_inject_packet(con, PROXY_QUEUE_ADD_PREPEND, + INJ_ID_CHANGE_USER, payload, TRUE); + + con->server->is_in_sess_context = 0; + g_string_free(hashed_password, TRUE); + return 0; + } +} + + +static int +adjust_multi_stmt(network_mysqld_con *con, enum enum_server_command cmd) +{ + if (con->client->is_multi_stmt_set != con->server->is_multi_stmt_set) { + GString *packet = g_string_new(NULL); + g_string_append_c(packet, (char) COM_SET_OPTION); + if (con->client->is_multi_stmt_set) { + g_string_append_c(packet, (char) 0); + } else { + g_string_append_c(packet, (char) 1); + } + g_string_append_c(packet, (char) 0); + proxy_inject_packet(con, PROXY_QUEUE_ADD_PREPEND, + INJ_ID_CHANGE_MULTI_STMT, packet, TRUE); + g_debug("%s: adjust multi stmt", G_STRLOC); + con->server->is_multi_stmt_set = con->client->is_multi_stmt_set; + } + + return 0; +} + +gboolean network_mysqld_con_is_trx_feature_changed(network_mysqld_con *con) +{ + proxy_plugin_con_t *st = con->plugin_con_state; + if (!st) { + return FALSE; + } + return st->trx_read_write != TF_READ_WRITE + || st->trx_isolation_level != TF_REPEATABLE_READ; +} + +void network_mysqld_con_reset_trx_feature(network_mysqld_con *con) +{ + proxy_plugin_con_t *st = con->plugin_con_state; + if (st) { + st->trx_read_write = TF_READ_WRITE; + st->trx_isolation_level = TF_REPEATABLE_READ; + } +} + +static int proxy_handle_local_query(network_mysqld_con *con, sql_context_t *context) +{ + g_assert(context->stmt_type == STMT_SELECT); + sql_select_t *select = context->sql_statement; + sql_expr_t *col = g_ptr_array_index(select->columns, 0); + if (sql_expr_is_function(col, "CURRENT_DATE")) { + network_mysqld_con_send_current_date(con->client, "CURRENT_DATE"); + } else if (sql_expr_is_function(col, "CETUS_VERSION")) { + network_mysqld_con_send_cetus_version(con->client); + } + return PROXY_SEND_RESULT; +} + + +static int +process_quit_cmd(network_mysqld_con *con, int backend_ndx, int *disp_flag) +{ + if (backend_ndx < 0 || + (!con->is_in_transaction && + !network_mysqld_con_is_trx_feature_changed(con))) + { + g_debug("%s: quit, backend ndx:%d", G_STRLOC, backend_ndx); + *disp_flag = PROXY_SEND_NONE; + return 0; + } + + if (con->server == NULL) { + g_critical("%s: server is null while ndx:%d", G_STRLOC, backend_ndx); + *disp_flag = PROXY_SEND_NONE; + return 0; + } + + if (con->is_in_transaction || network_mysqld_con_is_trx_feature_changed(con)) { + g_message("%s: change user when COM_QUIT:%d", G_STRLOC, backend_ndx); + int result; + if (con->srv->is_reset_conn_enabled) { + result = reset_connection(con); + } else { + result = adjust_user(con); + } + + if (result != -1) { + con->is_changed_user_when_quit = 1; + network_mysqld_con_reset_trx_feature(con); + *disp_flag = PROXY_SEND_INJECTION; + return 0; + } + } + + g_message("%s: unbelievable for COM_QUIT:%d", G_STRLOC, backend_ndx); + + return 1; +} + + +static int forced_visit(network_mysqld_con *con, proxy_plugin_con_t *st, + sql_context_t *context, int *disp_flag) +{ + if (con->server && con->server->is_in_sess_context) { + con->is_in_sess_context = 1; + *disp_flag = PROXY_NO_DECISION; + g_message("%s:use previous conn for forced_visit", G_STRLOC); + return 1; + } + + int type = (context->rw_flag & CF_FORCE_MASTER) + ? BACKEND_TYPE_RW : BACKEND_TYPE_RO; + + if (type == BACKEND_TYPE_RO) { + con->use_slave_forced = 1; + } + + if (st->backend == NULL || (st->backend && st->backend->type != type)) { + gboolean success = proxy_get_backend_ndx(con, type, + context->rw_flag & CF_FORCE_SLAVE); + if (!success) { + if (type == BACKEND_TYPE_RO) { + if (con->server == NULL) { + con->slave_conn_shortaged = 1; + g_debug("%s:slave_conn_shortaged is true", G_STRLOC); + success = proxy_get_backend_ndx(con, BACKEND_TYPE_RW, FALSE); + } + } + + if (!success) { + con->master_conn_shortaged = 1; + g_debug("%s:PROXY_NO_CONNECTION", G_STRLOC); + *disp_flag = PROXY_NO_CONNECTION; + return 0; + } + } + } + + return 1; +} + +static int process_rw_split(network_mysqld_con *con, proxy_plugin_con_t *st, + sql_context_t *context, mysqld_query_attr_t *query_attr, + int is_under_sess_scope, int command, int *disp_flag) +{ + if (!con->is_in_transaction && !is_under_sess_scope && command == COM_QUERY) + { + /* send all non-transactional SELECTs to a slave */ + int ret = process_non_trans_query(con, context, query_attr); + switch (ret) { + case PROXY_NO_CONNECTION: + *disp_flag = PROXY_NO_CONNECTION; + return 0; + case PROXY_SEND_RESULT: + *disp_flag = PROXY_SEND_RESULT; + return 0; + default: + break; + } + } else { + con->srv->query_stats.client_query.rw++; + if (con->is_in_transaction) { + query_attr->conn_reserved = 1; + if (command == COM_QUERY) { + process_trans_query(con); + } else if (command == COM_STMT_PREPARE) { + con->is_prepared = 1; + } + } else { + if (command == COM_STMT_PREPARE) { + query_attr->conn_reserved = 1; + con->is_prepared = 1; + if (process_non_trans_prepare_stmt(con) == PROXY_NO_CONNECTION) { + *disp_flag = PROXY_NO_CONNECTION; + return 0; + } + } else if (con->prepare_stmt_count > 0 || + !con->is_auto_commit) { + query_attr->conn_reserved = 1; + } else if (con->is_in_sess_context) { + query_attr->conn_reserved = 1; + } + } + } + + return 1; +} + + +static int +process_query_or_stmt_prepare(network_mysqld_con *con, proxy_plugin_con_t *st, + network_packet *packet, mysqld_query_attr_t *query_attr, + int command, int *disp_flag) +{ + network_mysqld_con_reset_query_state(con); + + gsize sql_len = packet->data->len - packet->offset; + network_mysqld_proto_get_gstr_len(packet, sql_len, con->orig_sql); + g_string_append_c(con->orig_sql, '\0');/* 2 more NULL for lexer EOB */ + g_string_append_c(con->orig_sql, '\0'); + + sql_context_t *context = st->sql_context; + sql_context_parse_len(context, con->orig_sql); + if (context->rc == PARSE_SYNTAX_ERR) { + char *msg = context->message; + g_message("%s SQL syntax error: %s. while parsing: %s", + G_STRLOC, msg, con->orig_sql->str); + network_mysqld_con_send_error_full(con->client, msg, strlen(msg), + ER_SYNTAX_ERROR, "42000"); + *disp_flag = PROXY_SEND_RESULT; + return 0; + } else if (context->rc == PARSE_NOT_SUPPORT) { + char *msg = context->message; + g_message("%s SQL unsupported: %s. while parsing: %s for con:%p, clt:%s", + G_STRLOC, msg, con->orig_sql->str, con, con->client->src->name->str); + network_mysqld_con_send_error_full(con->client, msg, strlen(msg), + ER_NOT_SUPPORTED_YET, "42000"); + *disp_flag = PROXY_SEND_RESULT; + return 0; + } else if (context->rc == PARSE_UNRECOGNIZED) { + g_message("%s SQL unrecognized: %s", G_STRLOC, con->orig_sql->str); + } + + /* forbid force write on slave */ + if ((context->rw_flag & CF_FORCE_SLAVE) && (context->rw_flag & CF_WRITE)) { + g_message("%s Comment usage error. SQL: %s", G_STRLOC, con->orig_sql->str); + network_mysqld_con_send_error(con->client, C("Force write on read-only slave")); + *disp_flag = PROXY_SEND_RESULT; + return 0; + } + + if (context->clause_flags & CF_LOCAL_QUERY) { + *disp_flag = proxy_handle_local_query(con, context); + return 0; + } + + /* query statistics */ + query_stats_t *stats = &(con->srv->query_stats); + switch (context->stmt_type) { + case STMT_SELECT:stats->com_select += 1;break; + case STMT_UPDATE:stats->com_update += 1;break; + case STMT_INSERT:stats->com_insert += 1;break; + case STMT_DELETE:stats->com_delete += 1;break; + default:break; + } + + if (context->rw_flag & (CF_FORCE_MASTER|CF_FORCE_SLAVE)) { + if (!forced_visit(con, st, context, disp_flag)) { + return 0; + } + } else { + int is_under_sess_scope = 0; + if (context->stmt_type == STMT_SET_TRANSACTION) { + is_under_sess_scope = 1; + g_debug("%s:call set tran here", G_STRLOC); + sql_set_transaction_t *feat; + feat = (sql_set_transaction_t *) context->sql_statement; + if (feat->scope == SCOPE_SESSION) { + g_message("%s:set session transaction sql:%s for con:%p", + G_STRLOC, con->orig_sql->str, con); + if (feat->rw_feature) { + st->trx_read_write = feat->rw_feature; + } else if (feat->level) { + st->trx_isolation_level = feat->level; + } else { + g_warning("%s:unexpected transaction feature:%s", + G_STRLOC, con->orig_sql->str); + } + } + } + + if (network_mysqld_con_is_trx_feature_changed(con)) { + g_debug("%s:transact feature changed for con:%p", G_STRLOC, con); + if (st->backend && st->backend->type != BACKEND_TYPE_RW) { + gboolean success = proxy_get_backend_ndx(con, BACKEND_TYPE_RW, FALSE); + if (!success) { + con->master_conn_shortaged = 1; + g_debug("%s:PROXY_NO_CONNECTION", G_STRLOC); + *disp_flag = PROXY_NO_CONNECTION; + return 0; + } + } + con->is_in_sess_context = 1; + is_under_sess_scope = 1; + } + + /* rw split */ + if (!process_rw_split(con, st, context, query_attr, is_under_sess_scope, + command, disp_flag)) + { + return 0; + } + } + + return 1; +} + +static network_mysqld_stmt_ret network_read_query(network_mysqld_con *con, + proxy_plugin_con_t *st) +{ + network_packet packet; + GQueue *recv_queue = con->client->recv_queue->chunks; + packet.data = g_queue_peek_head(recv_queue); + packet.offset = 0; + + mysqld_query_attr_t query_attr = {0}; + + con->master_conn_shortaged = 0; + con->slave_conn_shortaged = 0; + con->use_slave_forced = 0; + + network_injection_queue_reset(st->injected.queries); + + int backend_ndx = st->backend_ndx; + + /* check if it is a read request */ + guint8 command; + network_mysqld_proto_skip_network_header(&packet); + if (network_mysqld_proto_get_int8(&packet, &command) != 0) { + network_mysqld_con_send_error(con->client, C("(proxy) unable to retrieve command")); + return PROXY_SEND_RESULT; + } + + con->parse.command = command; + con->is_in_sess_context = 0; + + g_debug("%s: command:%d, backend ndx:%d, con:%p", G_STRLOC, command, backend_ndx, con); + + if (con->is_in_transaction) { + g_debug("%s: still in tran, backend ndx:%d", G_STRLOC, backend_ndx); + } + + int disp_flag = 0; + + switch(con->parse.command) { + case COM_QUIT: + if (!process_quit_cmd(con, backend_ndx, &disp_flag)) { + return disp_flag; + } + break; + case COM_BINLOG_DUMP: + network_mysqld_con_send_error(con->client, + C("(proxy) unable to process binlog dump")); + return PROXY_SEND_RESULT; + case COM_QUERY: + case COM_STMT_PREPARE: + if (!process_query_or_stmt_prepare(con, st, &packet, &query_attr, + command, &disp_flag)) + { + return disp_flag; + } + + break; + case COM_CHANGE_USER: + network_mysqld_con_send_error(con->client, C("(proxy) unable to process change user")); + return PROXY_SEND_RESULT; + default: + break; + } /* end switch */ + + gboolean last_resort = FALSE; + + if (con->server == NULL) { + last_resort = TRUE; + } else { + if (!con->is_prepared && st->backend && st->backend->type == BACKEND_TYPE_RO) { + if (st->backend->state != BACKEND_STATE_UP && + st->backend->state != BACKEND_STATE_UNKNOWN) + { + last_resort = TRUE; + } + } + } + + if (last_resort) { + g_debug("%s: con server is null", G_STRLOC); + /* we try to get a connection */ + if (!proxy_get_backend_ndx(con, BACKEND_TYPE_RW, FALSE)) + { + con->master_conn_shortaged = 1; + g_debug("%s:PROXY_NO_CONNECTION", G_STRLOC); + return PROXY_NO_CONNECTION; + } + } + + if (con->is_in_sess_context) { + con->server->is_in_sess_context = 1; + g_debug("%s: set is_in_sess_context true for con server:%p", G_STRLOC, con->server); + } else { + con->server->is_in_sess_context = 0; + g_debug("%s: set is_in_sess_context false for con server:%p", G_STRLOC, con->server); + } + + network_backend_t *backend = st->backend; + + if (backend == NULL) { + con->master_conn_shortaged = 1; + g_warning("%s:backend is null", G_STRLOC); + return PROXY_NO_CONNECTION; + } + + if (backend->state != BACKEND_STATE_UP && backend->state != BACKEND_STATE_UNKNOWN) { + switch(command) { + case COM_STMT_PREPARE: + case COM_STMT_EXECUTE: + case COM_QUERY: { + network_mysqld_con_send_error_full(con->client, + C("proxy stops serving requests now"), + ER_ABORTING_CONNECTION, "29001"); + g_message(G_STRLOC ": ER_ABORTING_CONNECTION, proxy stops serving requests"); + return PROXY_SEND_RESULT; + } + default: + break; + } + } else { + if (backend->type == BACKEND_TYPE_RW) { + con->srv->query_stats.proxyed_query.rw++; + con->srv->query_stats.server_query_details[st->backend_ndx].ro++; + } else { + con->srv->query_stats.proxyed_query.ro++; + con->srv->query_stats.server_query_details[st->backend_ndx].rw++; + } + } + + /* ! Normal packets also sent out through "injection" interface */ + int payload_len = packet.data->len - NET_HEADER_SIZE; + GString *payload = g_string_sized_new(payload_len); + g_string_append_len(payload, packet.data->str + NET_HEADER_SIZE, payload_len); + switch(command) { + case COM_QUERY: + proxy_inject_packet(con, PROXY_QUEUE_ADD_APPEND, + INJ_ID_COM_QUERY, payload, TRUE); + break; + case COM_STMT_PREPARE: + proxy_inject_packet(con, PROXY_QUEUE_ADD_APPEND, + INJ_ID_COM_STMT_PREPARE, payload, TRUE); + break; + default: + proxy_inject_packet(con, PROXY_QUEUE_ADD_APPEND, + INJ_ID_COM_DEFAULT, payload, TRUE); + } + + if (con->multiple_server_mode) { + query_attr.conn_reserved = 1; + if (command == COM_STMT_EXECUTE || command == COM_STMT_CLOSE) + { + uint32_t stmt_id; + packet.offset = NET_HEADER_SIZE; + + if (network_mysqld_proto_get_stmt_id(&packet, &stmt_id) == 0) { + change_stmt_id(con, stmt_id); + } + } else if (command == COM_QUERY) { + change_server_by_rw(con, backend_ndx); + } + } + + if (query_attr.conn_reserved == 1) { + con->client->is_server_conn_reserved = 1; + g_debug("%s: set is_server_conn_reserved true:%p", G_STRLOC, con); + } + + adjust_sql_mode(con, &query_attr); + + adjust_charset(con, &query_attr); + + if (command != COM_INIT_DB && con->rob_other_conn == 0) { + adjust_default_db(con, command); + } + + if (command != COM_SET_OPTION) { + adjust_multi_stmt(con, command); + } + + if (con->rob_other_conn) { + con->rob_other_conn = 0; + if (adjust_user(con) == -1) { + network_injection_queue_reset(st->injected.queries); + network_mysqld_con_send_error_full(con->client, + C("proxy stops serving requests"), + ER_NO_SUCH_USER, "29001"); + g_message("%s: ER_NO_SUCH_USER, proxy stops serving requests", G_STRLOC); + return PROXY_SEND_RESULT; + } + } + + return PROXY_SEND_INJECTION; +} + +/** + * gets called after a query has been read + * + * @see network_mysqld_con_handle_proxy_stmt + */ +NETWORK_MYSQLD_PLUGIN_PROTO(proxy_read_query) { + GString *packet; + network_socket *recv_sock, *send_sock; + proxy_plugin_con_t *st = con->plugin_con_state; + int proxy_query = 1; + int quietly_quit = 0; + network_mysqld_stmt_ret ret; + + con->resp_too_long = 0; + + if (st == NULL) return NETWORK_SOCKET_ERROR; + + send_sock = NULL; + recv_sock = con->client; + st->injected.sent_resultset = 0; + + int server_attr_changed = 0; + + if (con->server != NULL) { + if (con->last_backend_type != st->backend->type) { + server_attr_changed = 1; + } + } + + if (!server_attr_changed) { + ret = network_read_query(con, st); + + if (con->server != NULL) { + con->last_backend_type = st->backend->type; + } else { + con->last_backend_type = BACKEND_TYPE_UNKNOWN; + } + } else { + g_message("%s server attr changed", G_STRLOC); + network_mysqld_con_send_error(con->client, C("(proxy) unable to continue processing command")); + ret = PROXY_SEND_RESULT; + con->server_to_be_closed = 1; + } + + /** + * if we disconnected in read_query_result() we have no connection open + * when we try to execute the next query + * + * for PROXY_SEND_RESULT we don't need a server + */ + if (ret != PROXY_SEND_NONE && ret != PROXY_SEND_RESULT) { + if (con->server == NULL || ret == PROXY_NO_CONNECTION) { + g_debug("%s: I have no server backend, con:%p for user:%s", + G_STRLOC, con, + con->client->response->username->str); + if (con->master_unavailable) { + return NETWORK_SOCKET_ERROR; + } else { + return NETWORK_SOCKET_ERROR_RETRY; + } + } + } + + + GQueue *chunks; + switch (ret) { + case PROXY_NO_DECISION: + if (st->injected.queries->length) { + g_critical("%s: discarding %d elements from the queue.", + G_STRLOC, + st->injected.queries->length); + network_injection_queue_reset(st->injected.queries); + } + /* fall through */ + case PROXY_SEND_QUERY: + g_message("error: this assumes to dead path"); + send_sock = con->server; + + /* no injection, pass on the chunks as is */ + while ((packet = g_queue_pop_head(recv_sock->recv_queue->chunks))) { + network_mysqld_queue_append_raw(send_sock, + send_sock->send_queue, packet); + } + /* we don't want to buffer the result-set */ + con->resultset_is_needed = FALSE; + + break; + case PROXY_SEND_RESULT: { + gboolean is_first_packet = TRUE; + proxy_query = 0; + + send_sock = con->client; + + chunks = recv_sock->recv_queue->chunks; + /* flush the recv-queue and track the command-states */ + while ((packet = g_queue_pop_head(chunks))) { + if (is_first_packet) { + network_packet p; + + p.data = packet; + p.offset = 0; + + network_mysqld_con_reset_command_response_state(con); + + g_debug("%s: call network_mysqld_con_command_states_init for con:%p", + G_STRLOC, con); + + if (network_mysqld_con_command_states_init(con, &p)) { + g_message("%s: states init failure", G_STRLOC); + } + + is_first_packet = FALSE; + } + + g_string_free(packet, TRUE); + } + + break; + } + case PROXY_SEND_INJECTION: { + injection *inj; + + inj = g_queue_peek_head(st->injected.queries); + con->resultset_is_needed = inj->resultset_is_needed; + + send_sock = con->server; + + network_mysqld_queue_reset(send_sock); + network_mysqld_queue_append(send_sock, + send_sock->send_queue, S(inj->query)); + + network_queue_clear(recv_sock->recv_queue); + break; + } + case PROXY_SEND_NONE: { + quietly_quit = 1; + break; + } + default: + g_error("%s:ret:%d ", G_STRLOC, ret); + } + + if (proxy_query) { + if (quietly_quit) { + con->state = ST_CLIENT_QUIT; + } else { + con->state = ST_SEND_QUERY; + } + } else { + GList *cur; + + /* + * if we don't send the query to the backend, + * it won't be tracked. So track it here instead + * to get the packet tracking right (LOAD DATA LOCAL INFILE, ...) + */ + + for (cur = send_sock->send_queue->chunks->head; cur; cur = cur->next) { + network_packet p; + + p.data = cur->data; + p.offset = 0; + + network_mysqld_proto_get_query_result(&p, con); + } + + con->state = ST_SEND_QUERY_RESULT; + con->resultset_is_finished = TRUE; /* we don't have more too send */ + } + + return NETWORK_SOCKET_SUCCESS; +} + +static gboolean proxy_get_backend_ndx(network_mysqld_con *con, + int type, gboolean force_slave) +{ + proxy_plugin_con_t *st = con->plugin_con_state; + chassis_private *g = con->srv->priv; + + con->max_retry_serv_cnt = 72; + con->master_unavailable = 0; + + int idx; + if (type == BACKEND_TYPE_RO) { + if (force_slave) { + idx = network_backends_get_ro_ndx(g->backends, BACKEND_ALGO_ROUND_ROBIN); + } else { + int x = g_random_int_range(0, 100); + if (x < con->config->read_master_percentage) { + idx = network_backends_get_rw_ndx(g->backends); + } else { + idx = network_backends_get_ro_ndx(g->backends, BACKEND_ALGO_ROUND_ROBIN); + } + g_debug(G_STRLOC "x: %d, read_master_percentage: %d, read: %d\n", + x, con->config->read_master_percentage, idx); + } + } else { /* type == BACKEND_TYPE_RW */ + idx = network_backends_get_rw_ndx(g->backends); + } + + if (idx == -1) { + if (type == BACKEND_TYPE_RW) { + if (con->server) { + g_debug("%s: free server conn to pool:%p", G_STRLOC, con); + if (network_pool_add_conn(con, 0) != 0) { + g_message("%s, con:%p:conn to pool failed", G_STRLOC, con); + } + } + con->master_unavailable = 1; + con->max_retry_serv_cnt = 1; + return FALSE; + } + } else { + if (idx == st->backend_ndx && con->server) { + g_debug("%s: no need to change server:%d", G_STRLOC, st->backend_ndx); + return TRUE; + } + + network_socket *send_sock = network_connection_pool_swap(con, idx); + if (!send_sock) { + return FALSE; + } + con->server = send_sock; + st->backend_ndx = idx; + } + return TRUE; +} + +/** + * decide about the next state after the result-set has been written + * to the client + * + * if we still have data in the queue, back to proxy_send_query() + * otherwise back to proxy_read_query() to pick up a new client query + * + * @note we should only send one result back to the client + */ +NETWORK_MYSQLD_PLUGIN_PROTO(proxy_send_query_result) { + network_socket *send_sock; + injection *inj; + proxy_plugin_con_t *st = con->plugin_con_state; + + if (con->server_to_be_closed) { + if (con->servers != NULL) { + if (con->srv->maintain_close_mode) { + con->state = ST_CLOSE_CLIENT; + g_debug("%s:client needs to closed for con:%p", G_STRLOC, con); + } else { + con->state = ST_READ_QUERY; + } + return NETWORK_SOCKET_SUCCESS; + } else { + GString *packet; + while ((packet = g_queue_pop_head(con->server->recv_queue->chunks))) { + g_string_free(packet, TRUE); + } + + st->backend->connected_clients--; + network_socket_free(con->server); + g_debug("%s:server needs to closed for con:%p", G_STRLOC, con); + con->server = NULL; + st->backend_ndx = -1; + st->backend = NULL; + con->server_to_be_closed = 0; + con->server_closed = 0; + } + } + + if (con->is_changed_user_failed) { + con->is_changed_user_failed = 0; + con->state = ST_ERROR; + return NETWORK_SOCKET_SUCCESS; + } + + send_sock = con->server; + + /* + * if we don't have a backend, don't try to forward queries + */ + if (!send_sock) { + network_injection_queue_reset(st->injected.queries); + } + + if (st->injected.queries->length == 0) { + /* we have nothing more to send, let's see what the next state is */ + + con->state = ST_READ_QUERY; + + if (con->srv->maintain_close_mode) { + if (!con->is_in_transaction && !con->is_in_sess_context) { + con->state = ST_CLOSE_CLIENT; + g_debug("%s:client needs to closed for con:%p", G_STRLOC, con); + } + } + return NETWORK_SOCKET_SUCCESS; + } + + /* looks like we still have queries in the queue, + * push the next one + */ + inj = g_queue_peek_head(st->injected.queries); + con->resultset_is_needed = inj->resultset_is_needed; + + if (!inj->resultset_is_needed && st->injected.sent_resultset > 0) { + /* + * we already sent a resultset to the client and the next query + * wants to forward it's result-set too, that can't work + */ + g_critical("%s: append() mul-queries without true rs set.", + G_STRLOC); + + return NETWORK_SOCKET_ERROR; + } + + g_assert(inj); + g_assert(send_sock); + + network_mysqld_queue_reset(send_sock); + network_mysqld_queue_append(send_sock, send_sock->send_queue, + S(inj->query)); + + g_debug("%s: call reset_command_response_state for con:%p", G_STRLOC, con); + network_mysqld_con_reset_command_response_state(con); + + con->state = ST_SEND_QUERY; + + return NETWORK_SOCKET_SUCCESS; +} + +/** + * handle the query-result we received from the server + * + * - decode the result-set to track if we are finished already + * - handles BUG#25371 if requested + * - if the packet is finished, + * + * @see network_mysqld_con_handle_proxy_resultset + */ +NETWORK_MYSQLD_PLUGIN_PROTO(proxy_read_query_result) { + int is_finished = 0; + network_packet packet; + network_socket *recv_sock, *send_sock; + proxy_plugin_con_t *st = con->plugin_con_state; + injection *inj = NULL; + + recv_sock = con->server; + send_sock = con->client; + + /* check if the last packet is valid */ + packet.data = g_queue_peek_tail(recv_sock->recv_queue->chunks); + packet.offset = 0; + + if (0 != st->injected.queries->length) { + inj = g_queue_peek_head(st->injected.queries); + } + + g_debug("%s: here we visit network_mysqld_proto_get_query_result for con:%p", + G_STRLOC, con); + + if (con->resp_too_long) { + is_finished = 1; + } else { + is_finished = network_mysqld_proto_get_query_result(&packet, con); + } + + if (!con->resp_too_long && is_finished == 1) { + /* TODO if attribute adjustment fails, then the backend connection should not be put to pool */ + switch(con->parse.command) { + case COM_QUERY: + case COM_PROCESS_INFO: + case COM_STMT_EXECUTE: { + g_debug("%s: read finished: %p", G_STRLOC, con); + network_mysqld_com_query_result_t *query = con->parse.data; + if (query && query->query_status == MYSQLD_PACKET_ERR) { + int offset = packet.offset; + packet.offset = NET_HEADER_SIZE; + network_mysqld_err_packet_t *err_packet; + err_packet = network_mysqld_err_packet_new(); + if (!network_mysqld_proto_get_err_packet(&packet, err_packet)) { + g_message("%s:clt:%s,src:%s,dst:%s,db:%s,%s,error code:%d, \ + errmsg:%s,sqlstate:%s", + G_STRLOC, con->client->src->name->str, + con->server->src->name->str, + con->server->dst->name->str, + con->server->default_db->str, + con->orig_sql->str, + (int) err_packet->errcode, + err_packet->errmsg->str, + err_packet->sqlstate->str); + } else { + g_message("%s:clt:%s,src:%s,dst:%s,db:%s, %s", + G_STRLOC, con->client->src->name->str, + con->server->src->name->str, + con->server->dst->name->str, con->server->default_db->str, + con->orig_sql->str); + } + network_mysqld_err_packet_free(err_packet); + packet.offset = offset; + } + break; + } + case COM_STMT_PREPARE: { + network_mysqld_com_stmt_prep_result_t *r = con->parse.data; + if (r->status == MYSQLD_PACKET_OK) { + con->prepare_stmt_count++; + } + break; + } + case COM_INIT_DB: + break; + case COM_CHANGE_USER: + break; + default: + break; + } + } else if (is_finished == -1) { + g_debug("%s: is_finished -1: %p", G_STRLOC, con); + /* something happened, let's get out of here */ + return NETWORK_SOCKET_ERROR; + } + + con->resultset_is_finished = is_finished; + + /* copy the packet over to the send-queue if we don't need it */ + if (!con->resultset_is_needed) { + network_mysqld_queue_append_raw(send_sock, send_sock->send_queue, + g_queue_pop_tail(recv_sock->recv_queue->chunks)); + } + + if (is_finished) { + g_debug("%s: resultset_is_finished finished: %p", G_STRLOC, con); + network_mysqld_stmt_ret ret; + + /** + * the resultset handler might decide to trash the send-queue + * + */ + + if (inj) { + switch (con->parse.command) { + case COM_QUERY: + case COM_STMT_EXECUTE: + { + network_mysqld_com_query_result_t *com_query = con->parse.data; + + inj->bytes = com_query->bytes; + inj->rows = com_query->rows; + inj->qstat.was_resultset = com_query->was_resultset; + inj->qstat.binary_encoded = com_query->binary_encoded; + + /* INSERTs have a affected_rows */ + if (!com_query->was_resultset) { + inj->qstat.affected_rows = com_query->affected_rows; + inj->qstat.insert_id = com_query->insert_id; + if (inj->qstat.insert_id > 0) { + con->last_insert_id = inj->qstat.insert_id; + g_debug("%s: last insert id:%d for con:%p", + G_STRLOC, (int) con->last_insert_id, con); + } + } + inj->qstat.server_status = com_query->server_status; + inj->qstat.warning_count = com_query->warning_count; + inj->qstat.query_status = com_query->query_status; + g_debug("%s: server status, got: %d, con:%p", + G_STRLOC, + com_query->server_status, con); + break; + } + case COM_INIT_DB: + break; + case COM_CHANGE_USER: + break; + default: + g_debug("%s: no chance to get server status", + G_STRLOC); + } + } + + /* reset the packet-id checks as the server-side is finished */ + network_mysqld_queue_reset(recv_sock); + + ret = proxy_c_read_query_result(con); + + if (PROXY_IGNORE_RESULT != ret) { + /* reset the packet-id checks, if we sent something to the client */ + network_mysqld_queue_reset(send_sock); + } + + /** + * if the send-queue is empty, we have nothing to send + * and can read the next query */ + if (send_sock->send_queue->chunks) { + con->state = ST_SEND_QUERY_RESULT; + } else { + /* + * we already forwarded the resultset, + * no way someone has flushed the resultset-queue + */ + g_assert_cmpint(con->resultset_is_needed, ==, 1); + + con->state = ST_READ_QUERY; + } + } + + return NETWORK_SOCKET_SUCCESS; +} + +/** + * connect to a backend + * + * @return + * NETWORK_SOCKET_SUCCESS - connected successfully + * NETWORK_SOCKET_ERROR_RETRY - connecting backend failed, + * call again to connect to another backend + * NETWORK_SOCKET_ERROR - no backends available, + * adds a ERR packet to the client queue + */ +NETWORK_MYSQLD_PLUGIN_PROTO(proxy_connect_server) { + proxy_plugin_con_t *st = con->plugin_con_state; + return do_connect_cetus(con, &st->backend, &st->backend_ndx); +} + +static proxy_plugin_con_t *proxy_plugin_con_new() +{ + proxy_plugin_con_t *st; + + st = g_new0(proxy_plugin_con_t, 1); + + st->injected.queries = network_injection_queue_new(); + return st; +} + +NETWORK_MYSQLD_PLUGIN_PROTO(proxy_init) { + proxy_plugin_con_t *st = con->plugin_con_state; + chassis_plugin_config *config = con->config; + + g_assert(con->plugin_con_state == NULL); + + st = proxy_plugin_con_new(); + + /* TODO: this should inside "st"_new, but now "st" shared by many plugins */ + st->sql_context = g_new0(sql_context_t, 1); + sql_context_init(st->sql_context); + st->trx_read_write = TF_READ_WRITE; + st->trx_isolation_level = TF_REPEATABLE_READ; + + con->plugin_con_state = st; + + con->state = ST_CONNECT_SERVER; + + /* set the connection specific timeouts + * + * TODO: expose these settings at runtime + */ + if (config->connect_timeout_dbl >= 0) { + chassis_timeval_from_double(&con->connect_timeout, config->connect_timeout_dbl); + } + if (config->read_timeout_dbl >= 0) { + chassis_timeval_from_double(&con->read_timeout, config->read_timeout_dbl); + } + if (config->write_timeout_dbl >= 0) { + chassis_timeval_from_double(&con->write_timeout, config->write_timeout_dbl); + } + + return NETWORK_SOCKET_SUCCESS; +} + +static network_mysqld_stmt_ret +proxy_c_disconnect_client(network_mysqld_con *con) +{ + g_debug("%s: call proxy_c_disconnect_client: %p", G_STRLOC, con); + gboolean client_abnormal_close = FALSE; + if (con->state == ST_READ_QUERY_RESULT) { + client_abnormal_close = TRUE; + g_debug("%s: set client_abnormal_close true: %p", G_STRLOC, con); + } else { + if (con->prev_state > ST_READ_QUERY) { + client_abnormal_close = TRUE; + g_debug("%s: set client_abnormal_close true: %p", G_STRLOC, con); + } + } + + if (client_abnormal_close) { + con->server_to_be_closed = 1; + } else { + if (con->is_changed_user_when_quit) { + con->is_in_transaction = 0; + con->is_auto_commit = 1; + con->is_start_tran_command = 0; + g_debug("%s: auto commit true", G_STRLOC); + } + + if (con->is_in_transaction || !con->is_auto_commit || con->is_in_sess_context) + { + con->server_to_be_closed = 1; + } else { + + if (con->server != NULL) { + if (network_pool_add_conn(con, 0) != 0) { + g_debug("%s, con:%p:conn returned to pool failed", + G_STRLOC, con); + } + } + } + } + + return PROXY_NO_DECISION; +} + +static void mysqld_con_reserved_connections_free(network_mysqld_con *con) +{ + proxy_plugin_con_t *st = con->plugin_con_state; + chassis *srv = con->srv; + chassis_private *g = srv->priv; + if (st->backend_ndx_array) { + int i, checked = 0; + for (i = 0; i < MAX_SERVER_NUM; i++) { + if (st->backend_ndx_array[i] == 0) { + continue; + } + /* rw-edition: after filtering, now [i] is a valid backend index */ + int index = st->backend_ndx_array[i] - 1; + network_socket *server = g_ptr_array_index(con->servers, index); + network_backend_t *backend = network_backends_get(g->backends, i); + + CHECK_PENDING_EVENT(&(server->event)); + + network_socket_free(server); + backend->connected_clients--; + g_debug("%s: connected_clients sub, con:%p, now clients:%d", G_STRLOC, + con, backend->connected_clients); + checked++; + + if (checked >= con->servers->len) { + g_ptr_array_free(con->servers, TRUE); + con->servers = NULL; + break; + } + } + + if (st->backend_ndx_array) { + g_free(st->backend_ndx_array); + st->backend_ndx_array = NULL; + } + } +} + +static void +proxy_plugin_con_free(network_mysqld_con *con, proxy_plugin_con_t *st) +{ + g_debug("%s: call proxy_plugin_con_free con:%p", G_STRLOC, con); + + if (!st) return; + + network_injection_queue_free(st->injected.queries); + + /* If con still has server list, then all are closed */ + if (con->servers != NULL) { + mysqld_con_reserved_connections_free(con); + con->server = NULL; + } else { + if (con->server) { + st->backend->connected_clients--; + g_debug("%s: connected_clients sub, con:%p, now clients:%d", G_STRLOC, + con, st->backend->connected_clients); + } + } + + if (st->backend_ndx_array) { + g_warning("%s: st backend_ndx_array is not nill for con:%p", G_STRLOC, con); + } + + g_free(st); +} + + +/** + * cleanup the proxy specific data on the current connection + * + * move the server connection into the connection pool in case it is a + * good client-side close + * + * @return NETWORK_SOCKET_SUCCESS + * @see plugin_call_cleanup + */ +NETWORK_MYSQLD_PLUGIN_PROTO(proxy_disconnect_client) { + proxy_plugin_con_t *st = con->plugin_con_state; + + if (st == NULL) return NETWORK_SOCKET_SUCCESS; + + network_mysqld_stmt_ret ret; + ret = proxy_c_disconnect_client(con); + switch (ret) { + case PROXY_NO_DECISION: + /* just go on */ + + break; + case PROXY_IGNORE_RESULT: + break; + default: + g_error("%s: ... ", G_STRLOC); + break; + } + + if (con->server && !con->server_to_be_closed) { + if (con->state == ST_CLOSE_CLIENT || con->prev_state <= ST_READ_QUERY) + { + /* move the connection to the connection pool + * + * this disconnects con->server and safes it + * from getting free()ed later + */ + + if (network_pool_add_conn(con, 0) != 0) { + g_message("%s, con:%p:server conn returned to pool failed", + G_STRLOC, con); + } + } + } + + if (con->servers != NULL) { + g_critical("%s: conn server list is not freed:%p", G_STRLOC, con); + } + + network_mysqld_con_reset_query_state(con); + + /* TODO: this should inside "st"_free, but now "st" shared by many plugins */ + if (st->sql_context) { + sql_context_destroy(st->sql_context); + g_free(st->sql_context); + st->sql_context = NULL; + } + + proxy_plugin_con_free(con, st); + + con->plugin_con_state = NULL; + + g_debug("%s: set plugin_con_state null:%p", G_STRLOC, con); + + /** + * walk all pools and clean them up + */ + + return NETWORK_SOCKET_SUCCESS; +} + + +int network_mysqld_proxy_connection_init(network_mysqld_con *con) { + con->plugins.con_init = proxy_init; + con->plugins.con_connect_server = proxy_connect_server; + con->plugins.con_read_handshake = NULL; + con->plugins.con_read_auth = proxy_read_auth; + con->plugins.con_read_auth_result = NULL; + con->plugins.con_read_auth_old_password = NULL; + con->plugins.con_read_query = proxy_read_query; + con->plugins.con_get_server_conn_list = NULL; + con->plugins.con_read_query_result = proxy_read_query_result; + con->plugins.con_send_query_result = proxy_send_query_result; + con->plugins.con_cleanup = proxy_disconnect_client; + con->plugins.con_timeout = proxy_timeout; + + return 0; +} + +/** + * free the global scope which is shared between all connections + * + * make sure that is called after all connections are closed + */ +void network_mysqld_proxy_free(network_mysqld_con G_GNUC_UNUSED *con) { +} + +chassis_plugin_config *network_mysqld_proxy_plugin_new(void) { +#ifndef SIMPLE_PARSER + g_critical("try loading proxy-plugin.so from shard-edition, exit"); + exit(1); +#endif + chassis_plugin_config *config; + + config = g_new0(chassis_plugin_config, 1); + + /* use negative values as defaults to make them ignored */ + config->connect_timeout_dbl = -1.0; + config->read_timeout_dbl = -1.0; + config->write_timeout_dbl = -1.0; + + return config; +} + +void network_mysqld_proxy_plugin_free(chassis *chas, chassis_plugin_config *config) { + + g_strfreev(config->backend_addresses); + g_strfreev(config->read_only_backend_addresses); + + if (config->address) { + /* free the global scope */ + network_mysqld_proxy_free(NULL); + chassis_config_unregister_service(chas->config_manager, config->address); + g_free(config->address); + } + sql_filter_vars_destroy(); + + g_free(config); +} + +/** + * plugin options + */ +static GList * +network_mysqld_proxy_plugin_get_options(chassis_plugin_config *config) +{ + chassis_options_t opts = {0}; + + chassis_options_add(&opts, "proxy-address", + 'P', 0, OPTION_ARG_STRING, &(config->address), + "listening address:port of the proxy-server (default: :4040)", + ""); + + chassis_options_add(&opts, "proxy-read-only-backend-addresses", + 'r', 0, OPTION_ARG_STRING_ARRAY, &(config->read_only_backend_addresses), + "address:port of the remote slave-server (default: not set)", + ""); + + chassis_options_add(&opts, "proxy-backend-addresses", + 'b', 0, OPTION_ARG_STRING_ARRAY, &(config->backend_addresses), + "address:port of the remote backend-servers (default: 127.0.0.1:3306)", + ""); + + chassis_options_add(&opts, "proxy-connect-timeout", + 0, 0, OPTION_ARG_DOUBLE, &(config->connect_timeout_dbl), + "connect timeout in seconds (default: 2.0 seconds)", + NULL); + + chassis_options_add(&opts, "proxy-read-timeout", + 0, 0, OPTION_ARG_DOUBLE, &(config->read_timeout_dbl), + "read timeout in seconds (default: 10 minuates)", NULL); + + chassis_options_add(&opts, "proxy-write-timeout", + 0, 0, OPTION_ARG_DOUBLE, &(config->write_timeout_dbl), + "write timeout in seconds (default: 10 minuates)", NULL); + + chassis_options_add(&opts, "proxy-allow-ip", + 0, 0, OPTION_ARG_STRING, &(config->allow_ip), + "allow user@IP for proxy permission", NULL); + + chassis_options_add(&opts, "proxy-deny-ip", + 0, 0, OPTION_ARG_STRING, &(config->deny_ip), + "deny user@IP for proxy permission", NULL); + + chassis_options_add(&opts, "read-master-percentage", + 0, 0, OPTION_ARG_INT, &(config->read_master_percentage), + "range [0, 100]", NULL); + + return opts.options; +} + +/** + * init the plugin with the parsed config + */ +int network_mysqld_proxy_plugin_apply_config(chassis *chas, + chassis_plugin_config *config) +{ + network_mysqld_con *con; + network_socket *listen_sock; + chassis_private *g = chas->priv; + + if (!config->address) config->address = g_strdup(":4040"); + if (!config->backend_addresses) { + config->backend_addresses = g_new0(char *, 2); + config->backend_addresses[0] = g_strdup("127.0.0.1:3306"); + config->backend_addresses[1] = NULL; + } + + /* set allow_ip_table */ + GHashTable *allow_ip_table = NULL; + if (config->allow_ip) { + allow_ip_table = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + char **ip_arr = g_strsplit(config->allow_ip, ",", -1); + guint j; + for (j = 0; ip_arr[j]; j++) { + g_hash_table_insert(allow_ip_table, g_strdup(ip_arr[j]), (void *) TRUE); + } + g_strfreev(ip_arr); + } + config->allow_ip_table = allow_ip_table; + + /* set deny_ip_table */ + GHashTable *deny_ip_table = NULL; + if (config->deny_ip) { + deny_ip_table = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + char **ip_arr = g_strsplit(config->deny_ip, ",", -1); + guint j; + for (j = 0; ip_arr[j]; j++) { + g_hash_table_insert(deny_ip_table, g_strdup(ip_arr[j]), (void *) TRUE); + } + g_strfreev(ip_arr); + } + config->deny_ip_table = deny_ip_table; + + /** + * create a connection handle for the listen socket + */ + con = network_mysqld_con_new(); + network_mysqld_add_connection(chas, con, TRUE); + con->config = config; + + config->listen_con = con; + + listen_sock = network_socket_new(); + con->server = listen_sock; + + /* + * set the plugin hooks as we want to apply them + * to the new connections too later + */ + network_mysqld_proxy_connection_init(con); + + if (network_address_set_address(listen_sock->dst, config->address)) { + return -1; + } + + if (network_socket_bind(listen_sock)) { + return -1; + } + g_message("proxy listening on port %s, con:%p", config->address, con); + + plugin_add_backends(chas, config->backend_addresses, + config->read_only_backend_addresses); + + + /** + * call network_mysqld_con_accept() with this connection when we are done + */ + + event_set(&(listen_sock->event), listen_sock->fd, + EV_READ|EV_PERSIST, network_mysqld_con_accept, con); + event_base_set(chas->event_base, &(listen_sock->event)); + event_add(&(listen_sock->event), NULL); + g_debug("%s:listen sock, ev:%p",G_STRLOC, (&listen_sock->event)); + + if (network_backends_load_config(g->backends, chas) != -1) { + network_connection_pool_create_conns(chas); + } + chassis_config_register_service(chas->config_manager, config->address, "proxy"); + + sql_filter_vars_load_default_rules(); + char *variable_conf = g_build_filename(chas->conf_dir, "variables.json", NULL); + if (g_file_test(variable_conf, G_FILE_TEST_IS_REGULAR)) { + g_message("reading variable rules from %s", variable_conf); + gboolean ok = sql_filter_vars_load_rules(variable_conf); + if (!ok) g_warning("variable rule load error"); + } + g_free(variable_conf); + + return 0; +} + +GList *network_mysqld_proxy_plugin_allow_ip_get(chassis_plugin_config *config) { + if (config && config->allow_ip_table) { + return g_hash_table_get_keys(config->allow_ip_table); + } + return NULL; +} + +GList *network_mysqld_proxy_plugin_deny_ip_get(chassis_plugin_config *config) { + if (config && config->deny_ip_table) { + return g_hash_table_get_keys(config->deny_ip_table); + } + return NULL; +} + +gboolean network_mysqld_proxy_plugin_allow_ip_add(chassis_plugin_config *config, char *addr) { + if (!config || !addr) return FALSE; + if (!config->allow_ip_table) { + config->allow_ip_table = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + } + gboolean success = FALSE; + if (!g_hash_table_lookup(config->allow_ip_table, addr)) { + g_hash_table_insert(config->allow_ip_table, g_strdup(addr), (void *) TRUE); + success = TRUE; + } + return success; +} + +gboolean network_mysqld_proxy_plugin_deny_ip_add(chassis_plugin_config *config, char *addr) { + if (!config || !addr) return FALSE; + if (!config->deny_ip_table) { + config->deny_ip_table = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + } + gboolean success = FALSE; + if (!g_hash_table_lookup(config->deny_ip_table, addr)) { + g_hash_table_insert(config->deny_ip_table, g_strdup(addr), (void *) TRUE); + success = TRUE; + } + return success; +} + +gboolean network_mysqld_proxy_plugin_allow_ip_del(chassis_plugin_config *config, char *addr) { + if (!config || !addr || !config->allow_ip_table) return FALSE; + return g_hash_table_remove(config->allow_ip_table, addr); +} + +gboolean network_mysqld_proxy_plugin_deny_ip_del(chassis_plugin_config *config, char *addr) { + if (!config || !addr || !config->deny_ip_table) return FALSE; + return g_hash_table_remove(config->deny_ip_table, addr); +} + +G_MODULE_EXPORT int plugin_init(chassis_plugin *p) { + p->magic = CHASSIS_PLUGIN_MAGIC; + p->name = g_strdup("proxy"); + p->version = g_strdup(PLUGIN_VERSION); + + p->init = network_mysqld_proxy_plugin_new; + p->get_options = network_mysqld_proxy_plugin_get_options; + p->apply_config = network_mysqld_proxy_plugin_apply_config; + p->destroy = network_mysqld_proxy_plugin_free; + + /* For allow_ip configs */ + p->allow_ip_get = network_mysqld_proxy_plugin_allow_ip_get; + p->allow_ip_add = network_mysqld_proxy_plugin_allow_ip_add; + p->allow_ip_del = network_mysqld_proxy_plugin_allow_ip_del; + + /* For deny_ip configs */ + p->deny_ip_get = network_mysqld_proxy_plugin_deny_ip_get; + p->deny_ip_add = network_mysqld_proxy_plugin_deny_ip_add; + p->deny_ip_del = network_mysqld_proxy_plugin_deny_ip_del; + + return 0; +} diff --git a/plugins/shard/CMakeLists.txt b/plugins/shard/CMakeLists.txt new file mode 100644 index 0000000..504486a --- /dev/null +++ b/plugins/shard/CMakeLists.txt @@ -0,0 +1,49 @@ +# $%BEGINLICENSE%$ +# Copyright (c) 2009, Oracle and/or its affiliates. All rights reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2 of the +# License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA +# 02110-1301 USA +# +# $%ENDLICENSE%$ +INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/src/) +INCLUDE_DIRECTORIES(${PROJECT_BINARY_DIR}) # for config.h + +include_directories(${PROJECT_SOURCE_DIR}/lib) +include_directories(${PROJECT_BINARY_DIR}/lib) +INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}) + +INCLUDE_DIRECTORIES(${GLIB_INCLUDE_DIRS}) +INCLUDE_DIRECTORIES(${MYSQL_INCLUDE_DIRS}) +INCLUDE_DIRECTORIES(${EVENT_INCLUDE_DIRS}) + +LINK_DIRECTORIES(${GLIB_LIBRARY_DIRS}) +LINK_DIRECTORIES(${LIBINTL_LIBRARY_DIRS}) +LINK_DIRECTORIES(${MYSQL_LIBRARY_DIRS}) + +SET(_plugin_name shard) + +set(SHARD_SOURCES + ${_plugin_name}-plugin.c + sharding-parser.c + ) +ADD_LIBRARY(${_plugin_name} SHARED ${SHARD_SOURCES}) +TARGET_LINK_LIBRARIES(${_plugin_name} mysql-chassis-proxy) + +if(SIMPLE_PARSER) + target_compile_definitions(${_plugin_name} PRIVATE SIMPLE_PARSER=1) +endif(SIMPLE_PARSER) + +CHASSIS_PLUGIN_INSTALL(${_plugin_name}) + diff --git a/plugins/shard/shard-plugin.c b/plugins/shard/shard-plugin.c new file mode 100644 index 0000000..b41f8b7 --- /dev/null +++ b/plugins/shard/shard-plugin.c @@ -0,0 +1,2357 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include /** for ER_UNKNOWN_ERROR */ + +#include "cetus-users.h" +#include "cetus-util.h" +#include "character-set.h" +#include "chassis-event.h" +#include "chassis-options.h" +#include "cetus-monitor.h" +#include "glib-ext.h" +#include "network-backend.h" +#include "network-conn-pool.h" +#include "network-conn-pool-wrap.h" +#include "plugin-common.h" +#include "network-mysqld-packet.h" +#include "network-mysqld-proto.h" +#include "network-mysqld.h" +#include "server-session.h" +#include "shard-plugin-con.h" +#include "sharding-config.h" +#include "sharding-parser.h" +#include "sharding-query-plan.h" +#include "sql-filter-variables.h" +#include "cetus-log.h" + +#ifdef NETWORK_DEBUG_TRACE_STATE_CHANGES +#include "cetus-query-queue.h" +#endif + +#ifndef PLUGIN_VERSION +#ifdef CHASSIS_BUILD_TAG +#define PLUGIN_VERSION CHASSIS_BUILD_TAG +#else +#define PLUGIN_VERSION PACKAGE_VERSION +#endif +#endif + +#define XA_LOG_BUF_LEN 2048 + +struct chassis_plugin_config { + /**< listening address of the proxy */ + gchar *address; + + /**< read-write backends */ + gchar **backend_addresses; + + /**< read-only backends */ + gchar **read_only_backend_addresses; + + network_mysqld_con *listen_con; + + /* exposed in the config as double */ + gdouble connect_timeout_dbl; + /* exposed in the config as double */ + gdouble read_timeout_dbl; + /* exposed in the config as double */ + gdouble write_timeout_dbl; + + gchar *allow_ip; + GHashTable *allow_ip_table; + + gchar *deny_ip; + GHashTable *deny_ip_table; +}; + +/** + * handle event-timeouts on the different states + * + * @note con->state points to the current state + * + */ +NETWORK_MYSQLD_PLUGIN_PROTO(proxy_timeout) { + int diff; + shard_plugin_con_t *st = con->plugin_con_state; + + if (st == NULL) return NETWORK_SOCKET_ERROR; + + g_debug("%s, con:%p:call proxy_timeout, state:%d", G_STRLOC, con, con->state); + switch (con->state) { + case ST_READ_M_QUERY_RESULT: + case ST_READ_QUERY_RESULT: + g_warning("%s:read query result timeout", G_STRLOC); + if (con->dist_tran) { + if (con->dist_tran_state > NEXT_ST_XA_CANDIDATE_OVER) { + g_critical("%s:EV_TIMEOUT, phase two, not recv response:%p", + G_STRLOC, con); + } else { + con->dist_tran_failed = 1; + g_critical("%s:xa tran failed here:%p, xa state:%d, xid:%s", + G_STRLOC, con, con->dist_tran_state, con->xid_str); + } + } + break; + default: + diff = time(0) - con->client->last_visit_time; + if (diff < 8 * HOURS) { + if (!con->client->is_server_conn_reserved) { + g_debug("%s, is_server_conn_reserved is false", G_STRLOC); + if (con->servers && con->servers->len > 0) { + g_debug("%s, server conns returned to pool", G_STRLOC); + proxy_put_shard_conn_to_pool(con); + } + } + } else { + g_message("%s, client timeout, closeing, diff:%d, con:%p", + G_STRLOC, diff, con); + con->prev_state = con->state; + con->state = ST_ERROR; + } + break; + } + return NETWORK_SOCKET_SUCCESS; +} + +NETWORK_MYSQLD_PLUGIN_PROTO(proxy_read_auth) { + return do_read_auth(con, con->config->allow_ip_table, con->config->deny_ip_table); +} + +static int +process_other_set_command(network_mysqld_con *con, const char *key, const char *s, + mysqld_query_attr_t *query_attr) +{ + g_debug("%s: vist process_other_set_command", G_STRLOC); + con->conn_attr_check_omit = 1; + network_socket *sock = con->client; + size_t s_len = strlen(s); + + if (strcasecmp(key, "sql_mode") == 0) { + g_string_assign_len(sock->sql_mode, s, s_len); + query_attr->sql_mode_set = 1; + } + return 0; +} + +static int +process_set_names(network_mysqld_con *con, char *s, + mysqld_query_attr_t *query_attr) +{ + network_socket *sock = con->client; + size_t s_len = strlen(s); + g_string_assign_len(sock->charset, s, s_len); + + con->conn_attr_check_omit = 1; + + query_attr->charset_set = 1; + sock->charset_code = charset_get_number(s); + return 0; +} + +static int proxy_parse_query(network_mysqld_con *con); +static int proxy_get_server_list(network_mysqld_con *con); + +static int check_backends_attr_changed(network_mysqld_con *con) +{ + int server_attr_changed = 0; + size_t i; + + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + if (pmd->backend->type != con->last_backends_type[i]) { + server_attr_changed = 1; + break; + } + + if (pmd->backend->state != BACKEND_STATE_UP && + pmd->backend->state != BACKEND_STATE_UNKNOWN) + { + server_attr_changed = 1; + } + } + + return server_attr_changed; +} + +NETWORK_MYSQLD_PLUGIN_PROTO(proxy_read_query) { + GQueue *chunks = con->client->recv_queue->chunks; + network_packet p; + p.data = g_queue_peek_head(chunks); + p.offset = 0; + network_mysqld_con_reset_command_response_state(con); + g_debug("%s: call network_mysqld_con_command_states_init", G_STRLOC); + if (network_mysqld_con_command_states_init(con, &p)) { + g_warning("%s: tracking mysql proto states failed", G_STRLOC); + con->prev_state = con->state; + con->state = ST_ERROR; + return NETWORK_SOCKET_SUCCESS; + } +#ifdef NETWORK_DEBUG_TRACE_STATE_CHANGES + query_queue_append(con->recent_queries, p.data); +#endif + + int is_process_stopped = 0; + int rc; + + if (con->servers != NULL) { + is_process_stopped = check_backends_attr_changed(con); + if (is_process_stopped) { + if (!con->client->is_server_conn_reserved) { + is_process_stopped = 0; + proxy_put_shard_conn_to_pool(con); + g_debug("%s server attr changed, but process continues", G_STRLOC); + } else { + network_mysqld_con_send_error(con->client, C("(proxy) unable to continue processing command")); + rc = PROXY_SEND_RESULT; + con->server_to_be_closed = 1; + g_message("%s server attr changed", G_STRLOC); + } + } + } + + if (!is_process_stopped) { + rc = proxy_parse_query(con); + } + + switch (rc) { + case PROXY_NO_DECISION: + break; /* go on to get groups */ + case PROXY_SEND_RESULT: + con->state = ST_SEND_QUERY_RESULT; + network_queue_clear(con->client->recv_queue); + network_mysqld_queue_reset(con->client); + return NETWORK_SOCKET_SUCCESS; + case PROXY_SEND_NONE: + network_queue_clear(con->client->recv_queue); + network_mysqld_queue_reset(con->client); + return NETWORK_SOCKET_SUCCESS; + default: + g_assert(0); + break; + } + + if (con->srv->query_cache_enabled) { + shard_plugin_con_t *st = con->plugin_con_state; + if (sql_context_is_cacheable(st->sql_context)) { + shard_plugin_con_t *st = con->plugin_con_state; + if (!con->is_in_transaction && !con->srv->master_preferred && + !(st->sql_context->rw_flag & CF_FORCE_MASTER) && + !(st->sql_context->rw_flag & CF_FORCE_SLAVE)) + { + if (try_to_get_resp_from_query_cache(con)) { + return NETWORK_SOCKET_SUCCESS; + } + } + } + } + + rc = proxy_get_server_list(con); + + switch (rc) { + case RET_SUCCESS: + if (con->use_all_prev_servers || + (con->sharding_plan && con->sharding_plan->groups->len > 0)) { + con->state = ST_GET_SERVER_CONNECTION_LIST; + } else { + con->state = ST_SEND_QUERY_RESULT; + if (con->buffer_and_send_fake_resp) { + network_mysqld_con_send_ok_full(con->client, 0, 0, 0, 0); + g_debug("%s: send faked resp to client", G_STRLOC); + } else { + network_mysqld_con_send_error_full(con->client, + C("no group yet"), ER_NO_DB_ERROR, "3D000"); + g_debug("%s: no group yet for this query", G_STRLOC); + } + network_queue_clear(con->client->recv_queue); + network_mysqld_queue_reset(con->client); + } + break; + case PROXY_SEND_RESULT: + con->state = ST_SEND_QUERY_RESULT; + network_queue_clear(con->client->recv_queue); + network_mysqld_queue_reset(con->client); + break; + case PROXY_NO_DECISION: + con->state = ST_GET_SERVER_CONNECTION_LIST; + break; + + default: + g_critical("%s: plugin(GET_SERVER_LIST) failed", G_STRLOC); + con->state = ST_ERROR; + break; + } + return NETWORK_SOCKET_SUCCESS; +} + +static void +mysqld_con_send_sequence(network_mysqld_con *con) +{ + char buffer[32]; + chassis *srv = con->srv; + uint64_t uniq_id = incremental_guid_get_next(&(srv->guid_state)); + snprintf(buffer, sizeof(buffer), "%llu", (unsigned long long) uniq_id); + + GPtrArray *fields = network_mysqld_proto_fielddefs_new(); + + MYSQL_FIELD *field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("SEQUENCE"); + field->type = MYSQL_TYPE_LONGLONG; + g_ptr_array_add(fields, field); + + GPtrArray *rows = g_ptr_array_new(); + GPtrArray *row = g_ptr_array_new(); + g_ptr_array_add(row, buffer); + g_ptr_array_add(rows, row); + + network_mysqld_con_send_resultset(con->client, fields, rows); + + network_mysqld_proto_fielddefs_free(fields); + g_ptr_array_free(row, TRUE); + g_ptr_array_free(rows, TRUE); +} + +static int +explain_shard_sql(network_mysqld_con *con, sharding_plan_t *plan) +{ + int rv = 0; + + if (con->client->default_db->len == 0) { + if (con->srv->default_db != NULL) { + g_string_assign_len(con->client->default_db, con->srv->default_db, + strlen(con->srv->default_db)); + g_debug("%s:set client default db:%s for con:%p", + G_STRLOC, con->client->default_db->str, con); + } + } + + shard_plugin_con_t *st = con->plugin_con_state; + + rv = sharding_parse_groups(con->client->default_db, st->sql_context, + &(con->srv->query_stats), con->key, plan); + + con->modified_sql = sharding_modify_sql(st->sql_context, &(con->hav_condi)); + if (con->modified_sql) { + sharding_plan_set_modified_sql(plan, con->modified_sql); + } + + sharding_plan_sort_groups(plan); + int abnormal = 0; + if (rv == ERROR_UNPARSABLE) { + const char *msg = st->sql_context->message ?: "sql parse error"; + network_mysqld_con_send_error_full(con->client, L(msg), + ER_CETUS_PARSE_SHARDING, "HY000"); + g_message(G_STRLOC ": unparsable sql:%s", con->orig_sql->str); + abnormal = 1; + } + return abnormal; +} + +static void +proxy_generate_shard_explain_packet(network_mysqld_con *con) +{ + sharding_plan_t *plan = sharding_plan_new(con->orig_sql); + if (explain_shard_sql(con, plan) != 0) { + sharding_plan_free(plan); + return; + } + + GPtrArray *fields = network_mysqld_proto_fielddefs_new(); + + MYSQL_FIELD *field1 = network_mysqld_proto_fielddef_new(); + field1->name = g_strdup("groups"); + field1->type = MYSQL_TYPE_VAR_STRING; + g_ptr_array_add(fields, field1); + MYSQL_FIELD *field2 = network_mysqld_proto_fielddef_new(); + field2->name = g_strdup("sql"); + field2->type = MYSQL_TYPE_VAR_STRING; + g_ptr_array_add(fields, field2); + + GPtrArray *rows; + rows = g_ptr_array_new_with_free_func((void *) network_mysqld_mysql_field_row_free); + + int i; + for (i = 0; i < plan->groups->len; i++) { + GPtrArray *row = g_ptr_array_new(); + + GString *group = g_ptr_array_index(plan->groups,i); + g_ptr_array_add(row, group->str); + const GString *sql = sharding_plan_get_sql(plan, group); + g_ptr_array_add(row, sql->str); + + g_ptr_array_add(rows, row); + } + + network_mysqld_con_send_resultset(con->client, fields, rows); + + network_mysqld_proto_fielddefs_free(fields); + g_ptr_array_free(rows, TRUE); + sharding_plan_free(plan); +} + +static int +analysis_query(network_mysqld_con *con, mysqld_query_attr_t *query_attr) +{ + shard_plugin_con_t *st = con->plugin_con_state; + sql_context_t *context = st->sql_context; + query_stats_t *stats = &(con->srv->query_stats); + + con->could_be_tcp_streamed = 0; + con->candidate_tcp_streamed = 0; + + switch (context->stmt_type) { + case STMT_SELECT: { + if (con->srv->is_tcp_stream_enabled && !con->dist_tran) { + g_debug("%s: con dist tran is false", G_STRLOC); + con->could_be_tcp_streamed = 1; + } + stats->com_select += 1; + sql_select_t *select = (sql_select_t *)context->sql_statement; + + if (con->could_be_tcp_streamed) { + if (sql_expr_list_find_aggregate(select->columns)) { + con->could_be_tcp_streamed = 0; + g_debug("%s: con tcp stream false", G_STRLOC); + } + } + + gboolean is_insert_id = FALSE; + sql_expr_list_t *cols = select->columns; + if (cols && cols->len > 0) { + sql_expr_t *col = g_ptr_array_index(cols, 0); + if (sql_expr_is_function(col, "LAST_INSERT_ID")) { + is_insert_id = TRUE; + } else if (sql_expr_is_id(col, "LAST_INSERT_ID")) { + is_insert_id = TRUE; + } + } + if (is_insert_id == TRUE) { + g_debug("%s: return last insert id", G_STRLOC); + /* TODO last insert id processing */ + } + break; + } + case STMT_UPDATE: + stats->com_update += 1; break; + case STMT_INSERT: + stats->com_insert += 1; break; + case STMT_DELETE: + stats->com_delete += 1; break; + case STMT_SET_NAMES:{ + char *charset_name = (char *)context->sql_statement; + process_set_names(con, charset_name, query_attr); + g_debug("%s: set names", G_STRLOC); + break; + } + case STMT_SET_TRANSACTION: + if (sql_filter_vars_is_silent("TRANSACTION", "*")) { + network_mysqld_con_send_ok(con->client); + } else { + network_mysqld_con_send_error_full(con->client, + L("(cetus) SET TRANSACTION not supported"), + ER_CETUS_NOT_SUPPORTED, "HY000"); + } + return PROXY_SEND_RESULT; + case STMT_SET: { + sql_expr_list_t *set_list = context->sql_statement; + if (set_list && set_list->len > 0) { + sql_expr_t *expr = g_ptr_array_index(set_list, 0); + if (expr->op == TK_EQ) { + const char *lhs = sql_expr_id(expr->left); + const char *rhs = sql_expr_id(expr->right); + + if (sql_filter_vars_is_silent(lhs, rhs)) { + network_mysqld_con_send_ok(con->client); + g_string_free(g_queue_pop_tail(con->client->recv_queue->chunks), TRUE); + g_message("silent variable: %s\n", lhs); + return PROXY_SEND_RESULT; + } + + /* set autocomit = x */ + if (sql_context_is_autocommit_on(context)) { + con->is_auto_commit = 1; + con->is_auto_commit_trans_buffered = 0; + g_debug("%s: autocommit on", G_STRLOC); + } else if (sql_context_is_autocommit_off(context)) { + con->is_auto_commit = 0; + con->is_auto_commit_trans_buffered = 1; + g_debug("%s: autocommit off, now in transaction", G_STRLOC); + } else { + if (lhs && rhs) { + process_other_set_command(con, lhs, rhs, query_attr); + } + } + } + } + + break; + } + case STMT_COMMIT: + con->is_commit_or_rollback = 1; + break; + case STMT_ROLLBACK: + con->is_commit_or_rollback = 1; + con->is_rollback = 1; + break; + case STMT_USE: { + char *dbname = (char *)context->sql_statement; + g_string_assign(con->client->default_db, dbname); + g_debug("%s:set default db:%s for con:%p", + G_STRLOC, con->client->default_db->str, con); + break; + } + case STMT_START: + if (con->is_auto_commit) { + g_debug("%s: start transaction command here", G_STRLOC); + con->is_start_trans_buffered = 1; + con->is_start_tran_command = 1; + con->is_auto_commit = 0; + } + break; + default: + break; + } + return PROXY_NO_DECISION; +} + +static int shard_handle_local_query(network_mysqld_con *con, sql_context_t *context) +{ + /* currently 3 kinds of local query */ + if (context->explain == TK_SHARD_EXPLAIN) { + proxy_generate_shard_explain_packet(con); + return PROXY_SEND_RESULT; + } + g_assert(context->stmt_type == STMT_SELECT); + sql_select_t *select = context->sql_statement; + sql_expr_t *col = g_ptr_array_index(select->columns, 0); + if (sql_expr_is_function(col, "CURRENT_DATE")) { + network_mysqld_con_send_current_date(con->client, "CURRENT_DATE"); + } else if (sql_expr_is_function(col, "CETUS_SEQUENCE")) { + mysqld_con_send_sequence(con); + } else if (sql_expr_is_function(col, "CETUS_VERSION")) { + network_mysqld_con_send_cetus_version(con->client); + } + return PROXY_SEND_RESULT; +} + +static int proxy_parse_query(network_mysqld_con *con) +{ + shard_plugin_con_t *st = con->plugin_con_state; + + g_debug("%s: call proxy_parse_query:%p", G_STRLOC, con); + + if (con->is_commit_or_rollback) { /* previous sql */ + if (!con->is_auto_commit) { + con->is_auto_commit_trans_buffered = 1; + } + if (con->dist_tran_state >= NEXT_ST_XA_START && + con->dist_tran_state != NEXT_ST_XA_OVER) + { + g_warning("%s: xa is not over yet:%p, xa state:%d", G_STRLOC, + con, con->dist_tran_state); + con->server_to_be_closed = 1; + } else if (con->dist_tran_xa_start_generated && !con->dist_tran_decided) { + if (con->servers && con->servers->len > 0) { + con->server_to_be_closed = 1; + g_message("%s: server conn should be closed:%p", G_STRLOC, con); + } + } + con->client->is_server_conn_reserved = 0; + g_debug("%s: set is_server_conn_reserved false:%p", G_STRLOC, con); + } else { + g_debug("%s: is_commit_or_rollback is false:%p", G_STRLOC, con); + } + + con->conn_attr_check_omit = 0; + con->is_commit_or_rollback = 0; + con->is_rollback = 0; + con->is_timeout = 0; + con->is_xa_query_sent = 0; + con->xa_query_status_error_and_abort = 0; + + network_packet packet; + packet.data = g_queue_peek_head(con->client->recv_queue->chunks); + packet.offset = 0; + if (packet.data != NULL) { + guint8 command; + network_mysqld_proto_skip_network_header(&packet); + if (network_mysqld_proto_get_int8(&packet, &command) != 0) { + network_mysqld_con_send_error(con->client, + C("(proxy) unable to retrieve command")); + return PROXY_SEND_RESULT; + } + + con->parse.command = command; + g_debug("%s:command:%d", G_STRLOC, command); + switch (command) { + case COM_QUERY: { + network_mysqld_con_reset_query_state(con); + + gsize sql_len = packet.data->len - packet.offset; + network_mysqld_proto_get_gstr_len(&packet, sql_len, con->orig_sql); + g_string_append_c(con->orig_sql, '\0');/* 2 more NULL for lexer EOB */ + g_string_append_c(con->orig_sql, '\0'); + + g_debug("%s: sql:%s", G_STRLOC, con->orig_sql->str); + sql_context_t *context = st->sql_context; + sql_context_parse_len(context, con->orig_sql); + + if (context->rc == PARSE_SYNTAX_ERR) { + char *msg = context->message; + g_message("%s SQL syntax error: %s. while parsing: %s", + G_STRLOC, msg, con->orig_sql->str); + network_mysqld_con_send_error_full(con->client, msg, strlen(msg), + ER_SYNTAX_ERROR, "42000"); + return PROXY_SEND_RESULT; + } else if (context->rc == PARSE_NOT_SUPPORT) { + char *msg = context->message; + g_message("%s SQL unsupported: %s. while parsing: %s, clt:%s", + G_STRLOC, msg, con->orig_sql->str, con->client->src->name->str); + network_mysqld_con_send_error_full(con->client, msg, strlen(msg), + ER_CETUS_NOT_SUPPORTED, "HY000"); + return PROXY_SEND_RESULT; + } + /* forbid force write on slave */ + if ((context->rw_flag & CF_FORCE_SLAVE) && (context->rw_flag & CF_WRITE)) { + g_message("%s Comment usage error. SQL: %s", G_STRLOC, con->orig_sql->str); + network_mysqld_con_send_error(con->client, C("Force write on read-only slave")); + return PROXY_SEND_RESULT; + } + + if (context->clause_flags & CF_LOCAL_QUERY) { + return shard_handle_local_query(con, context); + } + memset(&(con->query_attr), 0, sizeof(mysqld_query_attr_t)); + return analysis_query(con, &(con->query_attr)); + } + case COM_INIT_DB: + break; + case COM_QUIT: + g_debug("%s: quit command:%d", G_STRLOC, command); + con->state = ST_CLOSE_CLIENT; + return PROXY_SEND_NONE; + case COM_STMT_PREPARE: { + network_mysqld_con_send_error_full(con->client, + C("sharding proxy does not support prepare stmt"), + ER_CETUS_NOT_SUPPORTED, "HY000"); + return PROXY_SEND_RESULT; + } + case COM_PING: + network_mysqld_con_send_ok(con->client); + return PROXY_SEND_RESULT; + default: { + GString *sql = g_string_new(NULL); + GString *data = g_queue_peek_head(con->client->recv_queue->chunks); + g_string_append_len(sql, data->str + (NET_HEADER_SIZE + 1), + data->len - (NET_HEADER_SIZE + 1)); + network_mysqld_con_send_error_full(con->client, + C("sharding proxy does not support this command now"), + ER_CETUS_NOT_SUPPORTED, "HY000"); + g_warning("%s: unknown command:%d, sql:%s", G_STRLOC, command, sql->str); + g_string_free(sql, TRUE); + return PROXY_SEND_RESULT; + } + } + } else { + g_warning("%s: chunk is null", G_STRLOC); + } + + return PROXY_NO_DECISION; +} + +static int +wrap_check_sql(network_mysqld_con *con, struct sql_context_t *sql_context) +{ + con->modified_sql = sharding_modify_sql(sql_context, &(con->hav_condi)); + if (con->modified_sql) { + g_message("orig_sql: %s", con->orig_sql->str); + g_message("modified: %s", con->modified_sql->str); + } + if (con->modified_sql) { + con->sql_modified = 1; + sharding_plan_set_modified_sql(con->sharding_plan, con->modified_sql); + } + + return con->sql_modified; +} + +static void record_last_backends_type(network_mysqld_con *con) +{ + size_t i; + + g_debug("%s record_last_backends_type", G_STRLOC); + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + con->last_backends_type[i] = pmd->backend->type; + } +} + +static void remove_ro_servers(network_mysqld_con *con) +{ + int has_rw_server = 0; + int has_ro_server = 0; + size_t i; + GPtrArray *new_servers = NULL; + + g_debug("%s: call remove_ro_servers:%p", G_STRLOC, con); + + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + if (!pmd->server->is_read_only) { + has_rw_server = 1; + break; + } else { + has_ro_server = 1; + } + } + + if (!has_ro_server) { + g_debug("%s: has no ro server:%p", G_STRLOC, con); + return; + } + + if (has_rw_server) { + g_debug("%s: has rw server:%p", G_STRLOC, con); + new_servers = g_ptr_array_new(); + } + + g_debug("%s: check servers:%p", G_STRLOC, con); + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + if (pmd->server->is_read_only) { + network_connection_pool *pool = pmd->backend->pool; + network_socket *server = pmd->server; + + CHECK_PENDING_EVENT(&(server->event)); + + network_pool_add_idle_conn(pool, con->srv, server); + pmd->backend->connected_clients--; + g_debug("%s: conn clients sub, total len:%d, back:%p, value:%d con:%p, s:%p", + G_STRLOC, con->servers->len, pmd->backend, + pmd->backend->connected_clients, con, server); + + pmd->sql = NULL; + g_free(pmd); + } else { + pmd->server->parse.qs_state = PARSE_COM_QUERY_INIT; + g_ptr_array_add(new_servers, pmd); + } + } + + gpointer *pdata = g_ptr_array_free(con->servers, FALSE); + g_free (pdata); + if (has_rw_server) { + con->servers = new_servers; + } else { + con->servers = NULL; + } +} + + +static int +process_init_db_when_get_server_list(network_mysqld_con *con, + sharding_plan_t *plan, int *rv, int *disp_flag) +{ + GPtrArray *groups = g_ptr_array_new(); + + network_packet packet; + packet.data = g_queue_peek_head(con->client->recv_queue->chunks); + packet.offset = NET_HEADER_SIZE + 1; + int name_len = network_mysqld_proto_get_packet_len(packet.data); + char *db_name = NULL; + + if (name_len > PACKET_LEN_MAX) { + g_warning("%s: name len is too long:%d", G_STRLOC, name_len); + } else { + name_len = name_len - 1; + network_mysqld_proto_get_str_len(&packet, &db_name, name_len); + shard_conf_get_fixed_group(groups, db_name, con->key); + } + + if (groups->len > 0) {/* has database */ + if (con->dist_tran) { + *rv = USE_PREVIOUS_TRAN_CONNS; + } else { + g_string_assign_len(con->client->default_db, db_name, name_len); + sharding_plan_add_groups(plan, groups); + g_ptr_array_free(groups, TRUE); + network_mysqld_con_set_sharding_plan(con, plan); + *disp_flag = PROXY_NO_DECISION; + if (db_name) g_free(db_name); + + return 0; + } + } else { + network_mysqld_con_send_error(con->client, C("not a configured DB")); + GString *data = g_queue_pop_head(con->client->recv_queue->chunks); + g_string_free(data, TRUE); + g_ptr_array_free(groups, TRUE); + sharding_plan_free(plan); + *disp_flag = PROXY_SEND_RESULT; + if (db_name) g_free(db_name); + + return 0; + } + + if (db_name) g_free(db_name); + + return 1; +} + + +static void before_get_server_list(network_mysqld_con *con) { + + shard_plugin_con_t *st = con->plugin_con_state; + + if (con->is_start_trans_buffered || con->is_auto_commit_trans_buffered) { + if (con->last_warning_met) { + con->last_warning_met = 0; + if (con->is_in_transaction) { + g_warning("%s: is_in_transaction true for con:%p", G_STRLOC, con); + } + con->server_closed = 1; + con->client->is_server_conn_reserved = 0; + } + con->is_in_transaction = 1; + con->dist_tran_xa_start_generated = 0; + if (sql_context_is_single_node_trx(st->sql_context)) { + con->is_tran_not_distributed_by_comment = 1; + g_debug("%s: set is_tran_not_distributed_by_comment true:%p", + G_STRLOC, con); + } + + g_debug("%s: check is_server_conn_reserved:%p", G_STRLOC, con); + if (!con->client->is_server_conn_reserved) { + g_debug("%s: is_server_conn_reserved false:%p", G_STRLOC, con); + if (con->servers && con->servers->len > 0) { + g_debug("%s: call proxy_put_shard_conn_to_pool:%p", + G_STRLOC, con); + proxy_put_shard_conn_to_pool(con); + } + } else { + g_message("%s: still hold conn when starting a new transaction:%p", + G_STRLOC, con); + } + } + + if (con->dist_tran_decided) { + if (con->servers && con->servers->len > 0) { + g_debug("%s: call proxy_put_shard_conn_to_pool:%p", G_STRLOC, con); + proxy_put_shard_conn_to_pool(con); + } + con->dist_tran_xa_start_generated = 0; + } +} + + +static void +process_rv_use_none(network_mysqld_con *con, sharding_plan_t *plan, + int *disp_flag) +{ + /* SET AUTOCOMMIT = 0 || START TRANSACTION */ + con->delay_send_auto_commit = 1; + g_debug("%s: delay send autocommit 0", G_STRLOC); + network_mysqld_con_set_sharding_plan(con, plan); + GString *packet = g_queue_pop_head(con->client->recv_queue->chunks); + g_string_free(packet, TRUE); + network_mysqld_con_send_ok_full(con->client, 0, 0, 0, 0); + *disp_flag = PROXY_SEND_RESULT; +} + +static int +process_rv_use_same(network_mysqld_con *con, sharding_plan_t *plan, + int *disp_flag) +{ + /* SET AUTOCOMMIT = 1 */ + con->is_auto_commit = 1; + if (con->dist_tran && con->dist_tran_state < NEXT_ST_XA_END) { + con->client->is_server_conn_reserved = 0; + con->is_commit_or_rollback = 1; + g_message("%s: no commit when set autocommit = 1:%p", G_STRLOC, con); + } else { + con->delay_send_auto_commit = 0; + network_mysqld_con_set_sharding_plan(con, plan); + g_debug("%s: no need to send autocommit true", G_STRLOC); + GString *packet = g_queue_pop_head(con->client->recv_queue->chunks); + g_string_free(packet, TRUE); + con->is_in_transaction = 0; + con->client->is_server_conn_reserved = 0; + network_mysqld_con_send_ok_full(con->client, 0, 0, 0, 0); + *disp_flag = PROXY_SEND_RESULT; + return 0; + } + + return 1; +} + + +static int +process_rv_use_previous_tran_conns(network_mysqld_con *con, sharding_plan_t *plan, + int *rv, int *disp_flag) +{ + /* COMMIT/ROLLBACK */ + g_debug("%s: use previous conn for con:%p", G_STRLOC, con); + + if (con->is_auto_commit || con->servers == NULL || con->servers->len == 0) + { + con->buffer_and_send_fake_resp = 1; + con->delay_send_auto_commit = 0; + con->is_auto_commit_trans_buffered = 0; + con->is_start_trans_buffered = 0; + g_debug("%s: buffer_and_send_fake_resp set true:%p", G_STRLOC, con); +#ifdef NETWORK_DEBUG_TRACE_STATE_CHANGES + query_queue_dump(con->recent_queries); +#endif + } else { + if (con->servers->len > 1) { + if (!con->dist_tran) { + network_mysqld_con_send_ok_full(con->client, 0, 0, 0, 0); + g_debug("%s: set ERROR_DUP_COMMIT_OR_ROLLBACK here", G_STRLOC); + sharding_plan_free(plan); + *disp_flag = PROXY_SEND_RESULT; + return 0; + } else { + *rv = USE_DIS_TRAN; + con->use_all_prev_servers = 1; + } + } else { + if (!con->dist_tran) { + if (!con->is_tran_not_distributed_by_comment) { + network_mysqld_con_send_ok_full(con->client, 0, 0, 0, 0); + g_debug("%s: set ERROR_DUP_COMMIT_OR_ROLLBACK here", G_STRLOC); + sharding_plan_free(plan); + *disp_flag = PROXY_SEND_RESULT; + return 0; + } + } else { + *rv = USE_DIS_TRAN; + con->use_all_prev_servers = 1; + } + } + } + + if (con->servers != NULL && con->servers->len > 0) { + sharding_plan_free(plan); + } else { + network_mysqld_con_set_sharding_plan(con, plan); + } + + return 1; +} + + +static int +process_rv_default(network_mysqld_con *con, sharding_plan_t *plan, + int *rv, int *disp_flag) +{ + if (con->is_tran_not_distributed_by_comment) { + g_debug("%s: default prcessing here for conn:%p", G_STRLOC, con); + + int valid_single_tran = 1; + if (plan->groups->len != 1) { + valid_single_tran = 0; + g_debug("%s: group num:%d for con:%p", G_STRLOC, plan->groups->len, con); + } else { + if (con->sharding_plan) { + if (con->sharding_plan->groups->len == 1) { + GString *prev_group = g_ptr_array_index(con->sharding_plan->groups, 0); + GString *cur_group = g_ptr_array_index(plan->groups, 0); + if (strcasecmp(prev_group->str, cur_group->str) != 0) { + valid_single_tran = 0; + } + } else if (con->sharding_plan->groups->len > 1) { + valid_single_tran = 0; + g_debug("%s: orig group num:%d for con:%p", G_STRLOC, + con->sharding_plan->groups->len, con); + } + } + } + + if (!valid_single_tran) { + sharding_plan_free(plan); + g_message("%s: tran conflicted here for con:%p", G_STRLOC, con); + network_mysqld_con_send_error_full(con->client, + C("conflict with stand-alone tran comment"), + ER_CETUS_SINGLE_NODE_FAIL, "HY000"); + *disp_flag = PROXY_SEND_RESULT; + return 0; + } else { + network_mysqld_con_set_sharding_plan(con, plan); + } + } else { + network_mysqld_con_set_sharding_plan(con, plan); + if (plan->groups->len >= 2) { + if (!con->is_auto_commit || + con->is_start_tran_command) { /* current sql START ? */ + *rv = USE_DIS_TRAN; + } else if (*rv == USE_DIS_TRAN) { + g_debug("%s: user distributed trans found for sql:%s", + G_STRLOC, con->orig_sql->str); + con->dist_tran_xa_start_generated = 0; + } else { + con->delay_send_auto_commit = 0; + g_debug("%s: not in transaction:%s", + G_STRLOC, con->orig_sql->str); + } + } else { + if (con->dist_tran) { + if (plan->groups->len == 0 && *rv != ERROR_UNPARSABLE) { + network_mysqld_con_send_error_full(con->client, + C("Cannot find backend groups"), + ER_CETUS_NO_GROUP, "HY000"); + sharding_plan_free(plan); + con->sharding_plan = NULL;/* already ref by con, remove! */ + *disp_flag = PROXY_SEND_RESULT; + return 0; + } + } else { + if (plan->groups->len == 1 && (!con->is_auto_commit)) { + con->delay_send_auto_commit = 0; + *rv = USE_DIS_TRAN; + } + } + } + } + + return 1; +} + + +static int +make_first_decision(network_mysqld_con *con, sharding_plan_t *plan, + int *rv, int *disp_flag) +{ + switch(*rv) + { + case USE_NONE: + process_rv_use_none(con, plan, disp_flag); + return 0; + + case USE_PREVIOUS_WARNING_CONN: + sharding_plan_free(plan); + if (con->last_warning_met) { + con->use_all_prev_servers = 1; + if (con->servers == NULL) { + con->client->is_server_conn_reserved = 0; + con->state = ST_SEND_QUERY_RESULT; + network_mysqld_con_send_ok_full(con->client, 0, 0, 0, 0); + g_warning("%s: show warnings has no servers yet", G_STRLOC); + network_queue_clear(con->client->recv_queue); + network_mysqld_queue_reset(con->client); + return 0; + } + } + break; + case USE_SAME: + if (!process_rv_use_same(con, plan, disp_flag)) { + return 0; + } else { + if (!process_rv_use_previous_tran_conns(con, plan, rv, disp_flag)) { + return 0; + } + } + break; + case USE_PREVIOUS_TRAN_CONNS: + if (!process_rv_use_previous_tran_conns(con, plan, rv, disp_flag)) { + return 0; + } + break; + + default: + if (!process_rv_default(con, plan, rv, disp_flag)) { + return 0; + } + break; + + }/* switch */ + + return 1; +} + + +static int make_decisions(network_mysqld_con *con, int rv, int *disp_flag) +{ + shard_plugin_con_t *st = con->plugin_con_state; + + query_stats_t *stats = &(con->srv->query_stats); + + switch(rv) + { /* TODO: move these inside to give specific reasons */ + case ERROR_UNPARSABLE: + { + const char *msg = st->sql_context->message ?: "sql parse error"; + int err_code = (st->sql_context->rc == PARSE_NOT_SUPPORT) + ? ER_CETUS_NOT_SUPPORTED : ER_CETUS_PARSE_SHARDING; + network_mysqld_con_send_error_full(con->client, L(msg), err_code, "HY000"); + g_message(G_STRLOC ": unparsable sql:%s", con->orig_sql->str); + *disp_flag = PROXY_SEND_RESULT; + return 0; + } + + case USE_DIS_TRAN: + if (!con->dist_tran) { + con->dist_tran_state = NEXT_ST_XA_START; + stats->xa_count += 1; + } + con->dist_tran = 1; + con->could_be_tcp_streamed = 0; + con->dist_tran_failed = 0; + con->delay_send_auto_commit = 0; + g_debug("%s: xa transaction query:%s for con:%p", + G_STRLOC, con->orig_sql->str, con); + if (con->sharding_plan && con->sharding_plan->groups->len > 1) { + wrap_check_sql(con, st->sql_context); + } + break; + + default: + con->dist_tran_failed = 0; + if (con->sharding_plan && con->sharding_plan->groups->len > 1) { + wrap_check_sql(con, st->sql_context); + } + break; + } + + return 1; +} + + +static int proxy_get_server_list(network_mysqld_con *con) +{ + + g_debug("%s: call proxy_get_server_list:%p for sql:%s, clt:%s, xa state:%d", G_STRLOC, + con, con->orig_sql->str, con->client->src->name->str, con->dist_tran_state); + + before_get_server_list(con); + + if (con->client->default_db->len == 0) { + g_string_assign(con->client->default_db, con->srv->default_db); + g_debug("%s:set default db:%s for con:%p", + G_STRLOC, con->client->default_db->str, con); + } + + con->use_all_prev_servers = 0; + + query_stats_t *stats = &(con->srv->query_stats); + sharding_plan_t *plan = sharding_plan_new(con->orig_sql); + int rv = 0, disp_flag = 0; + + shard_plugin_con_t *st = con->plugin_con_state; + + switch (con->parse.command) { + case COM_INIT_DB: + if (!process_init_db_when_get_server_list(con, plan, &rv, &disp_flag)) + { + return disp_flag; + } + break; + default: + rv = sharding_parse_groups(con->client->default_db, st->sql_context, stats, + con->key, plan); + break; + } + + if (plan->groups->len > 1) { + switch (st->sql_context->stmt_type) { + case STMT_SELECT: stats->com_select_shard += 1;break; + case STMT_INSERT: stats->com_insert_shard += 1;break; + case STMT_UPDATE: stats->com_update_shard += 1;break; + case STMT_DELETE: stats->com_delete_shard += 1;break; + default: break; + } + } + + con->dist_tran_decided = 0; + con->buffer_and_send_fake_resp = 0; + con->server_to_be_closed = 0; + con->server_closed = 0; + con->resp_too_long = 0; + con->all_participate_num = 0; + + if (con->srv->master_preferred || + st->sql_context->rw_flag & CF_WRITE || + st->sql_context->rw_flag & CF_FORCE_MASTER || /*# mode = READWRITE */ + !con->is_auto_commit || rv == USE_SAME) + { + g_debug("%s: check is_read_ro_server_allowed:%p", G_STRLOC, con); + if (con->is_read_ro_server_allowed) { + if (con->servers) { + remove_ro_servers(con); + } + } + con->is_read_ro_server_allowed = 0; + stats->client_query.rw++; + stats->proxyed_query.rw++; + } else { + con->is_read_ro_server_allowed = 1; + stats->client_query.ro++; + stats->proxyed_query.ro++; + } + if (st->sql_context->rw_flag & CF_FORCE_SLAVE) { + con->is_read_ro_server_allowed = 1; + } + + if (rv != USE_PREVIOUS_WARNING_CONN) { + con->last_warning_met = 0; + if (!con->is_in_transaction) { + if (con->client->is_server_conn_reserved) { + con->client->is_server_conn_reserved = 0; + g_message("%s: is_server_conn_reserved is set false", G_STRLOC); + } + } + } + + if (!make_first_decision(con, plan, &rv, &disp_flag)) { + return disp_flag; + } + + if (con->is_commit_or_rollback) { /* current sql */ + if (con->is_tran_not_distributed_by_comment) { + con->is_tran_not_distributed_by_comment = 0; + } + } + + if (!make_decisions(con, rv, &disp_flag)) { + return disp_flag; + } + + return RET_SUCCESS; +} + +static gboolean proxy_get_pooled_connection( + network_mysqld_con *con, + shard_plugin_con_t *st, + GString *group, + int type, + network_socket **sock, + int *is_robbed, + int *server_unavailable) +{ + chassis_private *g = con->srv->priv; + network_backend_t *backend = NULL; + + g_debug("%s:group:%s", G_STRLOC, group->str); + network_group_t *backend_group = network_backends_get_group(g->backends, group); + if (backend_group == NULL) { + g_message("%s:backend_group is nil", G_STRLOC); + *server_unavailable = 1; + return FALSE; + } + + if (type == BACKEND_TYPE_RO) { + backend = network_group_pick_slave_backend(backend_group); + if (backend == NULL) { /* fallback to readwrite backend */ + type = BACKEND_TYPE_RW; + } + } + + if (type == BACKEND_TYPE_RW) { + backend = backend_group->master; /* may be NULL if master down */ + if (!backend || (backend->state != BACKEND_STATE_UP && + backend->state != BACKEND_STATE_UNKNOWN)) { + *server_unavailable = 1; + return FALSE; + } + } + + if (backend == NULL) { + g_warning("%s: backend null, type:%d", G_STRLOC, type); + *server_unavailable = 1; + return FALSE; + } + + *sock = network_connection_pool_get(backend->pool, con->client->response->username, is_robbed); + if (*sock == NULL) { + return FALSE; + } + + (*sock)->is_read_only = (type == BACKEND_TYPE_RO) ? 1 : 0; + st->backend = backend; + + st->backend->connected_clients++; + + g_debug("%s: connected_clients add, backend:%p, now:%d, con:%p, server:%p", + G_STRLOC, backend, st->backend->connected_clients, con, *sock); + + return TRUE; +} + + +gboolean proxy_add_server_connection( + network_mysqld_con *con, + GString *group, + int *server_unavailable) + +{ + server_session_t *pmd; + network_socket *server; + + if (con->servers != NULL) { + size_t i; + for (i = 0; i < con->servers->len ; i++ ) { + pmd = (server_session_t *) (g_ptr_array_index(con->servers, i)); + if (pmd != NULL) { + if (g_string_equal(pmd->server->group, group)) { + pmd->participated = 1; + pmd->state = NET_RW_STATE_NONE; + pmd->sql = sharding_plan_get_sql(con->sharding_plan, group); + if (con->dist_tran) { + if (con->dist_tran_state == NEXT_ST_XA_START) { + pmd->dist_tran_state = NEXT_ST_XA_START; + pmd->xa_start_already_sent = 0; + } + + if (pmd->dist_tran_state == NEXT_ST_XA_OVER || + pmd->dist_tran_state == 0) + { + g_message("%s: reset xa state:%d for pmd ndx:%d, con:%p", + G_STRLOC, pmd->dist_tran_state, (int) i, con); + pmd->dist_tran_state = NEXT_ST_XA_START; + pmd->xa_start_already_sent = 0; + } + pmd->dist_tran_participated = 1; + } + return TRUE; + } + } + } + } else { + con->servers = g_ptr_array_new(); + } + + gboolean ok; + int type = BACKEND_TYPE_RW; + if (con->is_read_ro_server_allowed) { + type = BACKEND_TYPE_RO; + } + shard_plugin_con_t *st = con->plugin_con_state; + int is_robbed = 0; + if ((ok = proxy_get_pooled_connection( + con, + st, + group, + type, + &server, &is_robbed, server_unavailable))) + { + pmd = g_new0(server_session_t, 1); + + con->is_new_server_added = 1; + + pmd->con = con; + pmd->backend = st->backend; + pmd->server = server; + server->group = group; + pmd->sql = sharding_plan_get_sql(con->sharding_plan, group); + pmd->attr_consistent_checked = 0; + pmd->attr_consistent = 0; + pmd->server->last_packet_id = 0; + pmd->server->parse.qs_state = PARSE_COM_QUERY_INIT; + pmd->participated = 1; + pmd->state = NET_RW_STATE_NONE; + pmd->fresh = 1; + pmd->is_xa_over = 0; + + if (con->dist_tran) { + pmd->is_in_xa = 1; + pmd->dist_tran_state = NEXT_ST_XA_START; + pmd->dist_tran_participated = 1; + pmd->xa_start_already_sent = 0; + } else { + pmd->is_in_xa = 0; + } + pmd->server->is_robbed = is_robbed; + + g_ptr_array_add(con->servers, pmd);/* TODO: CHANGE SQL */ + } + + return ok; +} + +static gint pmd_comp(gconstpointer a1, gconstpointer a2) { + server_session_t *pmd1 = *(server_session_t **) a1; + server_session_t *pmd2 = *(server_session_t **) a2; + + return strcmp(pmd1->server->group->str, pmd2->server->group->str); +} + +static gboolean +proxy_add_server_connection_array(network_mysqld_con *con, int *server_unavailable) +{ + sharding_plan_t *plan = con->sharding_plan; + size_t i; + + gint8 server_map[MAX_SERVER_NUM] = {0}; + + if (con->dist_tran == 0 && con->servers != NULL && con->servers->len > 0) { + int hit = 0; + for (i = 0; i < con->servers->len ; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + const GString *group = pmd->server->group; + if (sharding_plan_has_group(plan, group)) { + if (con->is_read_ro_server_allowed && !pmd->server->is_read_only) { + g_debug("%s: use read server", G_STRLOC); + } else if (!con->is_read_ro_server_allowed && pmd->server->is_read_only) { + g_debug("%s: should release ro server to pool", G_STRLOC); + } else { + hit++; + server_map[i] = 1; + pmd->sql = sharding_plan_get_sql(con->sharding_plan, group); + g_debug("%s: hit server", G_STRLOC); + } + } + } + + if (hit == plan->groups->len && con->servers->len == hit) { + return TRUE; + } else { + if (con->is_in_transaction) { + g_warning("%s:in single tran, but visit multi servers for con:%p", + G_STRLOC, con); + return FALSE; + } + GPtrArray *new_servers = g_ptr_array_new(); + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + if (server_map[i] == 0) { + network_connection_pool *pool = pmd->backend->pool; + network_socket *server = pmd->server; + + CHECK_PENDING_EVENT(&(server->event)); + + network_pool_add_idle_conn(pool, con->srv, server); + pmd->backend->connected_clients--; + g_debug("%s: conn clients sub, total len:%d, back:%p, value:%d con:%p, s:%p", + G_STRLOC, con->servers->len, pmd->backend, + pmd->backend->connected_clients, con, server); + + pmd->sql = NULL; + g_free(pmd); + + } else { + pmd->server->parse.qs_state = PARSE_COM_QUERY_INIT; + g_ptr_array_add(new_servers, pmd); + } + } + + gpointer *pdata = g_ptr_array_free(con->servers, FALSE); + g_free (pdata); + + con->servers = new_servers; + } + } else { + if (con->dist_tran && con->servers) { + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + pmd->participated = 0; + } + } + } + + for(i = 0; i < plan->groups->len; i++) { + GString *group = g_ptr_array_index(plan->groups, i); + + if (!proxy_add_server_connection(con, group, server_unavailable)) { + return FALSE; + } + } + + if (con->is_new_server_added && con->dist_tran && con->servers->len > 1) { + g_ptr_array_sort(con->servers, pmd_comp); + } + + return TRUE; +} + +static gboolean check_and_set_attr_bitmap(network_mysqld_con *con) +{ + size_t i; + gboolean result = TRUE; + gboolean consistant; + + if (con->conn_attr_check_omit) { /* current sql is a SET statement */ + mysqld_query_attr_t *query_attr = &(con->query_attr); + if (query_attr->sql_mode_set) { + return result; + } + g_debug("%s:conn_attr_check_omit true", G_STRLOC); + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + if (query_attr->charset_set) { + g_string_assign(pmd->server->charset, con->client->charset->str); + } + } + return result; + } + + con->unmatched_attribute = 0; + + g_debug("%s:check conn attr, default db:%s", G_STRLOC, con->client->default_db->str); + + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + if (pmd->attr_consistent_checked) { + g_debug("%s:already checked for server:%p", G_STRLOC, pmd->server); + pmd->attr_consistent = 1; + continue; + } + + g_debug("%s:server:%p, query state:%d", G_STRLOC, pmd->server, + pmd->server->parse.qs_state); + + consistant = TRUE; + pmd->attr_diff = 0; + + if (pmd->server->is_robbed) { + pmd->attr_diff = ATTR_DIF_CHANGE_USER; + result = FALSE; + con->unmatched_attribute |= ATTR_DIF_CHANGE_USER; + consistant = FALSE; + } else { + if (con->parse.command != COM_INIT_DB) { + /* check default db */ + if (!g_string_equal(con->client->default_db, pmd->server->default_db)) { + g_debug("%s:default db for client:%s", + G_STRLOC, con->client->default_db->str); + pmd->attr_diff = ATTR_DIF_DEFAULT_DB; + result = FALSE; + con->unmatched_attribute |= ATTR_DIF_DEFAULT_DB; + consistant = FALSE; + g_debug("%s: default db different", G_STRLOC); + } + } + } + + if (!g_string_equal(con->client->sql_mode, pmd->server->sql_mode)) { + g_warning("%s: not support different sql modes", G_STRLOC); + } + + if (!g_string_equal(con->client->charset, pmd->server->charset)) { + pmd->attr_diff |= ATTR_DIF_CHARSET; + con->unmatched_attribute |= ATTR_DIF_CHARSET; + result = FALSE; + consistant = FALSE; + g_debug("%s: charset different, clt:%s, srv:%s, server:%p", + G_STRLOC, con->client->charset->str, + pmd->server->charset->str, pmd->server); + } + + if (con->client->is_multi_stmt_set != pmd->server->is_multi_stmt_set) { + pmd->attr_diff |= ATTR_DIF_SET_OPTION; + con->unmatched_attribute |= ATTR_DIF_SET_OPTION; + result = FALSE; + consistant = FALSE; + g_debug("%s:set option different", G_STRLOC); + } + + if (con->is_start_trans_buffered || con->is_auto_commit_trans_buffered) { + if (con->is_tran_not_distributed_by_comment) { + pmd->attr_diff |= ATTR_DIF_SET_AUTOCOMMIT; + con->unmatched_attribute |= ATTR_DIF_SET_AUTOCOMMIT; + result = FALSE; + consistant = FALSE; + g_debug("%s:need sending autocommit or start transaction", G_STRLOC); + } + } + + if (consistant) { + pmd->attr_consistent = 1; + } + g_debug("%s:set checked for server:%p, query state:%d", G_STRLOC, pmd->server, + pmd->server->parse.qs_state); + pmd->attr_consistent_checked = 1; + } + + return result; +} + + +static gboolean check_user_consistant(network_mysqld_con *con) +{ + enum enum_server_command command = con->parse.command; + + if (command == COM_CHANGE_USER) { + return TRUE; + } + + size_t i; + gboolean result = TRUE; + + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + if (!pmd->participated || pmd->attr_consistent) { + continue; + } + + pmd->attr_adjusted_now = 0; + + if ((pmd->attr_diff & ATTR_DIF_CHANGE_USER) == 0) { + continue; + } + + GString *hashed_password = g_string_new(NULL); + const char *user = con->client->response->username->str; + cetus_users_get_hashed_server_pwd(con->srv->priv->users, user, hashed_password); + if (hashed_password->len == 0) { + g_warning("%s: user:%s hashed password is null", G_STRLOC, user); + g_string_free(hashed_password, TRUE); + result = FALSE; + break; + } else { + g_debug("%s: COM_CHANGE_USER:%d for server:%p", + G_STRLOC, COM_CHANGE_USER, pmd->server); + mysqld_change_user_packet_t chuser = {0}; + chuser.username = con->client->response->username; + chuser.auth_plugin_data = pmd->server->challenge->auth_plugin_data; + chuser.hashed_pwd = hashed_password; + chuser.database = con->client->default_db; + chuser.charset = con->client->charset_code; + + GString *payload = g_string_new(NULL); + mysqld_proto_append_change_user_packet(payload, &chuser); + + network_mysqld_queue_reset(pmd->server); + network_mysqld_queue_append(pmd->server, pmd->server->send_queue, S(payload)); + g_string_free(payload, TRUE); + + pmd->server->is_robbed = 0; + pmd->attr_adjusted_now = 1; + pmd->server->parse.qs_state = PARSE_COM_QUERY_INIT; + g_debug("%s: change user for server", G_STRLOC); + + con->attr_adj_state = ATTR_DIF_CHANGE_USER; + con->resp_expected_num++; + g_string_free(hashed_password, TRUE); + } + } + return result; +} + + +static void +build_xa_end_command(network_mysqld_con *con, server_session_t *pmd, int first) +{ + char buffer[64]; + + snprintf(buffer, sizeof(buffer), "XA END %s", con->xid_str); + + if (con->dist_tran_failed || con->is_rollback) { + pmd->dist_tran_state = NEXT_ST_XA_ROLLBACK; + if (first) { + con->dist_tran_state = NEXT_ST_XA_ROLLBACK; + con->state = ST_SEND_QUERY; + } + } else { + pmd->dist_tran_state = NEXT_ST_XA_PREPARE; + if (first) { + con->dist_tran_state = NEXT_ST_XA_PREPARE; + con->state = ST_SEND_QUERY; + } + } + + if (pmd->server->unavailable) { + return; + } + + g_debug("%s:XA END %s, server:%s", G_STRLOC, + con->xid_str, pmd->server->dst->name->str); + + pmd->server->parse.qs_state = PARSE_COM_QUERY_INIT; + + GString *srv_packet; + + srv_packet = g_string_sized_new(64); + srv_packet->len = NET_HEADER_SIZE; + g_string_append_c(srv_packet, (char) COM_QUERY); + g_string_append(srv_packet, buffer); + network_mysqld_proto_set_packet_len(srv_packet, 1 + strlen(buffer)); + network_mysqld_proto_set_packet_id(srv_packet, 0); + + g_queue_push_tail(pmd->server->send_queue->chunks, srv_packet); + + pmd->state = NET_RW_STATE_NONE; +} + +NETWORK_MYSQLD_PLUGIN_PROTO(proxy_get_server_conn_list) { + if (con->srv->complement_conn_cnt > 0) { + network_connection_pool_create_conn(con); + con->srv->complement_conn_cnt--; + } + + GList *chunk = con->client->recv_queue->chunks->head; + GString *packet = (GString *)(chunk->data); + gboolean do_query = FALSE; + int is_xa_query = 0; + + con->is_new_server_added = 0; + + if (!con->use_all_prev_servers) { + int server_unavailable = 0; + if (!proxy_add_server_connection_array(con, &server_unavailable)) { + record_last_backends_type(con); + if (!server_unavailable) { + return NETWORK_SOCKET_WAIT_FOR_EVENT; + } else { + return NETWORK_SOCKET_ERROR; + } + } else { + record_last_backends_type(con); + } + + do_query = check_and_set_attr_bitmap(con); + if (do_query == FALSE) { + g_debug("%s: check_and_set_attr_bitmap is different", G_STRLOC); + g_debug("%s: resp expect num:%d", G_STRLOC, con->resp_expected_num); + con->resp_expected_num = 0; + con->candidate_tcp_streamed = 0; + con->is_attr_adjust = 1; + if (con->unmatched_attribute & ATTR_DIF_CHANGE_USER) { + check_user_consistant(con); + } else if (con->unmatched_attribute & ATTR_DIF_DEFAULT_DB) { + shard_set_default_db_consistant(con); + con->attr_adj_state = ATTR_DIF_DEFAULT_DB; + } else if (con->unmatched_attribute & ATTR_DIF_CHARSET) { + shard_set_charset_consistant(con); + con->attr_adj_state = ATTR_DIF_CHARSET; + } else if (con->unmatched_attribute & ATTR_DIF_SET_OPTION) { + shard_set_multi_stmt_consistant(con); + con->attr_adj_state = ATTR_DIF_SET_OPTION; + } else if (con->unmatched_attribute & ATTR_DIF_SET_AUTOCOMMIT) { + g_debug("%s: autocommit adjust", G_STRLOC); + shard_set_autocommit(con); + con->attr_adj_state = ATTR_DIF_SET_AUTOCOMMIT; + } + + return NETWORK_SOCKET_SUCCESS; + } + } else { + do_query = TRUE; + } + + if (do_query == TRUE) { + if (con->attr_adj_state != ATTR_START) { + g_critical("%s: con->attr_adj_state is not ATTR_START:%p", G_STRLOC, con); + } + con->is_attr_adjust = 0; + con->attr_adj_state = ATTR_START; + + if (con->could_be_tcp_streamed) { + con->candidate_tcp_streamed = 1; + } + g_debug("%s: check_and_set_attr_bitmap is the same:%p", G_STRLOC, con); + if (con->dist_tran && !con->dist_tran_xa_start_generated) { + /* append xa query to send queue */ + chassis *srv = con->srv; + con->dist_tran_state = NEXT_ST_XA_QUERY; + con->xa_id = srv->dist_tran_id++; + snprintf(con->xid_str, XID_LEN, "'%s_%02d_%llu'", + srv->dist_tran_prefix, tc_get_log_hour(), con->xa_id); + con->dist_tran_xa_start_generated = 1; + + con->is_start_trans_buffered = 0; + con->is_auto_commit_trans_buffered = 0; + + g_debug("%s:xa start:%s for con:%p", G_STRLOC, con->xid_str, con); + } + + size_t i; + + con->resp_expected_num = 0; + g_debug("%s: server num:%d", G_STRLOC, con->servers->len); + + gboolean xa_start_phase = FALSE; + if (con->dist_tran) { + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + if (!pmd->xa_start_already_sent) { + xa_start_phase = TRUE; + g_debug("%s: start phase is true:%d", G_STRLOC, (int) i); + break; + } + } + } + + int is_first_xa_query = 0; + char xa_log_buffer[XA_LOG_BUF_LEN] = {0}; + char *p_xa_log_buffer = xa_log_buffer; + + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + + if (con->dist_tran && !pmd->dist_tran_participated) { + g_debug("%s: omit it for server:%p", G_STRLOC, pmd->server); + continue; + } + + if (pmd->server->unavailable) { + continue; + } + + g_debug("%s:packet id:%d when get server", + G_STRLOC, pmd->server->last_packet_id); + + pmd->server->parse.qs_state = PARSE_COM_QUERY_INIT; + + if (con->dist_tran) { + pmd->xa_start_already_sent = 1; + if (pmd->dist_tran_state == NEXT_ST_XA_START) { + g_debug("%s:pmd start phase:%d", G_STRLOC, (int) i); + } else { + g_debug("%s:pmd not start phase:%d", G_STRLOC, (int) i); + } + + if (pmd->dist_tran_state == NEXT_ST_XA_START) { + network_mysqld_send_xa_start(pmd->server, con->xid_str); + pmd->dist_tran_state = NEXT_ST_XA_QUERY; + pmd->xa_start_already_sent = 0; + con->xa_start_phase = 1; + g_debug("%s:pmd start phase:%d", G_STRLOC, (int) i); + } else if (pmd->dist_tran_state == NEXT_ST_XA_OVER) { + g_debug("%s:omit here for server:%p", G_STRLOC, pmd->server); + continue; + } else { + if (con->is_commit_or_rollback /* current sql */ + || con->dist_tran_failed) { + pmd->dist_tran_state = NEXT_ST_XA_END; + pmd->participated = 1; + build_xa_end_command(con, pmd, 1); + if (con->dist_tran_failed) { + network_queue_clear(con->client->recv_queue); + network_mysqld_queue_reset(con->client); + } + } else { + if (!pmd->participated) { + g_debug("%s:omit here for server:%p", G_STRLOC, pmd->server); + continue; + } + if (xa_start_phase) { + g_debug("%s:omit here for server:%p", G_STRLOC, pmd->server); + continue; + } + pmd->dist_tran_state = NEXT_ST_XA_QUERY; + if (is_first_xa_query) { + p_xa_log_buffer[0] = ','; + p_xa_log_buffer++; + } else { + is_first_xa_query = 1; + } + snprintf(p_xa_log_buffer, XA_LOG_BUF_LEN - (p_xa_log_buffer - xa_log_buffer), + "%s@%d", pmd->server->dst->name->str, pmd->server->challenge->thread_id); + p_xa_log_buffer = p_xa_log_buffer + strlen(p_xa_log_buffer); + shard_build_xa_query(con, pmd); + is_xa_query = 1; + if (con->is_auto_commit) { + pmd->dist_tran_state = NEXT_ST_XA_END; + g_debug("%s:set dist_tran_state xa end for con:%p", + G_STRLOC, con); + } + } + } + } else { + if (con->parse.command == COM_QUERY) { + GString *payload = g_string_new(0); + network_mysqld_proto_append_query_packet(payload, pmd->sql->str); + network_mysqld_queue_reset(pmd->server); + network_mysqld_queue_append(pmd->server, pmd->server->send_queue, + S(payload)); + g_string_free(payload, TRUE); + } else { + network_queue_append(pmd->server->send_queue, + g_string_new_len(packet->str, packet->len)); + } + } + + if (!is_xa_query) { + con->resp_expected_num++; + pmd->state = NET_RW_STATE_NONE; + } + } + + if (is_xa_query) { + if (con->srv->xa_log_detailed) { + tc_log_info(LOG_INFO, 0, "XA QUERY %s %s %s", + con->xid_str, xa_log_buffer, con->orig_sql->str); + } + network_queue_clear(con->client->recv_queue); + } else { + if (!con->dist_tran) { + network_queue_clear(con->client->recv_queue); + } + } + } + + return NETWORK_SOCKET_SUCCESS; +} + + +/** + * decide about the next state after the result-set has been written + * to the client + * + * if we still have data in the queue, back to proxy_send_query() + * otherwise back to proxy_read_query() to pick up a new client query + * + * @note we should only send one result back to the client + */ +NETWORK_MYSQLD_PLUGIN_PROTO(proxy_send_query_result) { + if (con->server_to_be_closed) { + if (con->servers != NULL) { + g_debug("%s:call proxy_put_shard_conn_to_pool for con:%p", G_STRLOC, con); + proxy_put_shard_conn_to_pool(con); + + if (con->srv->maintain_close_mode) { + con->state = ST_CLOSE_CLIENT; + g_debug("%s:client needs to closed for con:%p", G_STRLOC, con); + } else { + con->state = ST_READ_QUERY; + } + return NETWORK_SOCKET_SUCCESS; + } + } + + if (con->is_changed_user_failed) { + con->is_changed_user_failed = 0; + con->state = ST_ERROR; + return NETWORK_SOCKET_SUCCESS; + } + + con->state = ST_READ_QUERY; + + if (con->srv->maintain_close_mode) { + if (!con->is_in_transaction) { + con->state = ST_CLOSE_CLIENT; + g_debug("%s:client needs to closed for con:%p", G_STRLOC, con); + } + } + return NETWORK_SOCKET_SUCCESS; +} + +/** + * connect to a backend + * + * @return + * NETWORK_SOCKET_SUCCESS - connected successfully + * NETWORK_SOCKET_ERROR_RETRY - connecting backend failed, + * call again to connect to another backend + * NETWORK_SOCKET_ERROR - no backends available, + * adds a ERR packet to the client queue + */ +NETWORK_MYSQLD_PLUGIN_PROTO(proxy_connect_server) { + shard_plugin_con_t *st = con->plugin_con_state; + return do_connect_cetus(con, &st->backend, &st->backend_ndx); +} + +NETWORK_MYSQLD_PLUGIN_PROTO(proxy_init) { + chassis_plugin_config *config = con->config; + + g_assert(con->plugin_con_state == NULL); + + shard_plugin_con_t *st = shard_plugin_con_new(); + + /* TODO: this should inside "st"_new, but now "st" shared by many plugins */ + st->sql_context = g_new0(sql_context_t, 1); + sql_context_init(st->sql_context); + st->trx_read_write = TF_READ_WRITE; + st->trx_isolation_level = TF_REPEATABLE_READ; + + con->plugin_con_state = st; + + con->state = ST_CONNECT_SERVER; + + /* set the connection specific timeouts + * + * TODO: expose these settings at runtime + */ + if (config->connect_timeout_dbl >= 0) { + chassis_timeval_from_double(&con->connect_timeout, config->connect_timeout_dbl); + } + if (config->read_timeout_dbl >= 0) { + chassis_timeval_from_double(&con->read_timeout, config->read_timeout_dbl); + } + if (config->write_timeout_dbl >= 0) { + chassis_timeval_from_double(&con->write_timeout, config->write_timeout_dbl); + } + + return NETWORK_SOCKET_SUCCESS; +} + +static int proxy_c_disconnect_shard_client(network_mysqld_con *con) +{ + if (con->is_in_transaction || con->is_auto_commit == 0) { + if (con->is_in_transaction) { + g_message("%s: con is still in trans for con:%p", G_STRLOC, con); + } + + if (!con->server_to_be_closed) { + if (con->dist_tran_state != NEXT_ST_XA_OVER) { + con->server_to_be_closed = 1; + } + } + } + + if (con->servers) { + g_debug("%s:call proxy_put_shard_conn_to_pool for con:%p", G_STRLOC, con); + proxy_put_shard_conn_to_pool(con); + } + + return PROXY_NO_DECISION; +} + +/** + * cleanup the proxy specific data on the current connection + * + * move the server connection into the connection pool in case it is a + * good client-side close + * + * @return NETWORK_SOCKET_SUCCESS + * @see plugin_call_cleanup + */ +NETWORK_MYSQLD_PLUGIN_PROTO(proxy_disconnect_client) { + shard_plugin_con_t *st = con->plugin_con_state; + + if (st == NULL) return NETWORK_SOCKET_SUCCESS; + + if (con->servers != NULL) { + g_debug("%s: call proxy_c_disconnect_shard_client:%p", G_STRLOC, con); + proxy_c_disconnect_shard_client(con); + } + + if (con->sharding_plan != NULL) { + sharding_plan_free(con->sharding_plan); + con->sharding_plan = NULL; + } + + network_mysqld_con_reset_query_state(con); + + /* TODO: this should inside "st"_free, but now "st" shared by many plugins */ + if (st->sql_context) { + sql_context_destroy(st->sql_context); + g_free(st->sql_context); + st->sql_context = NULL; + } + + shard_plugin_con_free(con, st); + + con->plugin_con_state = NULL; + + g_debug("%s: set plugin_con_state null:%p", G_STRLOC, con); + + return NETWORK_SOCKET_SUCCESS; +} + + +int network_mysqld_shard_connection_init(network_mysqld_con *con) { + con->plugins.con_init = proxy_init; + con->plugins.con_connect_server = proxy_connect_server; + con->plugins.con_read_handshake = NULL; + con->plugins.con_read_auth = proxy_read_auth; + con->plugins.con_read_auth_result = NULL; + con->plugins.con_read_auth_old_password = NULL; + con->plugins.con_read_query = proxy_read_query; + con->plugins.con_get_server_conn_list = proxy_get_server_conn_list; + con->plugins.con_read_query_result = NULL; + con->plugins.con_send_query_result = proxy_send_query_result; + con->plugins.con_cleanup = proxy_disconnect_client; + con->plugins.con_timeout = proxy_timeout; + + return 0; +} + +static chassis_plugin_config *network_mysqld_shard_plugin_new(void) { +#ifdef SIMPLE_PARSER + g_critical("try loading shard-plugin.so from rw-edition, exit"); + exit(1); +#endif + chassis_plugin_config *config; + + config = g_new0(chassis_plugin_config, 1); + + /* use negative values as defaults to make them ignored */ + config->connect_timeout_dbl = -1.0; + config->read_timeout_dbl = -1.0; + config->write_timeout_dbl = -1.0; + + return config; +} + +void network_mysqld_proxy_free(network_mysqld_con G_GNUC_UNUSED *con) { +} + +void network_mysqld_shard_plugin_free(chassis *chas, chassis_plugin_config *config) { + + g_strfreev(config->backend_addresses); + g_strfreev(config->read_only_backend_addresses); + + if (config->address) { + /* free the global scope */ + network_mysqld_proxy_free(NULL); + chassis_config_unregister_service(chas->config_manager, config->address); + g_free(config->address); + } + sql_filter_vars_destroy(); + g_debug("%s: call shard_conf_destroy", G_STRLOC); + shard_conf_destroy(); + + g_free(config); +} + +/** + * plugin options + */ +static GList * +network_mysqld_shard_plugin_get_options(chassis_plugin_config *config) +{ + chassis_options_t opts = {0}; + + chassis_options_add(&opts, "proxy-address", + 'P', 0, OPTION_ARG_STRING, &(config->address), + "listening address:port of the proxy-server (default: :4040)", + ""); + + chassis_options_add(&opts, "proxy-backend-addresses", + 'b', 0, OPTION_ARG_STRING_ARRAY, &(config->backend_addresses), + "address:port of the remote backend-servers (default: 127.0.0.1:3306)", + ""); + + chassis_options_add(&opts, "proxy-read-only-backend-addresses", + 'r', 0, OPTION_ARG_STRING_ARRAY, &(config->read_only_backend_addresses), + "address:port of the remote slave-server (default: not set)", + ""); + + chassis_options_add(&opts, "proxy-connect-timeout", + 0, 0, OPTION_ARG_DOUBLE, &(config->connect_timeout_dbl), + "connect timeout in seconds (default: 2.0 seconds)", + NULL); + + chassis_options_add(&opts, "proxy-read-timeout", + 0, 0, OPTION_ARG_DOUBLE, &(config->read_timeout_dbl), + "read timeout in seconds (default: 10 minuates)", NULL); + + chassis_options_add(&opts, "proxy-write-timeout", + 0, 0, OPTION_ARG_DOUBLE, &(config->write_timeout_dbl), + "write timeout in seconds (default: 10 minuates)", NULL); + + chassis_options_add(&opts, "proxy-allow-ip", + 0, 0, OPTION_ARG_STRING, &(config->allow_ip), + "allow user@IP for proxy permission", NULL); + + chassis_options_add(&opts, "proxy-deny-ip", + 0, 0, OPTION_ARG_STRING, &(config->deny_ip), + "deny user@IP for proxy permission", NULL); + + return opts.options; +} + +void sharding_conf_reload_callback(int fd, short what, void *arg) +{ + chassis *chas = arg; + char *shard_json = NULL; + gboolean ok = chassis_config_query_object(chas->config_manager, + "sharding", &shard_json); + if (!ok || !shard_json) { + g_critical("error on sharding configuration reloading."); + } + if (shard_conf_load(shard_json)) { + g_message("sharding config is updated"); + } else { + g_warning("sharding config update failed"); + } + g_free(shard_json); +} + +/** + * init the plugin with the parsed config + */ +static int network_mysqld_shard_plugin_apply_config(chassis *chas, + chassis_plugin_config *config) +{ + network_mysqld_con *con; + network_socket *listen_sock; + chassis_private *g = chas->priv; + + if (!config->address) config->address = g_strdup(":4040"); + if (!config->backend_addresses) { + config->backend_addresses = g_new0(char *, 2); + config->backend_addresses[0] = g_strdup("127.0.0.1:3306"); + config->backend_addresses[1] = NULL; + } + + /* set allow_ip_table */ + GHashTable *allow_ip_table = NULL; + if (config->allow_ip) { + allow_ip_table = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + char **ip_arr = g_strsplit(config->allow_ip, ",", -1); + guint j; + for (j = 0; ip_arr[j]; j++) { + g_hash_table_insert(allow_ip_table, g_strdup(ip_arr[j]), (void *) TRUE); + } + g_strfreev(ip_arr); + } + config->allow_ip_table = allow_ip_table; + + /* set deny_ip_table */ + GHashTable *deny_ip_table = NULL; + if (config->deny_ip) { + deny_ip_table = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + char **ip_arr = g_strsplit(config->deny_ip, ",", -1); + guint j; + for (j = 0; ip_arr[j]; j++) { + g_hash_table_insert(deny_ip_table, g_strdup(ip_arr[j]), (void *) TRUE); + } + g_strfreev(ip_arr); + } + config->deny_ip_table = deny_ip_table; + + /** + * create a connection handle for the listen socket + */ + con = network_mysqld_con_new(); + network_mysqld_add_connection(chas, con, TRUE); + con->config = config; + + config->listen_con = con; + + listen_sock = network_socket_new(); + con->server = listen_sock; + + /* + * set the plugin hooks as we want to apply them + * to the new connections too later + */ + network_mysqld_shard_connection_init(con); + + if (network_address_set_address(listen_sock->dst, config->address)) { + return -1; + } + + if (network_socket_bind(listen_sock)) { + return -1; + } + g_message("shard module listening on port %s, con:%p", config->address, con); + + plugin_add_backends(chas, config->backend_addresses, + config->read_only_backend_addresses); + + char *shard_json = NULL; + gboolean ok = chassis_config_query_object(chas->config_manager, + "sharding", &shard_json); + if (!ok || !shard_json || !shard_conf_load(shard_json)) { + g_critical("sharding configuration load error, exit program."); + exit(0); + } + g_free(shard_json); + + g_assert(chas->priv->monitor); + cetus_monitor_register_object(chas->priv->monitor, "sharding", + sharding_conf_reload_callback, chas); + + /** + * call network_mysqld_con_accept() with this connection when we are done + */ + + event_set(&(listen_sock->event), listen_sock->fd, + EV_READ|EV_PERSIST, network_mysqld_con_accept, con); + event_base_set(chas->event_base, &(listen_sock->event)); + event_add(&(listen_sock->event), NULL); + g_debug("%s:listen sock, ev:%p",G_STRLOC, (&listen_sock->event)); + + if (network_backends_load_config(g->backends, chas) != -1) { + network_connection_pool_create_conns(chas); + } + chassis_config_register_service(chas->config_manager, config->address, "shard"); + + sql_filter_vars_shard_load_default_rules(); + char *variable_conf = g_build_filename(chas->conf_dir, "variables.json", NULL); + if (g_file_test(variable_conf, G_FILE_TEST_IS_REGULAR)) { + g_message("reading variable rules from %s", variable_conf); + gboolean ok = sql_filter_vars_load_rules(variable_conf); + if (!ok) g_warning("variable rule load error"); + } + g_free(variable_conf); + + + return 0; +} + +GList *network_mysqld_shard_plugin_allow_ip_get(chassis_plugin_config *config) { + if (config && config->allow_ip_table) { + return g_hash_table_get_keys(config->allow_ip_table); + } + return NULL; +} + +GList *network_mysqld_shard_plugin_deny_ip_get(chassis_plugin_config *config) { + if (config && config->deny_ip_table) { + return g_hash_table_get_keys(config->deny_ip_table); + } + return NULL; +} + +static gboolean +network_mysqld_shard_plugin_allow_ip_add(chassis_plugin_config *config, char *addr) +{ + if (!config || !addr) return FALSE; + if (!config->allow_ip_table) { + config->allow_ip_table = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + } + gboolean success = FALSE; + if (!g_hash_table_lookup(config->allow_ip_table, addr)) { + g_hash_table_insert(config->allow_ip_table, g_strdup(addr), (void *) TRUE); + success = TRUE; + } + return success; +} + +static gboolean +network_mysqld_shard_plugin_deny_ip_add(chassis_plugin_config *config, char *addr) +{ + if (!config || !addr) return FALSE; + if (!config->deny_ip_table) { + config->deny_ip_table = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + } + gboolean success = FALSE; + if (!g_hash_table_lookup(config->deny_ip_table, addr)) { + g_hash_table_insert(config->deny_ip_table, g_strdup(addr), (void *) TRUE); + success = TRUE; + } + return success; +} + +static gboolean +network_mysqld_shard_plugin_allow_ip_del(chassis_plugin_config *config, char *addr) +{ + if (!config || !addr || !config->allow_ip_table) return FALSE; + return g_hash_table_remove(config->allow_ip_table, addr); +} + +static gboolean +network_mysqld_shard_plugin_deny_ip_del(chassis_plugin_config *config, char *addr) +{ + if (!config || !addr || !config->deny_ip_table) return FALSE; + return g_hash_table_remove(config->deny_ip_table, addr); +} + +G_MODULE_EXPORT int plugin_init(chassis_plugin *p) { + p->magic = CHASSIS_PLUGIN_MAGIC; + p->name = g_strdup("shard"); + p->version = g_strdup(PLUGIN_VERSION); + + p->init = network_mysqld_shard_plugin_new; + p->get_options = network_mysqld_shard_plugin_get_options; + p->apply_config = network_mysqld_shard_plugin_apply_config; + p->destroy = network_mysqld_shard_plugin_free; + + /* For allow_ip configs */ + p->allow_ip_get = network_mysqld_shard_plugin_allow_ip_get; + p->allow_ip_add = network_mysqld_shard_plugin_allow_ip_add; + p->allow_ip_del = network_mysqld_shard_plugin_allow_ip_del; + + /* For deny_ip configs */ + p->deny_ip_get = network_mysqld_shard_plugin_deny_ip_get; + p->deny_ip_add = network_mysqld_shard_plugin_deny_ip_add; + p->deny_ip_del = network_mysqld_shard_plugin_deny_ip_del; + return 0; +} diff --git a/plugins/shard/sharding-parser.c b/plugins/shard/sharding-parser.c new file mode 100644 index 0000000..f9fcfc1 --- /dev/null +++ b/plugins/shard/sharding-parser.c @@ -0,0 +1,1708 @@ +#include "sharding-parser.h" + +#include +#include +#include +#include +#include + +#include "glib-ext.h" +#include "sql-expression.h" +#include "sql-construction.h" +#include "sql-property.h" +#include "sharding-config.h" + +static gboolean is_compare_op(int op) { + return op == TK_EQ || op == TK_NE + || op == TK_LT || op == TK_LE || op == TK_GT || op == TK_GE; +} + +static gboolean is_logical_op(int op) { + return op == TK_AND || op == TK_OR || op == TK_NOT; +} + +static gboolean is_arithmetic_op(int op) { + return op == TK_PLUS || op == TK_MINUS || op == TK_STAR + || op == TK_SLASH || op == TK_REM; +} + +static gboolean expr_is_sharding_key(sql_expr_t *p, const sql_src_item_t *tb, + const char *key) { + if (p->op == TK_DOT) {/* table.colmun or db.table.col */ + assert(tb->table_name); + assert(p->left && p->right); + if (p->right->op == TK_DOT) { /* db.table.col */ + if (tb->dbname && strcasecmp(p->left->token_text, tb->dbname)==0 + && sql_expr_is_dotted_name(p->right, tb->table_name, key)) + return TRUE; + } else if (strcasecmp(p->right->token_text, key)==0) { + char* prefix = p->left->token_text; + if (strcasecmp(prefix, tb->table_name)==0) { + return TRUE; + } + if (tb->table_alias && strcmp(prefix, tb->table_alias) == 0) { + return TRUE; + } + } + } else if (p->op == TK_ID) { + if (strcasecmp(p->token_text, key) == 0) + return TRUE; + } + return FALSE; +} + +static gboolean expr_is_compound_value(sql_expr_t *p) +{ + if (!p) + return FALSE; + return p->op == TK_FUNCTION || + p->op == TK_SELECT || /* subquery */ + p->op == TK_ID || p->op == TK_DOT || /* col, tbl.col */ + is_arithmetic_op(p->op); /* 1+1, 3%2, 4 * 5, etc */ +} + +static unsigned int +supplemental_hash(unsigned int value) +{ + unsigned int tmp1 = value >> 20; + unsigned int tmp2 = value >> 12; + unsigned int tmp3 = tmp1 ^ tmp2; + unsigned int h = value ^ tmp3; + tmp1 = h >> 7; + tmp2 = h >> 4; + tmp3 = tmp1 ^ tmp2; + h = h ^ tmp3; + return h; +} + +static unsigned int +cetus_str_hash(const unsigned char *key) +{ + int len = strlen((const char *) key); + unsigned int hashcode_head = 0; + int i = 0; + int max = 8; + + if (max > len) { + max = len; + } + for (; i < max; i++) { + hashcode_head <<= 4; + hashcode_head += key[i]; + } + if (len > max) { + i = len - 8; + unsigned int hashcode_tail = 0; + for (; i < len; i++) { + hashcode_tail <<= 4; + hashcode_tail += key[i]; + } + return supplemental_hash(hashcode_head ^ hashcode_tail); + } else { + return supplemental_hash(hashcode_head); + } +} + + +static GString *sql_modify_limit(sql_select_t *select) +{ + GString *new_sql = NULL; + if (select->offset && select->offset->num_value > 0 && select->limit) {/* TODO: +ddd */ + guint64 limit_bak = 0; + guint64 offset_bak = 0; + if (select->offset && select->offset->op == TK_INTEGER) { + offset_bak = select->offset->num_value; + select->offset->num_value = 0; + } + if (select->limit->op == TK_INTEGER) { + limit_bak = select->limit->num_value; + select->limit->num_value += offset_bak; + } + new_sql = sql_construct_select(select); + g_string_append_c(new_sql, ';'); + select->limit->num_value = limit_bak; + if (select->offset) + select->offset->num_value = offset_bak; + } + /* union statement */ + if (new_sql && select->prior) { + select = select->prior; + GString *union_sql = g_string_new(NULL); + while (select) { + GString *sql = sql_construct_select(select); + g_string_append(union_sql, sql->str); + g_string_append(union_sql, " UNION "); + g_string_free(sql, TRUE); + select = select->prior; + } + g_string_append(union_sql, new_sql->str); + g_string_free(new_sql, TRUE); + return union_sql; + } else { + return new_sql; + } +} + +static GString *sql_modify_orderby(sql_select_t *select) +{ + GString *new_sql = NULL; + if (select->flags & SF_REWRITE_ORDERBY) { + sql_expr_list_t *columns = select->columns; + sql_column_list_t *orderby = NULL; + int i = 0; + if (select->orderby_clause) { + orderby = select->orderby_clause; + } + for (i = 0; i < columns->len; ++i) { + sql_expr_t *mcol = g_ptr_array_index(columns, i); + if (!(mcol->flags & EP_ORDER_BY)) { + sql_column_t *ordcol = sql_column_new(); + ordcol->expr = sql_expr_dup(mcol); + orderby = sql_column_list_append(orderby, ordcol); + } + } + select->orderby_clause = orderby; + new_sql = sql_construct_select(select); + g_string_append_c(new_sql, ';'); + } + /* union statement */ + if (new_sql && select->prior) { + select = select->prior; + GString *union_sql = g_string_new(NULL); + while (select) { + GString *sql = sql_construct_select(select); + g_string_append(union_sql, sql->str); + g_string_append(union_sql, " UNION "); + g_string_free(sql, TRUE); + select = select->prior; + } + g_string_append(union_sql, new_sql->str); + g_string_free(new_sql, TRUE); + return union_sql; + } else { + return new_sql; + } +} + +GString * +sharding_modify_sql(sql_context_t *context, having_condition_t *hav_condi) +{ + /* TODO: sql rewrite priority */ + if (context->stmt_type == STMT_SELECT && context->sql_statement) { + sql_select_t *select = context->sql_statement; + + sql_expr_t *having = select->having_clause; + if (having) { + if (is_compare_op(having->op)) { + hav_condi->rel_type = having->op; + sql_expr_t *val = having->right; + if (hav_condi->condition_value) { + g_free(hav_condi->condition_value); + hav_condi->condition_value = NULL; + } + + char *val_str = g_strndup(val->start, val->end - val->start); + sql_string_dequote(val_str); + hav_condi->condition_value = val_str; + if (val->op == TK_UMINUS || val->op == TK_UPLUS) { + hav_condi->data_type = TK_INTEGER; + } else { + hav_condi->data_type = val->op; + } + } + select->having_clause = NULL;/* temporarily remove HAVING */ + } + gboolean has_function = FALSE; + GString *modified_sql = NULL; + + /* (LIMIT a, b) ==> (LIMIT 0, a+b) */ + if (modified_sql == NULL && !has_function) { + modified_sql = sql_modify_limit(select); + } + + /* select DISTINCT x,y,z ==> select DISTINCT x,y,z ORDER BY x,y,z */ + if (modified_sql == NULL) { + modified_sql = sql_modify_orderby(select); + } + + if (modified_sql == NULL && having) { + modified_sql = sql_construct_select(select); + } + select->having_clause = having; /* get HAVING back */ + + return modified_sql; + } + return NULL; +} + +static gboolean sql_select_contains_sharding_table(sql_select_t *select, + char **current_db /* in_out */, char **table /* out */) +{ + char *db = *current_db; + while (select) { + sql_src_list_t *sources = select->from_src; + int i = 0; + for (i = 0; sources && i < sources->len; ++i) { + sql_src_item_t *src = g_ptr_array_index(sources, i); + if (src->dbname) { + db = src->dbname; + } + if (src->table_name) { + if (shard_conf_is_shard_table(db, src->table_name)) { + *current_db = db; + *table = src->table_name; + return TRUE; + } + } + } + select = select->prior; + } + return FALSE; +} + +typedef struct condition_t { + int op; + union { + gint64 num; + const char *str; + }v; +} condition_t; + +/** + * check if the group satisfies an inequation + * suppose condition is "Greater Than 42", (op = TK_GT, v.num = 42) + * if exists X -> (low, high] that satifies X > 42, return true + * if it doesn't exist such an X, return false; + */ +static gboolean partition_satisfies(sharding_partition_t *partition, condition_t cond) +{ + /* partition value -> (low, high] */ + const sharding_vdb_t *conf = partition->vdb; + if (conf->method == SHARD_METHOD_HASH) { + int64_t hash_value = (conf->key_type == SHARD_DATA_TYPE_STR) + ? cetus_str_hash((const unsigned char *) cond.v.str) : cond.v.num; + + int32_t hash_mod = hash_value % conf->logic_shard_num; + + if (cond.op == TK_EQ) { + return sharding_partition_contain_hash(partition, hash_mod); + } else { + return TRUE; + } + } + /* vvv SHARD_METHOD_RANGE vvv */ + if (conf->key_type == SHARD_DATA_TYPE_STR) { + const char *low = partition->low_value; + const char *high = partition->value; // high is NULL means unlimited + const char *val = cond.v.str; + switch (cond.op) { + case TK_EQ: + return (low == NULL || strcmp(val, low) > 0) && (high == NULL || strcmp(val, high) <= 0); + case TK_GT: + return high == NULL || strcmp(val, high) < 0; + case TK_LT: + return low == NULL || strcmp(val, low) > 0; + case TK_GE: + return high == NULL || strcmp(val, high) <= 0; + case TK_LE: + return low == NULL || strcmp(val, low) > 0; + case TK_NE: + return TRUE; + default: + g_warning(G_STRLOC ":error condition"); + } + } else {/* int and datetime */ + int low = (int) (int64_t)partition->low_value; + int high = (int) (int64_t)partition->value; /* high range OR hash value */ + int val = cond.v.num; + switch (cond.op) { + case TK_EQ: + return val > low && val <= high; + case TK_GT: + return val < high; + case TK_LT: + return val > low+1; + case TK_GE: + return val <= high; + case TK_LE: + return val > low; + case TK_NE: + if (val == high && high == (low+1)) { + return FALSE; + } else { + return TRUE; + } + default: + g_warning(G_STRLOC ":error condition"); + return FALSE; + } + } + + g_warning(G_STRLOC ":error reach here"); + return false; +} + +/* filter out those which not satisfy cond */ +static void partitions_filter(GPtrArray *partitions, condition_t cond) +{ + int i = 0; + for (i = 0; i < partitions->len; ++i) { + sharding_partition_t *gp = g_ptr_array_index(partitions, i); + if (!partition_satisfies(gp, cond)) { + g_ptr_array_remove_index(partitions, i); + --i; + } + } +} + +/* collect those which satisfy cond */ +static void partitions_collect(GPtrArray *from_partitions, condition_t cond, + GPtrArray *to_partitions) +{ + int i = 0; + for (i = 0; i < from_partitions->len; ++i) { + sharding_partition_t *gp = g_ptr_array_index(from_partitions, i); + if (partition_satisfies(gp, cond)) { + g_ptr_array_add(to_partitions, gp); + } + } +} + +/* get first group that satisfies cond */ +sharding_partition_t *partitions_get(GPtrArray *from_partitions, condition_t cond) +{ + int i = 0; + for (i = 0; i < from_partitions->len; ++i) { + sharding_partition_t *gp = g_ptr_array_index(from_partitions, i); + if (partition_satisfies(gp, cond)) { + return gp; + } + } + return NULL; +} + +static void partitions_merge(GPtrArray *partitions, GPtrArray *other) +{ + int i = 0; + for (i = 0; i < other->len; ++i) { + sharding_partition_t *gp = g_ptr_array_index(other, i); + g_ptr_array_add(partitions, gp); + /* duplication check is performed later */ + } +} + +static GPtrArray *partitions_dup(GPtrArray *partitions) +{ + int i = 0; + GPtrArray *dup = g_ptr_array_new(); + for (i = 0; i < partitions->len; ++i) { + sharding_partition_t *gp = g_ptr_array_index(partitions, i); + g_ptr_array_add(dup, gp); + } + return dup; +} + +/** + * parse cond.v from a string + */ +static int string_to_sharding_value(const char *str, int expected, condition_t *cond) +{ + assert(cond); + if (expected == SHARD_DATA_TYPE_STR) { + cond->v.str = str; + } else if (expected == SHARD_DATA_TYPE_DATE + || expected == SHARD_DATA_TYPE_DATETIME) { + gboolean ok; + cond->v.num = chassis_epoch_from_string(str, &ok); + if (!ok) { + g_warning(G_STRLOC ":error datetime format: %s", str); + return PARSE_ERROR; + } + } else if (expected == SHARD_DATA_TYPE_INT) { + char *endptr = NULL; + cond->v.num = strtol(str, &endptr, 10); + if (str == endptr || *endptr != '\0') { + g_warning(G_STRLOC ":cannot get INT from string token: %s", str); + return PARSE_ERROR; + } + } else { + g_warning(G_STRLOC ":unexpected key string: %s", str); + return PARSE_ERROR; + } + return PARSE_OK; +} + +/** + * parse cond.v from sql expression + */ +static int expr_parse_sharding_value(sql_expr_t *p, int expected, condition_t *cond) +{ + assert(p); + assert(cond); + gint64 intval; + if (sql_expr_get_int(p, &intval) && expected == SHARD_DATA_TYPE_INT) { + cond->v.num = intval; + } else if (p->op == TK_STRING) { + return string_to_sharding_value(p->token_text, expected, cond); + } else if (expr_is_compound_value(p)) { + g_debug(G_STRLOC ":compound value, use all shard"); + return PARSE_UNRECOGNIZED; + } else { + g_warning(G_STRLOC ":unexpected token type: %d, %s", p->op, p->token_text); + return PARSE_ERROR; + } + return PARSE_OK; +} + +static int partitions_filter_inequation_expr(GPtrArray *partitions, sql_expr_t *expr) +{ + const sharding_vdb_t *conf = NULL; + g_assert(partitions); + if (partitions->len > 0) { + sharding_partition_t *gp = g_ptr_array_index(partitions, 0); + conf = gp->vdb; + } else { + return PARSE_OK; + } + + condition_t cond = {0}; + cond.op = expr->op; + int rc = expr_parse_sharding_value(expr->right, conf->key_type, &cond); + if (rc != PARSE_OK) + return rc; + partitions_filter(partitions, cond); + return PARSE_OK; +} + +static int partitions_filter_BETWEEN_expr(GPtrArray *partitions, sql_expr_t *expr) +{ + const sharding_vdb_t *conf = NULL; + g_assert(partitions); + if (partitions->len > 0) { + sharding_partition_t *gp = g_ptr_array_index(partitions, 0); + conf = gp->vdb; + } else { + return PARSE_OK; + } + + condition_t cond = {0}; + sql_expr_list_t *btlist = expr->list; + if (btlist && btlist->len == 2) { + sql_expr_t *low = g_ptr_array_index(btlist, 0); + sql_expr_t *high = g_ptr_array_index(btlist, 1); + cond.op = TK_GT; + int rc = expr_parse_sharding_value(low, conf->key_type, &cond); + if (rc != PARSE_OK) + return rc; + partitions_filter(partitions, cond); + + cond.op = TK_LT; + rc = expr_parse_sharding_value(high, conf->key_type, &cond); + if (rc != PARSE_OK) + return rc; + partitions_filter(partitions, cond); + } + return PARSE_OK; +} + +static int partitions_collect_IN_expr(GPtrArray *partitions, sql_expr_t *expr) +{ + const sharding_vdb_t *conf = NULL; + g_assert(partitions); + if (partitions->len > 0) { + sharding_partition_t *gp = g_ptr_array_index(partitions, 0); + conf = gp->vdb; + } else { + return PARSE_OK; + } + + condition_t cond = {0}; + if (expr->list && expr->list->len > 0) { + GPtrArray *partitions = g_ptr_array_new(); + + sql_expr_list_t *args = expr->list; + int i; + for (i = 0; i < args->len; ++i) { + sql_expr_t *arg = g_ptr_array_index(args, i); + cond.op = TK_EQ; + int rc = expr_parse_sharding_value(arg, conf->key_type, &cond); + if (rc != PARSE_OK) { + g_ptr_array_free(partitions, TRUE); + return rc; + } + partitions_collect(partitions, cond, partitions); + } + + /* transfer partitions to partitions as output */ + for (i = partitions->len - 1; i >= 0; --i) { + g_ptr_array_remove_index(partitions, i); + } + for (i = 0; i < partitions->len; ++i) { + gpointer *gp = g_ptr_array_index(partitions, i); + g_ptr_array_add(partitions, gp); + } + g_ptr_array_free(partitions, TRUE); + return PARSE_OK; + + } else { + return PARSE_UNRECOGNIZED; + } +} + +static int partitions_filter_expr(GPtrArray *partitions, sql_expr_t *expr) +{ + g_assert(partitions); + if (partitions->len == 0) { + return PARSE_OK; + } + + GQueue *stack = g_queue_new(); + int rc = PARSE_OK; + g_queue_push_head(stack, expr); + + /* for each condition in where clause */ + while (!g_queue_is_empty(stack)) {/* TODO: NOT op is not supported */ + sql_expr_t *p = g_queue_pop_head(stack); + if (p->op == TK_OR) { + GPtrArray *new_partitions = partitions_dup(partitions); + rc = partitions_filter_expr(partitions, p->left); + if (rc != PARSE_OK) { + g_ptr_array_free(new_partitions, TRUE); + break; + } + rc = partitions_filter_expr(new_partitions, p->right); + if (rc != PARSE_OK) { + g_ptr_array_free(new_partitions, TRUE); + break; + } + partitions_merge(partitions, new_partitions); + g_ptr_array_free(new_partitions, TRUE); + continue; + } + if (p->op == TK_AND) { + if (p->right) + g_queue_push_head(stack, p->right); + if (p->left) + g_queue_push_head(stack, p->left); + continue; + } + if (p->flags & EP_SHARD_COND) {/* HACK: SHARD_COND under TK_NOT is skipped */ + if (is_compare_op(p->op) && !(p->flags & EP_JOIN_LINK)) { + rc = partitions_filter_inequation_expr(partitions, p); + } else if (p->op == TK_BETWEEN) { + rc = partitions_filter_BETWEEN_expr(partitions, p); + } else if (p->op == TK_IN) { + rc = partitions_collect_IN_expr(partitions, p); + } + } + if (rc != PARSE_OK) { + break; + } + } + g_queue_free(stack); + return rc; +} + +static int flip_compare_op(int op) { /* flip horizontally */ + switch (op) { + case TK_LE:return TK_GE; + case TK_GE:return TK_LE; + case TK_LT:return TK_GT; + case TK_GT:return TK_LT; + default:return op; + } +} + +/** + * 1.mark conditions that contains sharding key + * 2.switch const field to right side ( 3 x>3) + * //3.check if we can handle the value of sharding key + * @param src the sharding table + * @param field the sharding column name + * @return number of occurance of sharding key + */ +static int optimize_sharding_condition(sql_expr_t *where, const sql_src_item_t *src, + const char *field) +{ + if (!where) { + return FALSE; + } + GQueue *stack = g_queue_new(); + g_queue_push_head(stack, where); + gboolean key_occur = 0; + + while (!g_queue_is_empty(stack)) {/* TODO: NOT op is not supported */ + sql_expr_t *p = g_queue_pop_head(stack); + if (is_logical_op(p->op)) { + if (p->right) + g_queue_push_head(stack, p->right); + if (p->left) + g_queue_push_head(stack, p->left); + continue; + } + if (is_compare_op(p->op) && !(p->flags & EP_JOIN_LINK)) { + /* the key might be on either side, unify as field = CONST */ + sql_expr_t *lhs = NULL, *rhs = NULL; + if (p->left->op == TK_ID || p->left->op == TK_DOT) { + lhs = p->left; + rhs = p->right; + } else if (p->right->op == TK_ID || p->right->op == TK_DOT) { + lhs = p->right; + rhs = p->left; + } else { + g_debug(G_STRLOC ":both sides aren't ID, neglect"); + continue; + } + if (expr_is_sharding_key(lhs, src, field)) { + if (p->left != lhs) { + p->left = lhs; + p->right = rhs; + p->op = flip_compare_op(p->op); + } + p->flags |= EP_SHARD_COND; + key_occur += 1; + } + } else if (p->op == TK_BETWEEN) { + if (expr_is_sharding_key(p->left, src, field)) { + p->flags |= EP_SHARD_COND; + key_occur += 1; + } + } else if (p->op == TK_IN) { + if (expr_is_sharding_key(p->left, src, field)) { + p->flags |= EP_SHARD_COND; + key_occur += 1; + } + } + } + g_queue_free(stack); + return key_occur; +} + +static void partitions_get_group_names(GPtrArray *partitions, GPtrArray *groups) +{ + int i = 0; + for (i = 0; i < partitions->len; ++i) { + sharding_partition_t *gp = g_ptr_array_index(partitions, i); + g_ptr_array_add(groups, gp->group_name); + } +} + +/** + * find out which 2 tables are connected by expression 'p', then + * 1. record it in linkage array + * 2. mark the expression 'p' as EP_JOIN_LINK; + */ +static void find_linkage(sql_expr_t *p, const GPtrArray *tables, + const GPtrArray *keys, gint8 *linkage) +{ + int i,j; + int N = tables->len; + for (i = 0; i < N; ++i) { + sql_src_item_t *t1 = g_ptr_array_index(tables, i); + const char *key1 = g_ptr_array_index(keys, i); + + for (j = 0; j < N; ++j) { + if (i == j) + continue; + sql_src_item_t *t2 = g_ptr_array_index(tables, j); + const char *key2 = g_ptr_array_index(keys, j); + if (expr_is_sharding_key(p->left, t1, key1) + && expr_is_sharding_key(p->right, t2, key2)) { + p->flags |= EP_JOIN_LINK; + /* only record linkage from lower index to higher index */ + i < j ? (linkage[i *N + j] = 1) : (linkage[j *N + i] = 1); + return; + } + } + } +} + +/** + * 1.check if all the tables belong to same VDB + * 2.check if there exists tableA.key = tableB.key, + * 3.and mark that eqution with EP_JOIN_LINK + */ +static gboolean join_on_sharding_key(char *default_db, GPtrArray *sharding_tables, + sql_expr_t *where) +{ + int first_vdb_id; + GPtrArray *sharding_keys = g_ptr_array_new(); /* array */ + int i; + for (i = 0; i < sharding_tables->len; ++i) { + sql_src_item_t *src = g_ptr_array_index(sharding_tables, i); + char *db = src->dbname ? src->dbname : default_db; + sharding_table_t *tinfo = shard_conf_get_info(db, src->table_name); + assert(tinfo); + if (i == 0) + first_vdb_id = tinfo->vdb_id; + if (tinfo->vdb_id != first_vdb_id) { + g_ptr_array_free(sharding_keys, TRUE); + return FALSE; + } + g_ptr_array_add(sharding_keys, tinfo->pkey->str); + } + + /* 2-d array of table linkage */ + int N = sharding_tables->len; + gint8 *linkage = g_new0(gint8, N *N); + + /* for each equation in WHERE clause */ + GQueue *stack = g_queue_new(); + if (where) { + g_queue_push_head(stack, where); + } + /* for each equation in ON clause */ + for (i = 0; i < sharding_tables->len; ++i) { + sql_src_item_t *src = g_ptr_array_index(sharding_tables, i); + if (src->on_clause) { + g_queue_push_head(stack, src->on_clause); + } + } + + while (!g_queue_is_empty(stack)) {/* TODO: NOT op is not supported */ + sql_expr_t *p = g_queue_pop_head(stack); + if (is_logical_op(p->op)) { + if (p->right) + g_queue_push_head(stack, p->right); + if (p->left) + g_queue_push_head(stack, p->left); + continue; + } + if (p->op == TK_EQ + && sql_expr_is_field_name(p->left) + && sql_expr_is_field_name(p->right)) { + find_linkage(p, sharding_tables, sharding_keys, linkage); + } + } + g_queue_free(stack); + + int num_linkage = 0; + for (i = 0; i < N *N; ++i) num_linkage += linkage[i]; + g_free(linkage); + g_ptr_array_free(sharding_keys, TRUE); + return num_linkage + 1 == sharding_tables->len; +} + +static void sql_select_get_single_tables(sql_select_t *select, + char* current_db, GList** single_tables /*out*/) +{ + char *db = current_db; + while (select) { + sql_src_list_t *sources = select->from_src; + int i = 0; + for (i = 0; sources && i < sources->len; ++i) { + sql_src_item_t *src = g_ptr_array_index(sources, i); + if (src->dbname) { + db = src->dbname; + } + if (src->table_name && shard_conf_is_single_table(db, src->table_name)) { + *single_tables = g_list_append(*single_tables, src); + } + } + select = select->prior; + } +} + +static int routing_select(sql_context_t *context, const sql_select_t *select, + char *default_db, guint32 fixture, query_stats_t *stats, + GPtrArray *groups /* out */) +{ + sql_src_list_t *sources = select->from_src; + if (!sources) { + shard_conf_get_fixed_group(groups, default_db, fixture); + stats->com_select_global += 1; + return USE_NON_SHARDING_TABLE; + } + char *db = default_db; + GPtrArray *sharding_tables = g_ptr_array_new(); + GList *single_tables = NULL; + int i; + for (i = 0; i < sources->len; ++i) { + sql_src_item_t *src = g_ptr_array_index(sources, i); + char *table = NULL; + if (src->select && sql_select_contains_sharding_table(src->select, &db, &table)) { + sharding_filter_sql(context); /* sharding table inside sub-query, should be filterd */ + if (context->rc == PARSE_NOT_SUPPORT) { + g_ptr_array_free(sharding_tables, TRUE); + return ERROR_UNPARSABLE; + } + shard_conf_get_table_groups(groups, db, table); + g_ptr_array_free(sharding_tables, TRUE); + return USE_ALL_SHARDINGS; + } + if (src->select) { /* subquery not contain sharding table, try to find single table */ + sql_select_get_single_tables(src->select, db, &single_tables); + } + db = src->dbname ? src->dbname : db; + if (src->table_name) { + if (shard_conf_is_shard_table(db, src->table_name)) { + g_ptr_array_add(sharding_tables, src); + } else if (shard_conf_is_single_table(db, src->table_name)) { + single_tables = g_list_append(single_tables, src); + } + } + } + + /* handle single table */ + if (single_tables) { + if (sharding_tables->len > 0) { + g_ptr_array_free(sharding_tables, TRUE); + g_list_free(single_tables); + sql_context_append_msg(context, "(cetus) JOIN single-table WITH sharding-table"); + return ERROR_UNPARSABLE; + } + + GList *l; + for (l = single_tables; l; l = l->next) { + sql_src_item_t *src = l->data; + db = src->dbname ? src->dbname : db; + shard_conf_get_single_table_distinct_group(groups, db, src->table_name); + } + if (groups->len > 1) { + g_ptr_array_free(sharding_tables, TRUE); + g_list_free(single_tables); + sql_context_append_msg(context, "(cetus)JOIN multiple single-tables not allowed"); + return ERROR_UNPARSABLE; + } else { + g_ptr_array_free(sharding_tables, TRUE); + g_list_free(single_tables); + return USE_NON_SHARDING_TABLE; + } + } + + if (sharding_tables->len == 0) { + shard_conf_get_fixed_group(groups, db, fixture); + g_ptr_array_free(sharding_tables, TRUE); + stats->com_select_global += 1; + return USE_NON_SHARDING_TABLE; + } + + if (sharding_tables->len >= 2) { + if (!join_on_sharding_key(default_db, sharding_tables, select->where_clause)) { + g_ptr_array_free(sharding_tables, TRUE); + sql_context_append_msg(context, + "(proxy)JOIN must inside VDB and have explicit join-on condition"); + return ERROR_UNPARSABLE; + } + } + + gboolean has_sharding_key = FALSE; + for (i = 0; i < sharding_tables->len; ++i) { + sql_src_item_t *shard_table = g_ptr_array_index(sharding_tables, i); + db = shard_table->dbname ? shard_table->dbname : db; + + sharding_table_t *shard_info = shard_conf_get_info(db, shard_table->table_name); + + /* join tables have same sharding key, we are graunteed tableA.key = tableB.key + so tableA.key = x also applies to tableB */ + if (optimize_sharding_condition(select->where_clause, + shard_table, shard_info->pkey->str)) { + has_sharding_key = TRUE; + } + } + + if (has_sharding_key) { + GPtrArray *partitions = g_ptr_array_new(); /* GPtrArray */ + for (i = 0; i < sharding_tables->len; ++i) { + sql_src_item_t *shard_table = g_ptr_array_index(sharding_tables, 0); + db = shard_table->dbname ? shard_table->dbname : db; + + shard_conf_table_partitions(partitions, db, shard_table->table_name); + int rc = partitions_filter_expr(partitions, select->where_clause); + if (rc == PARSE_ERROR) { + g_warning(G_STRLOC ":unrecognized key ranges"); + g_ptr_array_free(partitions, TRUE); + g_ptr_array_free(sharding_tables, TRUE); + sql_context_append_msg(context, "(proxy)sharding key parse error"); + return ERROR_UNPARSABLE; + } else if (rc == PARSE_UNRECOGNIZED) { + g_ptr_array_free(partitions, TRUE); + g_ptr_array_free(sharding_tables, TRUE); + shard_conf_get_table_groups(groups, db, shard_table->table_name); + stats->com_select_bad_key += 1; + return USE_ALL_SHARDINGS; + } + partitions_get_group_names(partitions, groups); + } + g_ptr_array_free(partitions, TRUE); + } + + if (groups->len > 0) { + g_ptr_array_free(sharding_tables, TRUE); + return USE_SHARDING; + } else { + /* has sharding table, but no sharding key + OR sharding key filter out all groups */ + for (i = 0; i < sharding_tables->len; ++i) { + sql_src_item_t *shard_table = g_ptr_array_index(sharding_tables, 0); + db = shard_table->dbname ? shard_table->dbname : db; + shard_conf_get_table_groups(groups, db, shard_table->table_name); + } + g_ptr_array_free(sharding_tables, TRUE); + return USE_ALL_SHARDINGS; + } +} + +static gboolean expr_same_with_sharding_cond(sql_expr_t *equation, sql_expr_t *where) +{ + GQueue *stack = g_queue_new(); + g_queue_push_head(stack, where); + gboolean is_same = FALSE; + /* for each condition in where clause */ + while (!g_queue_is_empty(stack)) { + sql_expr_t *p = g_queue_pop_head(stack); + if (p->op == TK_OR || p->op == TK_AND) { + if (p->right) + g_queue_push_head(stack, p->right); + if (p->left) + g_queue_push_head(stack, p->left); + continue; + } + /* HACK: SHARD_COND under TK_NOT is skipped */ + if ((p->flags & EP_SHARD_COND) && p->op == TK_EQ) { + if (sql_expr_equals(equation->left, p->left) + && sql_expr_equals(equation->right, p->right)) { + is_same = TRUE; + } + } + } + g_queue_free(stack); + return is_same; +} + +static int routing_update(sql_context_t *context, sql_update_t *update, + char *default_db, sharding_plan_t *plan, + GPtrArray *groups, guint32 fixture) +{ + char *db = default_db; + sql_src_list_t *tables = update->table; + sql_src_item_t *table = g_ptr_array_index(tables, 0); + if (table->dbname) { + db = table->dbname; + } + + if (!shard_conf_is_shard_table(db, table->table_name)) { + if (shard_conf_is_single_table(db, table->table_name)) { + plan->table_type = SINGLE_TABLE; + shard_conf_get_single_table_distinct_group(groups, db, table->table_name); + return USE_NON_SHARDING_TABLE; + } + + shard_conf_get_all_groups(groups, db); + plan->table_type = GLOBAL_TABLE; + if (groups->len > 1) { + return USE_DIS_TRAN; + } + return USE_NON_SHARDING_TABLE; + } + plan->table_type = SHARDED_TABLE; + sharding_table_t *shard_info = shard_conf_get_info(db, table->table_name); + int key_occur = optimize_sharding_condition(update->where_clause, + table, shard_info->pkey->str); + int i = 0; + /* update sharding key is not allowed */ + for (i = 0; update->set_list && i < update->set_list->len; ++i) { + sql_expr_t *equation = g_ptr_array_index(update->set_list, i); + if (!equation || !equation->left) { + sql_context_append_msg(context, "(proxy)syntax error"); + return ERROR_UNPARSABLE; + } + if (expr_is_sharding_key(equation->left, table, shard_info->pkey->str)) { + if (!(key_occur == 1 /* "update set k = 1 where k = 1" is legal */ + && expr_same_with_sharding_cond(equation, update->where_clause))) { + sql_context_append_msg(context, + "(proxy)update of sharding key is not allowed"); + return ERROR_UNPARSABLE; + } + } + } + + GPtrArray *partitions = g_ptr_array_new(); + shard_conf_table_partitions(partitions, db, table->table_name); + if (key_occur) { + int rc = partitions_filter_expr(partitions, update->where_clause); + if (rc == PARSE_ERROR || rc == PARSE_UNRECOGNIZED) { + g_warning(G_STRLOC ":unrecognized key ranges"); + g_ptr_array_free(partitions, TRUE); + sql_context_append_msg(context, "(proxy)sharding key parse error"); + return ERROR_UNPARSABLE; + } + } + partitions_get_group_names(partitions, groups); + g_ptr_array_free(partitions, TRUE); + + if (groups->len == 1) { + return USE_SHARDING; + } else if (groups->len == 0) { + shard_conf_get_table_groups(groups, db, table->table_name); + return USE_DIS_TRAN; + } else { + return USE_DIS_TRAN; + } +} + +static void group_insert_values(GHashTable *groups, sharding_partition_t *part, + sql_select_t *new_values) +{ + sql_select_t *values = g_hash_table_lookup(groups, part); + if (values) { + new_values->prior = values; + g_hash_table_insert(groups, part, new_values); + } else { + g_hash_table_insert(groups, part, new_values); + } +} + +static sql_select_t *merge_insert_values(GHashTable *groups, sql_select_t *residual) +{ + sql_select_t *merged_values = NULL; + GList *values_list = g_hash_table_get_values(groups); + if (residual) { + values_list = g_list_append(values_list, residual); + } + GList *l; + for (l = values_list; l != NULL; l = l->next) { + sql_select_t *val = l->data; + sql_select_t *tail = val; + while (tail->prior) { + tail = tail->prior; + } + if (merged_values) { + tail->prior = merged_values; + merged_values = val; + } else { + merged_values = val; + } + } + g_list_free(values_list); + return merged_values; +} + +static int insert_multi_value(sql_context_t *context, sql_insert_t *insert, + const char *db, const char *table, + sharding_table_t *shard_info, + int shard_key_index, sharding_plan_t *plan) +{ + int rc = 0; + + GPtrArray *partitions = g_ptr_array_new(); + shard_conf_table_partitions(partitions, db, table); + + GHashTable *value_groups = g_hash_table_new(g_direct_hash, g_direct_equal); + + sql_select_t *values = insert->sel_val; + insert->sel_val = NULL; /* take away from insert AST */ + + while (values) { + if (values->columns->len <= shard_key_index) { + g_warning("%s:col list values not match", G_STRLOC); + sql_context_append_msg(context, "(proxy)no sharding key"); + rc = ERROR_UNPARSABLE; + goto out; + } + condition_t cond = {TK_EQ, {0}}; + sql_expr_t *val = g_ptr_array_index(values->columns, shard_key_index); + int rc = expr_parse_sharding_value(val, shard_info->shard_key_type, &cond); + if (rc != PARSE_OK) { + sql_context_append_msg(context, "(proxy)sharding key parse error"); + rc = ERROR_UNPARSABLE; + goto out; + } + sharding_partition_t *part = partitions_get(partitions, cond); + if (!part) { + rc = ERROR_UNPARSABLE; + goto out; + } + sql_select_t *node = values; + values = values->prior; + node->prior = NULL; /* must be single values node */ + group_insert_values(value_groups, part, node); + } + GHashTableIter iter; + sharding_partition_t *part; + sql_select_t *values_list; + g_hash_table_iter_init(&iter, value_groups); + while (g_hash_table_iter_next(&iter, (void **)&part, (void **)&values_list)) { + GString *sql = g_string_new(NULL); + insert->sel_val = values_list; + sql_construct_insert(sql, insert); + sharding_plan_add_group_sql(plan, part->group_name, sql); + } + rc = plan->groups->len > 1 ? USE_DIS_TRAN : USE_NON_SHARDING_TABLE; + + out: + /* restore the INSERT-AST */ + insert->sel_val = merge_insert_values(value_groups, values); + + g_hash_table_destroy(value_groups); + g_ptr_array_free(partitions, TRUE); + return rc; +} + +static int routing_insert(sql_context_t *context, sql_insert_t *insert, + char *default_db, sharding_plan_t *plan, guint32 fixture) +{ + sql_src_list_t *src_list = insert->table; + assert(src_list && src_list->len > 0); + sql_src_item_t *src = g_ptr_array_index(src_list, 0); + char *db = src->dbname ? src->dbname : default_db; + char *table = src->table_name; + g_debug(G_STRLOC ":db:%s, table:%s", db, table); + + sharding_table_t *shard_info = shard_conf_get_info(db, table); + if (shard_info == NULL) { + GPtrArray *groups = g_ptr_array_new(); + if (shard_conf_is_single_table(db, table)) { + shard_conf_get_single_table_distinct_group(groups, db, table); + sharding_plan_add_groups(plan, groups); + plan->table_type = SINGLE_TABLE; + g_ptr_array_free(groups, TRUE); + return USE_NON_SHARDING_TABLE; + } + + shard_conf_get_all_groups(groups, db); + plan->table_type = GLOBAL_TABLE; + if (groups->len > 1) { + sharding_plan_add_groups(plan, groups); + g_ptr_array_free(groups, TRUE); + return USE_DIS_TRAN; + } + sharding_plan_add_groups(plan, groups); + g_ptr_array_free(groups, TRUE); + return USE_NON_SHARDING_TABLE; + } + plan->table_type = SHARDED_TABLE; + sql_id_list_t *cols = insert->columns; + if (cols == NULL) { + g_warning("%s:unsupported INSERT format", G_STRLOC); + sql_context_append_msg(context, + "(proxy)INSERT must use explicit column names"); + return ERROR_UNPARSABLE; + } + const char *shard_key = shard_info->pkey->str; + int shard_key_index = -1; + int i; + for (i = 0; i < cols->len; ++i) { + char *col = (char *)g_ptr_array_index(cols, i); + if (strcasecmp(col, shard_key) == 0) { + shard_key_index = i; + break; + } + } + if (shard_key_index == -1) { + g_warning(G_STRLOC ":cannot find sharding colomn %s", shard_key); + sql_context_append_msg(context, + "(proxy)INSERTion into sharding table must use sharding key"); + return ERROR_UNPARSABLE; + } + sql_select_t *sel_val = insert->sel_val; + if (!sel_val || !sel_val->columns) { + g_warning("%s:could not find insert values", G_STRLOC); + sql_context_append_msg(context, "(proxy)no VALUES"); + return ERROR_UNPARSABLE; + } + if (sel_val->flags & SF_MULTI_VALUE) { + return insert_multi_value(context, insert, db, table, + shard_info, shard_key_index, plan); + } + + /* SINGLE VALUE */ + sql_expr_list_t *values = sel_val->columns; + if (values->len <= shard_key_index) { + g_warning("%s:col list values not match", G_STRLOC); + sql_context_append_msg(context, "(proxy)no sharding key"); + return ERROR_UNPARSABLE; + } + + condition_t cond = {TK_EQ, {0}}; + sql_expr_t *val = g_ptr_array_index(values, shard_key_index); + int rc = expr_parse_sharding_value(val, shard_info->shard_key_type, &cond); + if (rc != PARSE_OK) { + sql_context_append_msg(context, "(proxy)sharding key parse error"); + return ERROR_UNPARSABLE; + } + + GPtrArray *partitions = g_ptr_array_new(); + shard_conf_table_partitions(partitions, db, table); + partitions_filter(partitions, cond); + + GPtrArray *groups = g_ptr_array_new(); + partitions_get_group_names(partitions, groups); + g_ptr_array_free(partitions, TRUE); + sharding_plan_add_groups(plan, groups); + g_ptr_array_free(groups, TRUE); + + if (plan->groups->len == 0) { + /* TODO: return code when pkey out of range; */ + return USE_NON_SHARDING_TABLE; + } else { + if (plan->groups->len != 1) {/* can't happen */ + return ERROR_UNPARSABLE; + } + return USE_SHARDING; + } +} + +static int routing_delete(sql_context_t *context, sql_delete_t *delete, + char *default_db, sharding_plan_t *plan, + GPtrArray *groups, guint32 fixture) +{ + if (!delete) { + g_warning(G_STRLOC ":delete ast error"); + sql_context_append_msg(context, "(proxy)no ast"); + return ERROR_UNPARSABLE; + } + char *db = default_db; + sql_src_list_t *from = delete->from_src; + if (from->len < 1) { + shard_conf_get_all_groups(groups, db); + if (groups->len > 1) { + return USE_DIS_TRAN; + } + return USE_NON_SHARDING_TABLE; + } + sql_src_item_t *table = g_ptr_array_index(from, 0); + if (table->dbname) + db = table->dbname; + + if (!shard_conf_is_shard_table(db, table->table_name)) { + if (shard_conf_is_single_table(db, table->table_name)) { + shard_conf_get_single_table_distinct_group(groups, db, table->table_name); + plan->table_type = SINGLE_TABLE; + return USE_NON_SHARDING_TABLE; + } + + shard_conf_get_all_groups(groups, db); + plan->table_type = GLOBAL_TABLE; + if (groups->len > 1) { + return USE_DIS_TRAN; + } + return USE_NON_SHARDING_TABLE; + } + plan->table_type = SHARDED_TABLE; + if (!delete->where_clause) { + shard_conf_get_table_groups(groups, db, table->table_name); + if (groups->len == 1) { + return USE_SHARDING; + } else if (groups->len > 1) { + return USE_DIS_TRAN; + } + } + + sharding_table_t *shard_info = shard_conf_get_info(db, table->table_name); + GPtrArray *partitions = g_ptr_array_new(); + shard_conf_table_partitions(partitions, db, table->table_name); + gboolean has_sharding_key = + optimize_sharding_condition(delete->where_clause, table, shard_info->pkey->str); + int rc = PARSE_OK; + if (has_sharding_key) { + rc = partitions_filter_expr(partitions, delete->where_clause); + } + + if (rc == PARSE_ERROR || rc == PARSE_UNRECOGNIZED) { + g_warning(G_STRLOC ":unrecognized key"); + g_ptr_array_free(partitions, TRUE); + sql_context_append_msg(context, "(proxy)sharding key parse error"); + return ERROR_UNPARSABLE; + } + partitions_get_group_names(partitions, groups); + g_ptr_array_free(partitions, TRUE); + + if (groups->len == 1) { + return USE_SHARDING; + } else if (groups->len == 0) { + shard_conf_get_table_groups(groups, db, table->table_name); + return USE_DIS_TRAN; + } else if (groups->len > 1) { + return USE_DIS_TRAN; + } + + g_warning(G_STRLOC ":error reach here"); + return ERROR_UNPARSABLE; +} + +int routing_by_property(sql_context_t *context, sql_property_t *property, + char *default_db, GPtrArray *groups/* out */) +{ + if (property->table) { + char *db = default_db; + char *table = property->table; + + /* check for dotted name "db.table" */ + char *p = strrchr(property->table, '.'); + if (p) { + *p = '\0'; + db = property->table; + table = p+1; + } + GPtrArray *partitions = g_ptr_array_new(); + shard_conf_table_partitions(partitions, db, table); + if (property->key) { + condition_t cond = {TK_EQ, {0}}; + sharding_table_t *info = shard_conf_get_info(db, table); + if (!info) { + g_ptr_array_free(partitions, TRUE); + sql_context_append_msg(context, "(cetus)no such table"); + return ERROR_UNPARSABLE; + } + int rc = string_to_sharding_value(property->key, info->shard_key_type, &cond); + if (rc != PARSE_OK) { + g_ptr_array_free(partitions, TRUE); + sql_context_append_msg(context, "(proxy)comment error: key"); + return ERROR_UNPARSABLE; + } + partitions_filter(partitions, cond); + } + partitions_get_group_names(partitions, groups); + g_ptr_array_free(partitions, TRUE); + enum sql_clause_flag_t f = context->rw_flag; + if ((f & CF_WRITE) && !(f & CF_DDL) && groups->len > 1) { + return USE_DIS_TRAN; + } + return USE_SHARDING; + + } else if (property->group) { + shard_conf_find_groups(groups, property->group, default_db); + if (groups->len > 0) { + enum sql_clause_flag_t f = context->rw_flag; + if ((f & CF_WRITE) && !(f & CF_DDL) && groups->len > 1) { + return USE_DIS_TRAN; + } + return USE_SHARDING; + } else { + char msg[256] = {0}; + snprintf(msg, 256, "no group: %s for db: %s", property->group, default_db); + g_warning("%s", msg); + sql_context_append_msg(context, msg); + return ERROR_UNPARSABLE; + } + } + sql_context_append_msg(context, "(proxy)comment error, unknown property"); + return ERROR_UNPARSABLE; +} + +int sharding_parse_groups(GString *default_db, sql_context_t *context, query_stats_t *stats, + guint32 fixture, sharding_plan_t *plan) +{ + GPtrArray *groups = g_ptr_array_new(); + if (context == NULL) { + g_warning("%s:sql is not parsed", G_STRLOC); + shard_conf_get_fixed_group(groups, default_db->str, fixture); + sharding_plan_add_groups(plan, groups); + g_ptr_array_free(groups, TRUE); + return USE_NON_SHARDING_TABLE; + } + + char *db = default_db->str; + g_debug(G_STRLOC ":default db:%s", db); + + if (sql_context_has_sharding_property(context)) { + int rc = routing_by_property(context, context->property, db, groups); + sharding_plan_add_groups(plan, groups); + g_ptr_array_free(groups, TRUE); + return rc; + } + + int rc = ERROR_UNPARSABLE; + switch (context->stmt_type) { + case STMT_SELECT: { + sql_select_t *select = context->sql_statement; + while (select) { + rc = routing_select(context, select, db, fixture, stats, groups); + if (rc < 0) { + break; + } + select = select->prior;/* select->prior UNION select */ + } + sharding_plan_add_groups(plan, groups); + g_ptr_array_free(groups, TRUE); + + if ((rc == USE_SHARDING || rc == USE_ALL_SHARDINGS) && plan->groups->len > 1) { + sharding_filter_sql(context); /* only filter queries with sharding table */ + if (context->rc == PARSE_NOT_SUPPORT) { + sharding_plan_clear_group(plan); + return ERROR_UNPARSABLE; + } + } + return rc; /* TODO: result of first select */ + } + case STMT_UPDATE: + rc = routing_update(context, context->sql_statement, + db, plan, groups, fixture); + sharding_plan_add_groups(plan, groups); + g_ptr_array_free(groups, TRUE); + return rc; + case STMT_INSERT: + rc = routing_insert(context, context->sql_statement, + db, plan, fixture); + g_ptr_array_free(groups, TRUE); + return rc; + case STMT_DELETE: + rc = routing_delete(context, context->sql_statement, + db, plan, groups, fixture); + sharding_plan_add_groups(plan, groups); + g_ptr_array_free(groups, TRUE); + return rc; + case STMT_SHOW_WARNINGS: + g_ptr_array_free(groups, TRUE); + return USE_PREVIOUS_WARNING_CONN; + case STMT_SHOW_COLUMNS: + case STMT_SHOW_CREATE: + case STMT_EXPLAIN_TABLE: {/* DESCRIBE tablename; */ + sql_src_list_t *tables = context->sql_statement; + sql_src_item_t *src_item = g_ptr_array_index(tables, 0); + g_assert(src_item); + if (src_item->dbname) + db = src_item->dbname; + if (shard_conf_is_shard_table(db, src_item->table_name)) { + shard_conf_get_any_group(groups, db, src_item->table_name); + sharding_plan_add_groups(plan, groups); + g_ptr_array_free(groups, TRUE); + return USE_ANY_SHARDINGS; + } + if (shard_conf_is_single_table(db, src_item->table_name)) { + shard_conf_get_single_table_distinct_group(groups, db, src_item->table_name); + sharding_plan_add_groups(plan, groups); + g_ptr_array_free(groups, TRUE); + return USE_NON_SHARDING_TABLE; + } + shard_conf_get_fixed_group(groups, db, fixture); + sharding_plan_add_groups(plan, groups); + g_ptr_array_free(groups, TRUE); + return USE_NON_SHARDING_TABLE; + } + case STMT_SET: + if (sql_context_is_autocommit_on(context)) { + g_ptr_array_free(groups, TRUE); + return USE_SAME; + } else if (sql_context_is_autocommit_off(context)) { + g_ptr_array_free(groups, TRUE); + return USE_NONE; + } else { + shard_conf_get_all_groups(groups, default_db->str); + sharding_plan_add_groups(plan, groups); + g_ptr_array_free(groups, TRUE); + return USE_ALL; + } + case STMT_START: + g_ptr_array_free(groups, TRUE); + return USE_NONE; + case STMT_COMMIT: + case STMT_ROLLBACK: + g_ptr_array_free(groups, TRUE); + return USE_PREVIOUS_TRAN_CONNS; + case STMT_COMMON_DDL: /* ddl without comments sent to all */ + case STMT_CALL: + shard_conf_get_all_groups(groups, default_db->str); + sharding_plan_add_groups(plan, groups); + g_ptr_array_free(groups, TRUE); + return USE_ALL; + case STMT_SHOW: + shard_conf_get_fixed_group(groups, default_db->str, fixture); + sharding_plan_add_groups(plan, groups); + g_ptr_array_free(groups, TRUE); + return USE_NON_SHARDING_TABLE; + default: + g_debug("unrecognized query, using default master db, sql:%s", plan->orig_sql->str); + shard_conf_get_fixed_group(groups, default_db->str, fixture); + sharding_plan_add_groups(plan, groups); + g_ptr_array_free(groups, TRUE); + return USE_NON_SHARDING_TABLE; + } +} + +/* is ORDERBY column a subset of SELECT column */ +static gboolean select_compare_orderby(sql_select_t *select) +{ + sql_expr_list_t *columns = select->columns; + sql_column_list_t *ord_cols = select->orderby_clause; + + if (ord_cols == NULL) { + sql_expr_t *mcol = g_ptr_array_index(columns, 0); + if (mcol->op == TK_STAR) { + return FALSE; /* reject: select DISTINCT *where .. */ + } else if (mcol->op == TK_DOT && mcol->right + && mcol->right->op == TK_STAR) { + return FALSE; /* reject: select DISTINCT t.* where .. */ + } + select->flags |= SF_REWRITE_ORDERBY; + return TRUE; /* accept: select DISTINCT a, b where .., rewrite later */ + } + + if (columns->len >= ord_cols->len) { + int i = 0; + for (i = 0; i < ord_cols->len; ++i) { + sql_column_t *ordcol = g_ptr_array_index(ord_cols, i); + sql_expr_t *ord = ordcol->expr; + if (sql_expr_is_id(ord, NULL)) { + sql_expr_t *match = sql_expr_list_find(columns, ord->token_text); + if (match) { + match->flags |= EP_ORDER_BY; + } else { + return FALSE; /* reject: select DISTINCT a,b,c ORDER BY x,y */ + } + } else if (sql_expr_is_dotted_name(ord, NULL, NULL)) { + sql_expr_t *match = sql_expr_list_find_fullname(columns, ord); + if (match) { + match->flags |= EP_ORDER_BY; + } else { + return FALSE; /* reject: select DISTINCT a,b,c ORDER BY x,y */ + } + } else { + g_warning(G_STRLOC ":unrecognized order by column"); + return FALSE; + } + } + if (columns->len != ord_cols->len) + select->flags |= SF_REWRITE_ORDERBY; + return TRUE; /* accept: 1) select DISTINCT a,b,c,d ORDER BY b,c,a. rewrite later */ + /* 2) select DISTINCT a,b,c,d ORDER BY a,c,b,d. no rewrite */ + } + return FALSE; /* reject: DISTINCT a ORDER BY a,b,c. might be syntax error */ +} + +static gboolean select_check_HAVING_column(sql_select_t *select) +{ + sql_expr_t *having = select->having_clause; + if (!(having && having->left)) {/* no HAVING is alright */ + return TRUE; + } + gboolean found = FALSE; /* found having cond in columns */ + int num_aggregate = 0; + const char *having_func = having->left->token_text; + sql_expr_list_t *columns = select->columns; + int i; + for (i = 0; columns && i < columns->len; ++i) { + sql_expr_t *col = g_ptr_array_index(columns, i); + if (col->op == TK_FUNCTION && sql_func_type(col->token_text) != FT_UNKNOWN) { + if (strcasecmp(having_func, col->token_text) == 0) { + found = TRUE; + } + if (col->alias && strcasecmp(having_func, col->alias)==0) { + found = TRUE; + } + ++num_aggregate; + } + } + if (num_aggregate > 1) /* if HAVING is used, only 1 aggregate can appear in column */ + return FALSE; + return found; +} + +static gboolean select_has_distincted_aggregate(sql_select_t *select, + int has_subquery, char **aggr_name) +{ + int i; + for (i = 0; select->columns && i < select->columns->len; ++i) { + sql_expr_t *expr = g_ptr_array_index(select->columns, i); + if (expr->op == TK_FUNCTION && expr->flags & EP_DISTINCT) { + if (strcasecmp(expr->token_text, "count") == 0 + ||strcasecmp(expr->token_text, "sum") == 0 + ||strcasecmp(expr->token_text, "avg") == 0) { + *aggr_name = expr->token_text; + return TRUE; + } + } + } + if (has_subquery && select->from_src) { + for (i = 0; i < select->from_src->len; ++i) { + sql_src_item_t *src = g_ptr_array_index(select->from_src, i); + if (src->select) + return select_has_distincted_aggregate(src->select, 0, aggr_name); + } + } + return FALSE; +} + +static gboolean select_has_AVG(sql_select_t *select) +{ + int i; + for (i = 0; select->columns && i < select->columns->len; ++i) { + sql_expr_t *expr = g_ptr_array_index(select->columns, i); + if (expr->op == TK_FUNCTION && expr->flags & EP_AGGREGATE) { + if (strcasecmp(expr->token_text, "avg") == 0) + return TRUE; + } + } + return FALSE; +} + +/* group by & order by have only 1 column, and they are same */ +static gboolean select_groupby_orderby_have_same_column(sql_select_t *select) +{ + g_assert(select->groupby_clause && select->orderby_clause); + sql_expr_list_t *grp = select->groupby_clause; + sql_column_list_t *ord = select->orderby_clause; + if (grp->len > 1 || ord->len > 1) { /* only support one col */ + return FALSE; + } + sql_expr_t *grp_expr = g_ptr_array_index(grp, 0); + sql_column_t *ord_col = g_ptr_array_index(ord, 0); + return g_strcmp0(ord_col->expr->token_text, grp_expr->token_text) == 0; +} + +void sharding_filter_sql(sql_context_t *context)/* TODO:should be in sql-operations.c */ +{ + if (context->stmt_type == STMT_SELECT) { + sql_select_t *select = context->sql_statement; + if (select->flags & SF_DISTINCT) { + gboolean same = select_compare_orderby(select); + if (!same) { + sql_context_set_error(context, PARSE_NOT_SUPPORT, + "(proxy)ORDER BY columns must be a subset of DISTINCT columns"); + return; + } + } + + /* grauntee HAVING condition show up in column */ + if (select->having_clause) { + if (!is_compare_op(select->having_clause->op)) { + sql_context_set_error(context, PARSE_NOT_SUPPORT, + "(cetus) Only support simple HAVING condition"); + return; + } + if (!select_check_HAVING_column(select)) { + sql_context_set_error(context, PARSE_NOT_SUPPORT, + "(cetus) HAVING condition must show up in column, " + "and only 1 aggregate can appear in column when HAVING is used"); + return; + } + } + + if (select->groupby_clause) { + sql_expr_list_t *groupby = select->groupby_clause; + int i = 0; + for (i = 0; i < groupby->len; ++i) { + sql_expr_t *col = g_ptr_array_index(groupby, i); + if (col->op == TK_CASE) { + sql_context_set_error(context, PARSE_NOT_SUPPORT, + "(proxy) group by CASE-WHEN not supported"); + return; + } + } + } + if (select_has_AVG(select)) { + sql_context_set_error(context, PARSE_NOT_SUPPORT, + "(cetus)this AVG would be routed to multiple shards, not allowed"); + return; + } + if (select->groupby_clause && select->orderby_clause && + !select_groupby_orderby_have_same_column(select)) + { + sql_context_set_error(context, PARSE_NOT_SUPPORT, + "(cetus) can't ORDER BY and GROUP BY different columns on sharded sql"); + return; + } + /* reject SELECT COUNT(DISTINCT) / SUM(DISTINCT) / AVG(DISTINCT) */ + if (context->clause_flags & CF_DISTINCT_AGGR) { + char *aggr_name = NULL; + int subquery = context->clause_flags & CF_SUBQUERY; + if (select_has_distincted_aggregate(select, subquery, &aggr_name)) { + char msg[100]; + snprintf(msg, 100, "(proxy) %s(DISTINCT ...) not supported", aggr_name); + sql_context_set_error(context, PARSE_NOT_SUPPORT, msg); + return; + } + } + } +} diff --git a/plugins/shard/sharding-parser.h b/plugins/shard/sharding-parser.h new file mode 100644 index 0000000..9b6a3c3 --- /dev/null +++ b/plugins/shard/sharding-parser.h @@ -0,0 +1,28 @@ +#ifndef __SHARDING_PARSER_H__ +#define __SHARDING_PARSER_H__ + +#include "network-mysqld.h" +#include "sharding-query-plan.h" +#include "sql-context.h" + +#define USE_NON_SHARDING_TABLE 0 /* default */ +#define USE_SHARDING 1 /* default */ +#define USE_DIS_TRAN 2 +#define USE_ANY_SHARDINGS 3 /* default */ +#define USE_ALL_SHARDINGS 4 /* default */ +#define USE_ALL 5 /* default */ +#define USE_SAME 6 +#define USE_PREVIOUS_WARNING_CONN 7 +#define USE_NONE 8 +#define USE_PREVIOUS_TRAN_CONNS 9 +#define ERROR_UNPARSABLE -1 + + +NETWORK_API int sharding_parse_groups(GString *, sql_context_t *, query_stats_t *, + unsigned int, sharding_plan_t *); + +NETWORK_API GString *sharding_modify_sql(sql_context_t *, having_condition_t *); + +NETWORK_API void sharding_filter_sql(sql_context_t *); + +#endif //__SHARDING_PARSER_H__ diff --git a/scripts/cetus-binwrapper.in b/scripts/cetus-binwrapper.in new file mode 100644 index 0000000..b862fd1 --- /dev/null +++ b/scripts/cetus-binwrapper.in @@ -0,0 +1,25 @@ +#!/bin/sh +# +# wrapper script for self-contained packaging +# +# we assume that this script is located in +# +# {basedir}/bin/cetus +# +# and wants to call +# +# {basedir}/libexec/cetus +# +# You can inject a wrapper like +# +# $ MYSQL_PROXY_WRAPPER=valgrind ./bin/cetus +# + +scriptdir=`dirname $0` +scriptname=`basename $0` +scriptdir=`(cd "$scriptdir/"; pwd)` +scriptdir=`dirname "$scriptdir"` + +@DYNLIB_PATH_VAR@="$@DYNLIB_PATH_VAR@:$scriptdir/lib/" +export @DYNLIB_PATH_VAR@ +exec $MYSQL_PROXY_WRAPPER "$scriptdir/libexec/$scriptname" "$@" diff --git a/scripts/proxy-client b/scripts/proxy-client new file mode 100755 index 0000000..c01cf0d --- /dev/null +++ b/scripts/proxy-client @@ -0,0 +1,949 @@ +#!/usr/bin/env python +# coding: utf-8 + +### +# A script console for cetus. +# For changing proxy config on remote server. +### + + +import os +import sys +import re +import traceback +import MySQLdb +import json + +### +# DB settings +# +# Please EDIT this before running +### +HOST = '10.238.7.6' +PORT = 3306 +USER = 'root' +PASSWD = '123456' +DB = 'proxy_management' + + +conn = None +PROXY_ID = None +BACKENDS_NDX_MAP = {} + +# a pattern for check IP format +IP_RE_PATTERN = ('^(?:[1-9]?\d|1\d{2}|2(?:[0-4]\d|5[0-5]))' + '(?:\.(?:[1-9]?\d|1\d{2}|2(?:[0-4]\d|5[0-5]))){3}$') + + +# Print msg to console with color +def print_console(msg): + print('{0}{1}{2}'.format('\033[1;32m', msg, '\033[0m')) + + +def strwide(msg): + return '\n{0}\n'.format(msg) + + +def print_console_wide(msg): + print_console(strwide(msg)) + + +# The decoration for reconnecting db +def reconnect(f): + def deco(*args, **kwargs): + global conn + if not conn: + conn = connect_db(HOST, PORT, USER, PASSWD, DB) + return f(*args, **kwargs) + return deco + + +# Connect to database +def connect_db(host, port, user, passwd, db): + return MySQLdb.Connection( + host=host, + port=port, + user=user, + passwd=passwd, + db=db, + charset='utf8') + + +# Select data +@reconnect +def query_data(sql): + global conn + cur = conn.cursor() + cur.execute(sql) + data = cur.fetchall() + cur.close() + return data + + +@reconnect +def query(sql): + global conn + cur = conn.cursor() + ret = cur.execute(sql) + conn.commit() + cur.close() + return ret + + +@reconnect +def querymany(sql, data): + global conn + cur = conn.cursor() + ret = cur.executemany(sql, data) + conn.commit() + cur.close() + return ret + + +@reconnect +def insertone(sql): + global conn + cur = conn.cursor() + ret = 0 + try: + cur.execute(sql) + ret = cur.lastrowid + except MySQLdb.Error: + pass + finally: + conn.commit() + cur.close() + return ret + + +def gen_backend_ndx(): + global BACKENDS_NDX_MAP + BACKENDS_NDX_MAP = {} + + +# Process user's command +def run_cmd(cmd): + global PROXY_ID + global BACKENDS_NDX_MAP + + ### + # Show simple settings for proxy to start + ### + if cmd == 'show': + cmap = {} + data = query_data('select proxy_id,s_key,s_value from settings') + max_key = 0 + for line in data: + id, k, v = line + if not id in cmap: + cmap[id] = {} + if not k in cmap[id]: + cmap[id][k] = [] + cmap[id][k].append(v) + if len(k) > max_key: + max_key = len(k) + msg_list = [''] + for ndx in cmap: + msg_list.append('-'*10) + msg_list.append('PROXY_ID: {0}'.format(ndx)) + msg_list.append('') + for k in sorted(cmap[ndx].keys()): + i = 0 + for v in cmap[ndx][k]: + if i is 0: + msg_list.append('{0}{1}'.format(k.ljust(max_key+3), v)) + else: + msg_list.append('{0}{1}'.format(''.ljust(max_key+3), v)) + i += 1 + msg_list.append('-'*10) + msg_list.append('') + msg = '\n'.join(msg_list) + + ### + # Show backends info details + ### + elif cmd == 'show backends': + data = query_data( + 'select a.id, a.proxy_id, a.s_key, a.s_value, b.stat from ' + 'settings as a, backends as b where a.s_key in ' + '("proxy-backend-addresses", "proxy-read-only-backend-addresses")' + ' and a.id=b.settings_id') + cmap = {} + BACKENDS_NDX_MAP = {} + ndx_map = {} + for line in data: + id, pxy_id, k, v, st = line + if not pxy_id in BACKENDS_NDX_MAP: + BACKENDS_NDX_MAP[pxy_id] = {} + ndx_map[pxy_id] = 1 + BACKENDS_NDX_MAP[pxy_id][str(ndx_map[pxy_id])] = id + if not pxy_id in cmap: + cmap[pxy_id] = [] + bk_addr_list = re.split('[@:]', v) + addr = bk_addr_list[0] + port = bk_addr_list[1] + if k == 'proxy-backend-addresses': + tp = 'rw' + else: + tp = 'ro' + cmap[pxy_id].append({'ndx': ndx_map[pxy_id], 'addr':addr, 'port':port, 'type':tp, 'stat':st}) + ndx_map[pxy_id] += 1 + msg_list = [''] + for pxy_id in cmap: + msg_list.append('-'*10) + msg_list.append('PROXY_ID: {0}'.format(pxy_id)) + msg_list.append('') + for backend in cmap[pxy_id]: + msg_list.append('{0}{1}'.format('NDX'.ljust(8), backend['ndx'])) + msg_list.append('{0}{1}'.format('ADDR'.ljust(8), backend['addr'])) + msg_list.append('{0}{1}'.format('PORT'.ljust(8), backend['port'])) + msg_list.append('{0}{1}'.format('TYPE'.ljust(8), backend['type'])) + msg_list.append('{0}{1}'.format('STAT'.ljust(8), backend['stat'])) + msg_list.append('') + msg_list[-1] = '-'*10 + msg_list.append('') + msg = '\n'.join(msg_list) + + ### + # Show sharding info details + ### + elif cmd == 'show sharding': + data = query_data('select proxy_id, shard_db, shard_table, pkey, shard_key_type, ' + 'method, partitions, logic_shard_num from sharding ') + shard_map = {} + for line in data: + proxy_id = line[0] + if not proxy_id in shard_map: + shard_map[proxy_id] = [] + shard_map[proxy_id].append({ + 'shard_db': line[1], + 'shard_table': line[2], + 'pkey': line[3], + 'shard_key_type': line[4], + 'method': line[5], + 'partitions': line[6], + 'logic_shard_num': line[7] + }) + msg_list = [''] + for pid in shard_map: + msg_list.append('PROXY_ID: {0}'.format(pid)) + msg_list.append('='*20) + for shard in shard_map[pid]: + if shard['shard_db'] == '_default_group': + msg_list.append('{0}{1}'.format('DEFAULT_GROUP'.ljust(18), shard['partitions'])) + elif shard['shard_db'] == '_all_groups': + msg_list.append('{0}{1}'.format('ALL_GROUPS'.ljust(18), shard['partitions'])) + else: + msg_list.append('{0}{1}'.format('DB'.ljust(18), shard['shard_db'])) + msg_list.append('{0}{1}'.format('TABLE'.ljust(18), shard['shard_table'])) + msg_list.append('{0}{1}'.format('KEY'.ljust(18), shard['pkey'])) + msg_list.append('{0}{1}'.format('KEYTYPE'.ljust(18), shard['shard_key_type'])) + msg_list.append('{0}{1}'.format('METHOD'.ljust(18), shard['method'])) + msg_list.append('{0}{1}'.format('PARTITIONS'.ljust(18), shard['partitions'])) + msg_list.append('{0}{1}'.format('SHARDNUM'.ljust(18), shard['logic_shard_num'])) + msg_list.append('-'*10) + msg_list.append('') + msg = '\n'.join(msg_list) + + ### + # set global proxy_id, can ignore proxy_id in where clause below. + ### + elif cmd.startswith('use '): + try: + PROXY_ID = cmd.split(' ', 1)[1] + msg = strwide('Proxy changed.') + except IndexError: + msg = strwide('Unknown Proxy.') + + ### + # Add or update settings + ### + elif cmd.startswith('add ') or cmd.startswith('update '): + syntax_error = 0 + msg = '' + ### + # RE Pattern for add/update + # match `update (cap1) set (cap2) where (cap3)` + # Consider a JSON string may contains any character + ### + up_pat = ('^\s*(add|update)\s+([a-zA-Z_]+)\s+' + 'set\s+(\S+\s*=\s*(?:(\'|").*?(? 65535 or int(bk_port) < 0: + return strwide('PORT format error.\nShould be integer in 0~65535') + bk_type = set_map.get('type', 'ro') + if not bk_type in ['ro', 'rw']: + return strwide("TYPE format error.\nShould be 'rw' or 'ro'.") + bk_stat = set_map.get('stat', 'unknown') + if not bk_stat in ['unknown', 'up', 'down', 'delete', 'maintaining']: + return strwide("STAT format error.\n" + "Should be 'unknown', 'up', 'down', 'delete' or 'maintaining'") + bk_shard = set_map.get('shard_group', None) + exist = query_data( + ('select a.s_key, a.s_value, b.stat from settings as a, backends as b' + ' where s_key in ("proxy-backend-addresses", "proxy-read-only-backend-addresses")' + ' and a.id=b.settings_id and proxy_id="{0}"').format(pxy)) + bk_full_addr = '{0}:{1}'.format(bk_addr, bk_port) + if bk_shard: + bk_full = '{0}@{1}'.format(bk_full_addr, bk_shard) + else: + bk_full = bk_addr_list + for bk in exist: + if bk[1] == bk_full_addr or bk[1].startswith('{0}@'.format(bk_full_addr)): + return strwide('backend "{0}:{1}" exists'.format(bk_addr, bk_port)) + if bk_type == 'rw': + bk_key = 'proxy-backend-addresses' + else: + bk_key = 'proxy-read-only-backend-addresses' + last_id = insertone( + ('insert into settings set proxy_id="{0}", ' + 's_key="{1}", s_value="{2}"').format(pxy, bk_key, bk_full)) + if last_id: + r += 1 + if insertone('insert into backends set settings_id={0}, stat="{1}"'.format( + last_id, bk_stat)): + r += 1 + else: + query('delete from settings where id={0}'.format(last_id)) + print_console('Insert failed. Dirty data exists in table backends.') + elif c1 == 'sharding': + shard_keys = set_map.keys() + for k in ['default_group', 'all_groups']: + if k in shard_keys: + shard_keys.remove(k) + real_k = '_' + k + v = set_map[k] + exist = query_data( + ('select id from sharding where proxy_id="{0}" and ' + 'shard_db="{1}"').format(pxy, real_k)) + try: + json.loads(v) + except ValueError: + print_console('"{0}" value should be in JSON format.'.format(real_k)) + continue + if exist: + print_console('"{0}" exists, skip.'.format(real_k)) + continue + in_sql = ('insert into sharding set proxy_id="{0}",' + 'shard_db="{1}", shard_table="{1}", partitions="{2}"' + ).format(pxy, real_k, v.replace('"', '\\"')) + r += query(in_sql) + # Check if all params is set. + if shard_keys: + for kk in ['db', 'table', 'key', 'keytype', 'method', + 'partitions', 'shardnum']: + if not kk in set_map: + return strwide('Lack parameter "{0}".'.format(kk)) + if not set_map['keytype'] in ['int', 'str', 'datetime', 'date']: + return strwide('"keytype" should be [int|str|date|datetime].') + if not set_map['method'] in ['range', 'hash']: + return strwide('"method" should be hash or range.') + try: + json.loads(set_map['partitions']) + except ValueError: + return strwide('"partitions" should be in JSON format.') + set_map['partitions'] = set_map['partitions'].replace('"', '\\"') + r += query( + ('insert into sharding set proxy_id="{0}", shard_db="{1[db]}",' + 'shard_table="{1[table]}", pkey="{1[key]}", shard_key_type=' + '"{1[keytype]}", method="{1[method]}", partitions="{1[partitions]}",' + 'logic_shard_num="{1[shardnum]}"').format(pxy, set_map)) + else: + syntax_error = 1 + print_console_wide('{0} rows added.'.format(r)) + elif c0 == 'update': + r = 0 + # Set normal start config + if c1 == 'settings': + for k, v in set_map.items(): + exist = query_data(('select id from settings where proxy_id="{0}"' + ' and s_key="{1}"').format(pxy, k)) + if exist and exist[0]: + r += query('update settings set s_value="{0}" where id={1}'\ + .format(v, exist[0][0])) + else: + print_console('"{0}" does not exist, use `add settings` instead.') + print_console_wide('{0} rows changed.'.format(r)) + # Set backends config + elif c1 == 'backend': + # `show backends` should be run before update + # so that backend ndx will be create + if not BACKENDS_NDX_MAP: + msg = strwide('Please type `show backends` to confirm backend index.') + elif not 'ndx' in cond_map: + msg = strwide("'ndx' should be specified.") + else: + try: + backend = query_data(( + 'select a.s_key, a.s_value, b.stat from settings as a,' + 'backends as b where a.id={0} and a.id=b.settings_id' + ).format(BACKENDS_NDX_MAP[pxy][cond_map['ndx']])) + except IndexError: + return strwide('PROXY_ID or NDX error.') + if backend and backend[0]: + bk_spl = re.split('[@:]', backend[0][1]) + bk_addr = bk_spl[0] + bk_port = bk_spl[1] + if len(bk_spl) is 3: + bk_shard = bk_spl[2] + else: + bk_shard = None + if backend[0][0] == 'proxy-backend-addresses': + bk_type = 'rw' + else: + bk_type = 'ro' + bk_stat = backend[0][2] + else: + return strwide('Backend info error.\n' + 'Retry `show backends` and update again.') + for k, v in set_map.items(): + if k == 'addr': + if not re.match(IP_RE_PATTERN, v): + return strwide('IP format error.') + bk_addr = v + elif k == 'port': + try: + new_port = int(v) + if new_port > 65535: + raise ValueError + except ValueError: + return strwide('PORT format error.\nShould be integer in 0~65535') + bk_port = v + elif k == 'type': + if v != 'rw' and v != 'ro': + print v + return strwide("TYPE format error.\nShould be 'rw' or 'ro'.") + bk_type = v + elif k == 'stat': + if not v in ['unknown', 'up', 'down', 'delete', 'maintaining']: + return strwide("STAT format error.\n" + "Should be 'unknown', 'up', 'down', 'delete' or 'maintaining'") + bk_stat = v + elif k == 'shard_group': + bk_shard = v + else: + return strwide("Backends setting key should be either 'addr', 'port', 'type' or 'stat'.") + if bk_shard: + new_bk = '{0}:{1}@{2}'.format(bk_addr, bk_port, bk_shard) + else: + new_bk = ':'.join([bk_addr, bk_port]) + if bk_type == 'rw': + bk_key = 'proxy-backend-addresses' + else: + bk_key = 'proxy-read-only-backend-addresses' + r += query('update settings set s_key="{0}", s_value="{1}" where id={2}'.format( + bk_key, new_bk, BACKENDS_NDX_MAP[pxy][cond_map['ndx']])) + r += query('update backends set stat="{0}" where settings_id={1}'.format( + bk_stat, BACKENDS_NDX_MAP[pxy][cond_map['ndx']])) + print_console_wide('{0} rows changed.'.format(r)) + msg_list = [''] + msg_list.append('PROXY_ID: {0}'.format(pxy)) + msg_list.append('') + msg_list.append('{0}{1}'.format('NDX'.ljust(8), cond_map['ndx'])) + msg_list.append('{0}{1}'.format('ADDR'.ljust(8), bk_addr)) + msg_list.append('{0}{1}'.format('PORT'.ljust(8), bk_port)) + msg_list.append('{0}{1}'.format('TYPE'.ljust(8), bk_type)) + msg_list.append('{0}{1}'.format('STAT'.ljust(8), bk_stat)) + msg_list.append('') + msg = '\n'.join(msg_list) + elif c1 == 'sharding': + # shard settings for table need 'db' and 'table' + shard_keys = set_map.keys() + try: + shard_keys.remove('default_group') + shard_keys.remove('all_groups') + except ValueError: + pass + if shard_keys and (not 'db' in cond_map or not 'table' in cond_map): + return strwide('"db" and "table" are needed in where clause') + # set 'default_group' and 'all_groups' + if 'default_group' in set_map: + r += query(('update sharding set partitions="{0}" where proxy_id="{1}"' + ' and shard_db="_default_group"').format( + set_map['default_group'].replace('"', '\\"'), pxy)) + if 'all_groups' in set_map: + r += query(('update sharding set partitions="{0}" where proxy_id="{1}"' + ' and shard_db="_all_groups"').format( + set_map['all_groups'].replace('"', '\\"'), pxy)) + # Normal shard settings + for k in shard_keys: + exist = query_data(('select id from sharding where proxy_id={0}' + ' and shard_db="{1}" and shard_table="{2}"').format( + pxy, cond_map['db'], cond_map['table'])) + if not exist or not exist[0]: + return strwide('shard rule does not exists, use "add sharding" instead') + rule_id = exist[0][0] + shard_set_list = [] + for k, v in set_map.items(): + if k == 'db': + shard_set_list.append('shard_db="{0}"'.format(v)) + elif k == 'table': + shard_set_list.append('shard_table="{0}"'.format(v)) + elif k == 'key': + shard_set_list.append('pkey="{0}"'.format(v)) + elif k == 'keytype': + if not v in ['int', 'str', 'datetime', 'date']: + return strwide('"keytype" should be [int|str|date|datetime]') + shard_set_list.append('shard_key_type="{0}"'.format(v)) + elif k == 'method': + if not v in ['range', 'hash']: + return strwide('"method" should be hash or range') + shard_set_list.append('method="{0}"'.format(v)) + elif k == 'partitions': + try: + json.loads(v) + except ValueError: + return strwide('"partitions" should be a json string') + shard_set_list.append('partitions="{0}"'.format(v.replace('"', '\\"'))) + r += query('update sharding set {0} where id={id}'.format( + ','.join(shard_set_list), id=rule_id)) + print_console_wide('{0} rows changed.'.format(r)) + else: + syntax_error = 1 + if syntax_error is 1: + msg = strwide('Syntax error with command `{0}`').format(cmd) + + ### + # delete settings, include sharding and backend + ### + elif cmd.startswith('remove '): + syntax_error = 0 + msg = '' + r = 0 + rm_pat = ('^\s*remove\s+([a-zA-Z_]+)\s+' + 'where\s+(\S+\s*=\s*(?:(\'|").*?(?\033[0m ') + cmd = cmd.strip().lower().rstrip(';') + if not cmd: + continue + print_console(run_cmd(cmd)) + except EOFError: + print_console_wide('### Good Bye! ###') + except: + traceback.print_exc() + print_console_wide('Error detected! Exit.') + finally: + if conn: + conn.close() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000..c36c432 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,188 @@ +# $%BEGINLICENSE%$ +# Copyright (c) 2007, 2012, Oracle and/or its affiliates. All rights reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2 of the +# License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA +# 02110-1301 USA +# +# $%ENDLICENSE%$ + +INCLUDE_DIRECTORIES(${PROJECT_BINARY_DIR}) # for config.h + +INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}) +include_directories(${PROJECT_SOURCE_DIR}/lib) +include_directories(${PROJECT_BINARY_DIR}/lib) + +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) + +INCLUDE_DIRECTORIES(${GLIB_INCLUDE_DIRS}) +LINK_DIRECTORIES(${GLIB_LIBRARY_DIRS}) + +INCLUDE_DIRECTORIES(${MYSQL_INCLUDE_DIRS}) +LINK_DIRECTORIES(${MYSQL_LIBRARY_DIRS}) + +INCLUDE_DIRECTORIES(${EVENT_INCLUDE_DIRS}) +LINK_DIRECTORIES(${EVENT_LIBRARY_DIRS}) + +LINK_DIRECTORIES(${LIBINTL_LIBRARY_DIRS}) + +STRING(REPLACE "." "" SHARED_LIBRARY_SUFFIX ${CMAKE_SHARED_LIBRARY_SUFFIX}) +ADD_DEFINITIONS(-DSHARED_LIBRARY_SUFFIX="${SHARED_LIBRARY_SUFFIX}") + +SET(chassis_sources + chassis-plugin.c + chassis-event.c + chassis-log.c + chassis-mainloop.c + chassis-shutdown-hooks.c + chassis-keyfile.c + chassis-path.c + chassis-filemode.c + chassis-limits.c + chassis-frontend.c + chassis-options.c + chassis-unix-daemon.c + chassis-config.c + cJSON.c +) + +SET(timing_sources + chassis-timings.c +) + +SET(glibext_sources + glib-ext.c +) + +SET(proxy_sources + network-mysqld.c + network-mysqld-proto.c + network-mysqld-packet.c + network-conn-pool.c + network-conn-pool-wrap.c + network-queue.c + network-socket.c + network-address.c + network-injection.c + resultset_merge.c + cetus-log.c + plugin-common.c + network-backend.c + sharding-config.c + sharding-query-plan.c + shard-plugin-con.c + character-set.c + server-session.c + network-compress.c + cetus-users.c + cetus-util.c + cetus-variable.c + cetus-monitor.c +) + +if(NETWORK_DEBUG_TRACE_STATE_CHANGES) + list(APPEND proxy_sources cetus-query-queue.c) +endif(NETWORK_DEBUG_TRACE_STATE_CHANGES) + +ADD_LIBRARY(mysql-chassis SHARED ${chassis_sources}) +ADD_LIBRARY(mysql-chassis-proxy SHARED ${proxy_sources}) +ADD_LIBRARY(mysql-chassis-glibext SHARED ${glibext_sources}) +ADD_LIBRARY(mysql-chassis-timing SHARED ${timing_sources}) +ADD_EXECUTABLE(cetus mysql-proxy-cli.c) + +if(SIMPLE_PARSER) + target_compile_definitions(cetus PRIVATE DEFAULT_PLUGIN="proxy") +else(SIMPLE_PARSER) + target_compile_definitions(cetus PRIVATE DEFAULT_PLUGIN="shard") +endif(SIMPLE_PARSER) + +TARGET_LINK_LIBRARIES(mysql-chassis-glibext + ${GLIB_LIBRARIES} + ${GMODULE_LIBRARIES} + ${GTHREAD_LIBRARIES} +) + +TARGET_LINK_LIBRARIES(mysql-chassis-timing + ${GLIB_LIBRARIES} + ${GMODULE_LIBRARIES} + ${GTHREAD_LIBRARIES} + mysql-chassis-glibext +) + +TARGET_LINK_LIBRARIES(mysql-chassis + ${GLIB_LIBRARIES} + ${GMODULE_LIBRARIES} + ${GTHREAD_LIBRARIES} + ${EVENT_LIBRARIES} + ${MYSQL_LIBRARIES} + mysql-chassis-timing + mysql-chassis-glibext +) + +TARGET_LINK_LIBRARIES(mysql-chassis-proxy + mysql-chassis + sqlparser + mysql-chassis-glibext + mysql-chassis-timing + ${OPENSSL_LIBRARIES} + ) + +TARGET_LINK_LIBRARIES(cetus + mysql-chassis + mysql-chassis-proxy + mysql-chassis-glibext + mysql-chassis-timing + ${GLIB_LIBRARIES} + ${GMODULE_LIBRARIES} + ${GTHREAD_LIBRARIES} + ${EVENT_LIBRARIES} + ${MYSQL_LIBRARIES} +) + +# Unix platforms provide a wrapper script to avoid relinking at install time +# figure out the correct name of the shared linker lookup path for this system, default to LD_LIBRARY_PATH +SET(DYNLIB_PATH_VAR "LD_LIBRARY_PATH") + +IF("AIX" STREQUAL ${CMAKE_SYSTEM_NAME}) + SET(DYNLIB_PATH_VAR "LIBPATH") +ENDIF("AIX" STREQUAL ${CMAKE_SYSTEM_NAME}) + +IF("HP-UX" STREQUAL ${CMAKE_SYSTEM_NAME}) + SET(DYNLIB_PATH_VAR "SHLIB_PATH") +ENDIF("HP-UX" STREQUAL ${CMAKE_SYSTEM_NAME}) + +IF(APPLE) + SET(DYNLIB_PATH_VAR "DYLD_LIBRARY_PATH") +ENDIF(APPLE) + +# write the wrapper script, which uses DYNLIB_PATH_VAR +CONFIGURE_FILE( + ${CMAKE_SOURCE_DIR}/scripts/cetus-binwrapper.in + ${PROJECT_BINARY_DIR}/cetus.sh + @ONLY + ) +INSTALL(FILES ${PROJECT_BINARY_DIR}/cetus.sh + DESTINATION bin/ + PERMISSIONS OWNER_EXECUTE OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ + RENAME cetus + ) + +INSTALL(TARGETS cetus + RUNTIME DESTINATION libexec + ) + +CHASSIS_INSTALL_TARGET(mysql-chassis) +CHASSIS_INSTALL_TARGET(mysql-chassis-proxy) +CHASSIS_INSTALL_TARGET(mysql-chassis-glibext) +CHASSIS_INSTALL_TARGET(mysql-chassis-timing) diff --git a/src/cJSON.c b/src/cJSON.c new file mode 100644 index 0000000..0026d5b --- /dev/null +++ b/src/cJSON.c @@ -0,0 +1,724 @@ +#include +#include +#include +#include +#include +#include +#include +#include "cJSON.h" + +static const char *ep; + +const char *cJSON_GetErrorPtr(void) {return ep;} + +static int cJSON_strcasecmp(const char *s1,const char *s2) +{ + if (!s1) return (s1==s2)?0:1;if (!s2) return 1; + for(; tolower(*s1) == tolower(*s2); ++s1, ++s2) if (*s1 == 0) return 0; + return tolower(*(const unsigned char *)s1) - tolower(*(const unsigned char *)s2); +} + +static void *(*cJSON_malloc)(size_t sz) = malloc; +static void (*cJSON_free)(void *ptr) = free; + +static char *cJSON_strdup(const char *str) +{ + size_t len; + char *copy; + + len = strlen(str) + 1; + if (!(copy = (char *)cJSON_malloc(len))) return 0; + memcpy(copy,str,len); + return copy; +} + +void cJSON_InitHooks(cJSON_Hooks *hooks) +{ + if (!hooks) { /* Reset hooks */ + cJSON_malloc = malloc; + cJSON_free = free; + return; + } + + cJSON_malloc = (hooks->malloc_fn)?hooks->malloc_fn:malloc; + cJSON_free = (hooks->free_fn)?hooks->free_fn:free; +} + +/* Internal constructor. */ +static cJSON *cJSON_New_Item(void) +{ + cJSON *node = (cJSON *)cJSON_malloc(sizeof(cJSON)); + if (node) memset(node,0,sizeof(cJSON)); + return node; +} + +/* Delete a cJSON structure. */ +void cJSON_Delete(cJSON *c) +{ + cJSON *next; + while (c) + { + next=c->next; + if (!(c->type&cJSON_IsReference) && c->child) cJSON_Delete(c->child); + if (!(c->type&cJSON_IsReference) && c->valuestring) cJSON_free(c->valuestring); + if (!(c->type&cJSON_StringIsConst) && c->string) cJSON_free(c->string); + cJSON_free(c); + c=next; + } +} + +/* Parse the input text to generate a number, and populate the result into item. */ +static const char *parse_number(cJSON *item,const char *num) +{ + double n=0,sign=1,scale=0;int subscale=0,signsubscale=1; + + if (*num == '-') sign=-1,num++; /* Has sign? */ + if (*num == '0') num++; /* is zero */ + if (*num>='1' && *num<='9') do n=(n *10.0)+(*num++ -'0'); while (*num>='0' && *num<='9'); /* Number? */ + if (*num == '.' && num[1]>='0' && num[1]<='9') {num++; do n=(n *10.0)+(*num++ -'0'),scale--; while (*num>='0' && *num<='9');} /* Fractional part? */ + if (*num == 'e' || *num == 'E') /* Exponent? */ + { num++;if (*num == '+') num++; else if (*num == '-') signsubscale=-1,num++; /* With sign? */ + while (*num>='0' && *num<='9') subscale=(subscale *10)+(*num++ - '0'); /* Number? */ + } + + n=sign *n *pow(10.0,(scale+subscale *signsubscale)); /* number = +/- number.fraction *10^+/- exponent */ + + item->valuedouble=n; + item->valueint=(int)n; + item->type=cJSON_Number; + return num; +} + +static int pow2gt (int x) { --x; x|=x>>1; x|=x>>2; x|=x>>4; x|=x>>8; x|=x>>16; return x+1; } + +typedef struct {char *buffer; int length; int offset; } printbuffer; + +static char *ensure(printbuffer *p,int needed) +{ + char *newbuffer;int newsize; + if (!p || !p->buffer) return NULL; + needed += p->offset; + if (needed <= p->length) return p->buffer+p->offset; + + newsize = pow2gt(needed); + newbuffer = (char *) cJSON_malloc(newsize); + if (!newbuffer) {cJSON_free(p->buffer); p->length=0, p->buffer=0; return NULL;} + if (newbuffer) memcpy(newbuffer, p->buffer, p->length); + cJSON_free(p->buffer); + p->length = newsize; + p->buffer = newbuffer; + + return (char *) (newbuffer + p->offset); +} + +static int update(printbuffer *p) +{ + char *str; + if (!p || !p->buffer) return 0; + str=p->buffer+p->offset; + return p->offset+strlen(str); +} + +/* Render the number nicely from the given item into a string. */ +static char *print_number(cJSON *item,printbuffer *p) +{ + char *str=0; + double d=item->valuedouble; + if (d == 0) + { + if (p) str=ensure(p,2); + else str=(char *)cJSON_malloc(2); /* special case for 0. */ + if (str) strcpy(str,"0"); + } + else if (fabs(((double)item->valueint)-d)<=DBL_EPSILON && d<=INT_MAX && d>=INT_MIN) + { + if (p) str=ensure(p,21); + else str=(char *)cJSON_malloc(21); /* 2^64+1 can be represented in 21 chars. */ + if (str) sprintf(str,"%d",item->valueint); + } + else + { + if (p) str=ensure(p,64); + else str=(char *)cJSON_malloc(64); /* This is a nice tradeoff. */ + if (str) + { + if (fabs(floor(d)-d)<=DBL_EPSILON && fabs(d)<1.0e60)sprintf(str,"%.0f",d); + else if (fabs(d)<1.0e-6 || fabs(d)>1.0e9) sprintf(str,"%e",d); + else sprintf(str,"%f",d); + } + } + return str; +} + +static unsigned parse_hex4(const char *str) +{ + unsigned h=0; + if (*str>='0' && *str<='9') h+=(*str)-'0'; else if (*str>='A' && *str<='F') h+=10+(*str)-'A'; else if (*str>='a' && *str<='f') h+=10+(*str)-'a'; else return 0; + h=h<<4;str++; + if (*str>='0' && *str<='9') h+=(*str)-'0'; else if (*str>='A' && *str<='F') h+=10+(*str)-'A'; else if (*str>='a' && *str<='f') h+=10+(*str)-'a'; else return 0; + h=h<<4;str++; + if (*str>='0' && *str<='9') h+=(*str)-'0'; else if (*str>='A' && *str<='F') h+=10+(*str)-'A'; else if (*str>='a' && *str<='f') h+=10+(*str)-'a'; else return 0; + h=h<<4;str++; + if (*str>='0' && *str<='9') h+=(*str)-'0'; else if (*str>='A' && *str<='F') h+=10+(*str)-'A'; else if (*str>='a' && *str<='f') h+=10+(*str)-'a'; else return 0; + return h; +} + +/* Parse the input text into an unescaped cstring, and populate item. */ +static const unsigned char firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; +static const char *parse_string(cJSON *item,const char *str) +{ + const char *ptr=str+1;char *ptr2;char *out;int len=0;unsigned uc,uc2; + if (*str!='\"') {ep=str;return 0;} /* not a string! */ + + while (*ptr!='\"' && *ptr && ++len) if (*ptr++ == '\\') ptr++; /* Skip escaped quotes. */ + + out=(char *)cJSON_malloc(len+1); /* This is how long we need for the string, roughly. */ + if (!out) return 0; + + ptr=str+1;ptr2=out; + while (*ptr!='\"' && *ptr) + { + if (*ptr!='\\') *ptr2++=*ptr++; + else + { + ptr++; + switch (*ptr) + { + case 'b': *ptr2++='\b'; break; + case 'f': *ptr2++='\f'; break; + case 'n': *ptr2++='\n'; break; + case 'r': *ptr2++='\r'; break; + case 't': *ptr2++='\t'; break; + case 'u': /* transcode utf16 to utf8. */ + uc=parse_hex4(ptr+1);ptr+=4; /* get the unicode char. */ + + if ((uc>=0xDC00 && uc<=0xDFFF) || uc == 0) break; /* check for invalid. */ + + if (uc>=0xD800 && uc<=0xDBFF) /* UTF16 surrogate pairs. */ + { + if (ptr[1]!='\\' || ptr[2]!='u') break; /* missing second-half of surrogate. */ + uc2=parse_hex4(ptr+3);ptr+=6; + if (uc2<0xDC00 || uc2>0xDFFF) break; /* invalid second-half of surrogate. */ + uc=0x10000 + (((uc&0x3FF)<<10) | (uc2&0x3FF)); + } + + len=4;if (uc<0x80) len=1;else if (uc<0x800) len=2;else if (uc<0x10000) len=3; ptr2+=len; + + switch (len) { + case 4: *--ptr2 =((uc | 0x80) & 0xBF); uc >>= 6; + case 3: *--ptr2 =((uc | 0x80) & 0xBF); uc >>= 6; + case 2: *--ptr2 =((uc | 0x80) & 0xBF); uc >>= 6; + case 1: *--ptr2 =(uc | firstByteMark[len]); + } + ptr2+=len; + break; + default: *ptr2++=*ptr; break; + } + ptr++; + } + } + *ptr2=0; + if (*ptr == '\"') ptr++; + item->valuestring=out; + item->type=cJSON_String; + return ptr; +} + +/* Render the cstring provided to an escaped version that can be printed. */ +static char *print_string_ptr(const char *str,printbuffer *p) +{ + const char *ptr;char *ptr2,*out;int len=0,flag=0;unsigned char token; + + if (!str) + { + if (p) out=ensure(p,3); + else out=(char *)cJSON_malloc(3); + if (!out) return 0; + strcpy(out,"\"\""); + return out; + } + + for (ptr=str;*ptr;ptr++) flag|=((*ptr>0 && *ptr<32)||(*ptr == '\"')||(*ptr == '\\'))?1:0; + if (!flag) + { + len=ptr-str; + if (p) out=ensure(p,len+3); + else out=(char *)cJSON_malloc(len+3); + if (!out) return 0; + ptr2=out;*ptr2++='\"'; + strcpy(ptr2,str); + ptr2[len]='\"'; + ptr2[len+1]=0; + return out; + } + + ptr=str;while ((token=*ptr) && ++len) {if (strchr("\"\\\b\f\n\r\t",token)) len++; else if (token<32) len+=5;ptr++;} + + if (p) out=ensure(p,len+3); + else out=(char *)cJSON_malloc(len+3); + if (!out) return 0; + + ptr2=out;ptr=str; + *ptr2++='\"'; + while (*ptr) + { + if ((unsigned char)*ptr>31 && *ptr!='\"' && *ptr!='\\') *ptr2++=*ptr++; + else + { + *ptr2++='\\'; + switch (token=*ptr++) + { + case '\\': *ptr2++='\\'; break; + case '\"': *ptr2++='\"'; break; + case '\b': *ptr2++='b'; break; + case '\f': *ptr2++='f'; break; + case '\n': *ptr2++='n'; break; + case '\r': *ptr2++='r'; break; + case '\t': *ptr2++='t'; break; + default: sprintf(ptr2,"u%04x",token);ptr2+=5; break; /* escape and print */ + } + } + } + *ptr2++='\"';*ptr2++=0; + return out; +} +/* Invote print_string_ptr (which is useful) on an item. */ +static char *print_string(cJSON *item,printbuffer *p) {return print_string_ptr(item->valuestring,p);} + +/* Predeclare these prototypes. */ +static const char *parse_value(cJSON *item,const char *value); +static char *print_value(cJSON *item,int depth,int fmt,printbuffer *p); +static const char *parse_array(cJSON *item,const char *value); +static char *print_array(cJSON *item,int depth,int fmt,printbuffer *p); +static const char *parse_object(cJSON *item,const char *value); +static char *print_object(cJSON *item,int depth,int fmt,printbuffer *p); + +/* Utility to jump whitespace and cr/lf */ +static const char *skip(const char *in) {while (in && *in && (unsigned char)*in<=32) in++; return in;} + +/* Parse an object - create a new root, and populate. */ +cJSON *cJSON_ParseWithOpts(const char *value,const char **return_parse_end,int require_null_terminated) +{ + const char *end=0; + cJSON *c=cJSON_New_Item(); + ep=0; + if (!c) return 0; /* memory fail */ + + end=parse_value(c,skip(value)); + if (!end) {cJSON_Delete(c);return 0;} /* parse failure. ep is set. */ + + /* if we require null-terminated JSON without appended garbage, skip and then check for a null terminator */ + if (require_null_terminated) {end=skip(end);if (*end) {cJSON_Delete(c);ep=end;return 0;}} + if (return_parse_end) *return_parse_end=end; + return c; +} +/* Default options for cJSON_Parse */ +cJSON *cJSON_Parse(const char *value) {return cJSON_ParseWithOpts(value,0,0);} + +/* Render a cJSON item/entity/structure to text. */ +char *cJSON_Print(cJSON *item) {return print_value(item,0,1,0);} +char *cJSON_PrintUnformatted(cJSON *item) {return print_value(item,0,0,0);} + +char *cJSON_PrintBuffered(cJSON *item,int prebuffer,int fmt) +{ + printbuffer p; + p.buffer=(char *)cJSON_malloc(prebuffer); + p.length=prebuffer; + p.offset=0; + return print_value(item,0,fmt,&p); +} + + +/* Parser core - when encountering text, process appropriately. */ +static const char *parse_value(cJSON *item,const char *value) +{ + if (!value) return 0; /* Fail on null. */ + if (!strncmp(value,"null",4)) { item->type=cJSON_NULL; return value+4; } + if (!strncmp(value,"false",5)) { item->type=cJSON_False; return value+5; } + if (!strncmp(value,"true",4)) { item->type=cJSON_True; item->valueint=1; return value+4; } + if (*value == '\"') { return parse_string(item,value); } + if (*value == '-' || (*value>='0' && *value<='9')) { return parse_number(item,value); } + if (*value == '[') { return parse_array(item,value); } + if (*value == '{') { return parse_object(item,value); } + + ep=value;return 0; /* failure. */ +} + +/* Render a value to text. */ +static char *print_value(cJSON *item,int depth,int fmt,printbuffer *p) +{ + char *out=0; + if (!item) return 0; + if (p) + { + switch ((item->type)&255) + { + case cJSON_NULL: {out=ensure(p,5); if (out) strcpy(out,"null"); break;} + case cJSON_False: {out=ensure(p,6); if (out) strcpy(out,"false"); break;} + case cJSON_True: {out=ensure(p,5); if (out) strcpy(out,"true"); break;} + case cJSON_Number: out=print_number(item,p);break; + case cJSON_String: out=print_string(item,p);break; + case cJSON_Array: out=print_array(item,depth,fmt,p);break; + case cJSON_Object: out=print_object(item,depth,fmt,p);break; + } + } + else + { + switch ((item->type)&255) + { + case cJSON_NULL: out=cJSON_strdup("null"); break; + case cJSON_False: out=cJSON_strdup("false");break; + case cJSON_True: out=cJSON_strdup("true"); break; + case cJSON_Number: out=print_number(item,0);break; + case cJSON_String: out=print_string(item,0);break; + case cJSON_Array: out=print_array(item,depth,fmt,0);break; + case cJSON_Object: out=print_object(item,depth,fmt,0);break; + } + } + return out; +} + +/* Build an array from input text. */ +static const char *parse_array(cJSON *item,const char *value) +{ + cJSON *child; + if (*value!='[') {ep=value;return 0;} /* not an array! */ + + item->type=cJSON_Array; + value=skip(value+1); + if (*value == ']') return value+1; /* empty array. */ + + item->child=child=cJSON_New_Item(); + if (!item->child) return 0; /* memory fail */ + value=skip(parse_value(child,skip(value))); /* skip any spacing, get the value. */ + if (!value) return 0; + + while (*value == ',') + { + cJSON *new_item; + if (!(new_item=cJSON_New_Item())) return 0; /* memory fail */ + child->next=new_item;new_item->prev=child;child=new_item; + value=skip(parse_value(child,skip(value+1))); + if (!value) return 0; /* memory fail */ + } + + if (*value == ']') return value+1; /* end of array */ + ep=value;return 0; /* malformed. */ +} + +/* Render an array to text */ +static char *print_array(cJSON *item,int depth,int fmt,printbuffer *p) +{ + char *out=0,*ptr,*ret;int len=5; + cJSON *child=item->child; + int numentries=0,i = 0 ,fail=0; + + /* How many entries in the array? */ + while (child) numentries++,child=child->next; + /* Explicitly handle numentries == 0 */ + if (!numentries) + { + if (p) out=ensure(p,3); + else out=(char *)cJSON_malloc(3); + if (out) strcpy(out,"[]"); + return out; + } + + if (p) + { + /* Compose the output array. */ + i=p->offset; + ptr=ensure(p,1);if (!ptr) return 0; *ptr='['; p->offset++; + child=item->child; + while (child && !fail) + { + print_value(child,depth+1,fmt,p); + p->offset=update(p); + if (child->next) {len=fmt?2:1;ptr=ensure(p,len+1);if (!ptr) return 0;*ptr++=',';if (fmt)*ptr++=' ';*ptr=0;p->offset+=len;} + child=child->next; + } + ptr=ensure(p,2);if (!ptr) return 0; *ptr++=']';*ptr=0; + out=(p->buffer)+i; + } + else + { + /* Allocate an array to hold the values for each */ + char **entries=(char **)cJSON_malloc(numentries *sizeof(char *)); + if (!entries) return 0; + memset(entries,0,numentries *sizeof(char *)); + /* Retrieve all the results: */ + child=item->child; + while (child && !fail) + { + ret=print_value(child,depth+1,fmt,0); + entries[i++]=ret; + if (ret) len+=strlen(ret)+2+(fmt?1:0); else fail=1; + child=child->next; + } + + /* If we didn't fail, try to malloc the output string */ + if (!fail) out=(char *)cJSON_malloc(len); + /* If that fails, we fail. */ + if (!out) fail=1; + + /* Handle failure. */ + if (fail) + { + for (i = 0 ;i < numentries;i++) if (entries[i]) cJSON_free(entries[i]); + cJSON_free(entries); + return 0; + } + + /* Compose the output array. */ + *out='['; + ptr=out+1;*ptr=0; + for (i = 0 ;i < numentries;i++) + { + size_t tmplen=strlen(entries[i]);memcpy(ptr,entries[i],tmplen);ptr+=tmplen; + if (i!=numentries-1) {*ptr++=',';if (fmt)*ptr++=' ';*ptr=0;} + cJSON_free(entries[i]); + } + cJSON_free(entries); + *ptr++=']';*ptr++=0; + } + return out; +} + +/* Build an object from the text. */ +static const char *parse_object(cJSON *item,const char *value) +{ + cJSON *child; + if (*value!='{') {ep=value;return 0;} /* not an object! */ + + item->type=cJSON_Object; + value=skip(value+1); + if (*value == '}') return value+1; /* empty array. */ + + item->child=child=cJSON_New_Item(); + if (!item->child) return 0; + value=skip(parse_string(child,skip(value))); + if (!value) return 0; + child->string=child->valuestring;child->valuestring=0; + if (*value!=':') {ep=value;return 0;} /* fail! */ + value=skip(parse_value(child,skip(value+1))); /* skip any spacing, get the value. */ + if (!value) return 0; + + while (*value == ',') + { + cJSON *new_item; + if (!(new_item=cJSON_New_Item())) return 0; /* memory fail */ + child->next=new_item;new_item->prev=child;child=new_item; + value=skip(parse_string(child,skip(value+1))); + if (!value) return 0; + child->string=child->valuestring;child->valuestring=0; + if (*value!=':') {ep=value;return 0;} /* fail! */ + value=skip(parse_value(child,skip(value+1))); /* skip any spacing, get the value. */ + if (!value) return 0; + } + + if (*value == '}') return value+1; /* end of array */ + ep=value;return 0; /* malformed. */ +} + +/* Render an object to text. */ +static char *print_object(cJSON *item,int depth,int fmt,printbuffer *p) +{ + char *out=0,*ptr,*ret,*str;int len=7,i = 0 ,j; + cJSON *child=item->child; + int numentries=0; + /* Count the number of entries. */ + while (child) numentries++,child=child->next; + /* Explicitly handle empty object case */ + if (!numentries) + { + if (p) out=ensure(p,fmt?depth+4:3); + else out=(char *)cJSON_malloc(fmt?depth+4:3); + if (!out) return 0; + ptr=out;*ptr++='{'; + if (fmt) {*ptr++='\n';for (i = 0 ;i < depth-1;i++) *ptr++='\t';} + *ptr++='}';*ptr++=0; + return out; + } + if (p) + { + /* Compose the output: */ + i=p->offset; + len=fmt?2:1; ptr=ensure(p,len+1); if (!ptr) return 0; + *ptr++='{'; if (fmt) *ptr++='\n'; *ptr=0; p->offset+=len; + child=item->child;depth++; + while (child) + { + if (fmt) + { + ptr=ensure(p,depth); if (!ptr) return 0; + for (j=0;joffset+=depth; + } + print_string_ptr(child->string,p); + p->offset=update(p); + + len=fmt?2:1; + ptr=ensure(p,len); if (!ptr) return 0; + *ptr++=':';if (fmt) *ptr++='\t'; + p->offset+=len; + + print_value(child,depth,fmt,p); + p->offset=update(p); + + len=(fmt?1:0)+(child->next?1:0); + ptr=ensure(p,len+1); if (!ptr) return 0; + if (child->next) *ptr++=','; + if (fmt) *ptr++='\n';*ptr=0; + p->offset+=len; + child=child->next; + } + ptr=ensure(p,fmt?(depth+1):2); if (!ptr) return 0; + if (fmt) for (i = 0 ;i < depth-1;i++) *ptr++='\t'; + *ptr++='}';*ptr=0; + out=(p->buffer)+i; + } + else + { + /* Allocate space for the names and the objects */ + char **entries=(char **)cJSON_malloc(numentries *sizeof(char *)); + if (!entries) return 0; + char **names=(char **)cJSON_malloc(numentries *sizeof(char *)); + if (!names) {cJSON_free(entries);return 0;} + memset(entries,0,sizeof(char *)*numentries); + memset(names,0,sizeof(char *)*numentries); + + /* Collect all the results into our arrays: */ + child=item->child;depth++;if (fmt) len+=depth; + + int fail = 0; + while (child) + { + names[i]=str=print_string_ptr(child->string,0); + entries[i++]=ret=print_value(child,depth,fmt,0); + if (str && ret) len+=strlen(ret)+strlen(str)+2+(fmt?2+depth:0); else fail=1; + child=child->next; + } + + /* Try to allocate the output string */ + if (!fail) out=(char *)cJSON_malloc(len); + if (!out) fail=1; + + /* Handle failure */ + if (fail) + { + for (i = 0 ;i < numentries;i++) {if (names[i]) cJSON_free(names[i]);if (entries[i]) cJSON_free(entries[i]);} + cJSON_free(names);cJSON_free(entries); + return 0; + } + + /* Compose the output: */ + *out='{';ptr=out+1;if (fmt)*ptr++='\n';*ptr=0; + for (i = 0 ;i < numentries;i++) + { + if (fmt) for (j=0;jchild;int i = 0 ;while(c)i++,c=c->next;return i;} +cJSON *cJSON_GetArrayItem(cJSON *array,int item) {cJSON *c=array->child; while (c && item>0) item--,c=c->next; return c;} +cJSON *cJSON_GetObjectItem(cJSON *object,const char *string) {cJSON *c=object->child; while (c && cJSON_strcasecmp(c->string,string)) c=c->next; return c;} + +/* Utility for array list handling. */ +static void suffix_object(cJSON *prev,cJSON *item) {prev->next=item;item->prev=prev;} +/* Utility for handling references. */ +static cJSON *create_reference(cJSON *item) {cJSON *ref=cJSON_New_Item();if (!ref) return 0;memcpy(ref,item,sizeof(cJSON));ref->string=0;ref->type|=cJSON_IsReference;ref->next=ref->prev=0;return ref;} + +/* Add item to array/object. */ +void cJSON_AddItemToArray(cJSON *array, cJSON *item) {cJSON *c=array->child;if (!item) return; if (!c) {array->child=item;} else {while (c && c->next) c=c->next; suffix_object(c,item);}} +void cJSON_AddItemToObject(cJSON *object,const char *string,cJSON *item) {if (!item) return; if (item->string) cJSON_free(item->string);item->string=cJSON_strdup(string);cJSON_AddItemToArray(object,item);} +void cJSON_AddItemToObjectCS(cJSON *object,const char *string,cJSON *item) {if (!item) return; if (!(item->type&cJSON_StringIsConst) && item->string) cJSON_free(item->string);item->string=(char *)string;item->type|=cJSON_StringIsConst;cJSON_AddItemToArray(object,item);} +void cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item) {cJSON_AddItemToArray(array,create_reference(item));} +void cJSON_AddItemReferenceToObject(cJSON *object,const char *string,cJSON *item) {cJSON_AddItemToObject(object,string,create_reference(item));} + +cJSON *cJSON_DetachItemFromArray(cJSON *array,int which) {cJSON *c=array->child;while (c && which>0) c=c->next,which--;if (!c) return 0; + if (c->prev) c->prev->next=c->next;if (c->next) c->next->prev=c->prev;if (c == array->child) array->child=c->next;c->prev=c->next=0;return c;} +void cJSON_DeleteItemFromArray(cJSON *array,int which) {cJSON_Delete(cJSON_DetachItemFromArray(array,which));} +cJSON *cJSON_DetachItemFromObject(cJSON *object,const char *string) {int i = 0 ;cJSON *c=object->child;while (c && cJSON_strcasecmp(c->string,string)) i++,c=c->next;if (c) return cJSON_DetachItemFromArray(object,i);return 0;} +void cJSON_DeleteItemFromObject(cJSON *object,const char *string) {cJSON_Delete(cJSON_DetachItemFromObject(object,string));} + +/* Replace array/object items with new ones. */ +void cJSON_InsertItemInArray(cJSON *array,int which,cJSON *newitem) {cJSON *c=array->child;while (c && which>0) c=c->next,which--;if (!c) {cJSON_AddItemToArray(array,newitem);return;} + newitem->next=c;newitem->prev=c->prev;c->prev=newitem;if (c == array->child) array->child=newitem; else newitem->prev->next=newitem;} +void cJSON_ReplaceItemInArray(cJSON *array,int which,cJSON *newitem) {cJSON *c=array->child;while (c && which>0) c=c->next,which--;if (!c) return; + newitem->next=c->next;newitem->prev=c->prev;if (newitem->next) newitem->next->prev=newitem; + if (c == array->child) array->child=newitem; else newitem->prev->next=newitem;c->next=c->prev=0;cJSON_Delete(c);} +void cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem) {int i = 0 ;cJSON *c=object->child;while(c && cJSON_strcasecmp(c->string,string))i++,c=c->next;if (c) {newitem->string=cJSON_strdup(string);cJSON_ReplaceItemInArray(object,i,newitem);}} + +/* Create basic types: */ +cJSON *cJSON_CreateNull(void) {cJSON *item=cJSON_New_Item();if (item)item->type=cJSON_NULL;return item;} +cJSON *cJSON_CreateTrue(void) {cJSON *item=cJSON_New_Item();if (item)item->type=cJSON_True;return item;} +cJSON *cJSON_CreateFalse(void) {cJSON *item=cJSON_New_Item();if (item)item->type=cJSON_False;return item;} +cJSON *cJSON_CreateBool(int b) {cJSON *item=cJSON_New_Item();if (item)item->type=b?cJSON_True:cJSON_False;return item;} +cJSON *cJSON_CreateNumber(double num) {cJSON *item=cJSON_New_Item();if (item) {item->type=cJSON_Number;item->valuedouble=num;item->valueint=(int)num;}return item;} +cJSON *cJSON_CreateString(const char *string) {cJSON *item=cJSON_New_Item();if (item) {item->type=cJSON_String;item->valuestring=cJSON_strdup(string);}return item;} +cJSON *cJSON_CreateArray(void) {cJSON *item=cJSON_New_Item();if (item)item->type=cJSON_Array;return item;} +cJSON *cJSON_CreateObject(void) {cJSON *item=cJSON_New_Item();if (item)item->type=cJSON_Object;return item;} + +/* Create Arrays: */ +cJSON *cJSON_CreateIntArray(const int *numbers,int count) {int i;cJSON *n=0,*p=0,*a=cJSON_CreateArray();for(i = 0 ;a && i < count;i++) {n=cJSON_CreateNumber(numbers[i]);if (!i)a->child=n;else suffix_object(p,n);p=n;}return a;} +cJSON *cJSON_CreateFloatArray(const float *numbers,int count) {int i;cJSON *n=0,*p=0,*a=cJSON_CreateArray();for(i = 0 ;a && i < count;i++) {n=cJSON_CreateNumber(numbers[i]);if (!i)a->child=n;else suffix_object(p,n);p=n;}return a;} +cJSON *cJSON_CreateDoubleArray(const double *numbers,int count) {int i;cJSON *n=0,*p=0,*a=cJSON_CreateArray();for(i = 0 ;a && i < count;i++) {n=cJSON_CreateNumber(numbers[i]);if (!i)a->child=n;else suffix_object(p,n);p=n;}return a;} +cJSON *cJSON_CreateStringArray(const char **strings,int count) {int i;cJSON *n=0,*p=0,*a=cJSON_CreateArray();for(i = 0 ;a && i < count;i++) {n=cJSON_CreateString(strings[i]);if (!i)a->child=n;else suffix_object(p,n);p=n;}return a;} + +/* Duplication */ +cJSON *cJSON_Duplicate(cJSON *item,int recurse) +{ + cJSON *newitem,*cptr,*nptr=0,*newchild; + /* Bail on bad ptr */ + if (!item) return 0; + /* Create new item */ + newitem=cJSON_New_Item(); + if (!newitem) return 0; + /* Copy over all vars */ + newitem->type=item->type&(~cJSON_IsReference),newitem->valueint=item->valueint,newitem->valuedouble=item->valuedouble; + if (item->valuestring) {newitem->valuestring=cJSON_strdup(item->valuestring); if (!newitem->valuestring) {cJSON_Delete(newitem);return 0;}} + if (item->string) {newitem->string=cJSON_strdup(item->string); if (!newitem->string) {cJSON_Delete(newitem);return 0;}} + /* If non-recursive, then we're done! */ + if (!recurse) return newitem; + /* Walk the ->next chain for the child. */ + cptr=item->child; + while (cptr) + { + newchild=cJSON_Duplicate(cptr,1); /* Duplicate (with recurse) each item in the ->next chain */ + if (!newchild) {cJSON_Delete(newitem);return 0;} + if (nptr) {nptr->next=newchild,newchild->prev=nptr;nptr=newchild;} /* If newitem->child already set, then crosswire ->prev and ->next and move on */ + else {newitem->child=newchild;nptr=newchild;} /* Set newitem->child and move to it */ + cptr=cptr->next; + } + return newitem; +} + +void cJSON_Minify(char *json) +{ + char *into=json; + while (*json) + { + if (*json == ' ') json++; + else if (*json == '\t') json++; /* Whitespace characters. */ + else if (*json == '\r') json++; + else if (*json == '\n') json++; + else if (*json == '/' && json[1] == '/') while (*json && *json!='\n') json++; /* double-slash comments, to end of line. */ + else if (*json == '/' && json[1] == '*') {while (*json && !(*json == '*' && json[1] == '/')) json++;json+=2;} /* multiline comments. */ + else if (*json == '\"') {*into++=*json++;while (*json && *json!='\"') {if (*json == '\\') *into++=*json++;*into++=*json++;}*into++=*json++;} /* string literals, which are \" sensitive. */ + else *into++=*json++; /* All other characters. */ + } + *into=0; /* and null-terminate. */ +} diff --git a/src/cJSON.h b/src/cJSON.h new file mode 100644 index 0000000..72b8a69 --- /dev/null +++ b/src/cJSON.h @@ -0,0 +1,127 @@ +#ifndef cJSON__h +#define cJSON__h + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* cJSON Types: */ +#define cJSON_False 0 +#define cJSON_True 1 +#define cJSON_NULL 2 +#define cJSON_Number 3 +#define cJSON_String 4 +#define cJSON_Array 5 +#define cJSON_Object 6 + +#define cJSON_IsReference 256 +#define cJSON_StringIsConst 512 + +/* The cJSON structure: */ +typedef struct cJSON { + struct cJSON *next,*prev; /* next/prev allow you to walk array/object chains. Alternatively, use GetArraySize/GetArrayItem/GetObjectItem */ + struct cJSON *child; /* An array or object item will have a child pointer pointing to a chain of the items in the array/object. */ + + int type; /* The type of the item, as above. */ + + char *valuestring; /* The item's string, if type == cJSON_String */ + int valueint; /* The item's number, if type == cJSON_Number */ + double valuedouble; /* The item's number, if type == cJSON_Number */ + + char *string; /* The item's name string, if this item is the child of, or is in the list of subitems of an object. */ +} cJSON; + +typedef struct cJSON_Hooks { + void *(*malloc_fn)(size_t sz); + void (*free_fn)(void *ptr); +} cJSON_Hooks; + +/* Supply malloc, realloc and free functions to cJSON */ +extern void cJSON_InitHooks(cJSON_Hooks *hooks); + + +/* Supply a block of JSON, and this returns a cJSON object you can interrogate. Call cJSON_Delete when finished. */ +extern cJSON *cJSON_Parse(const char *value); +/* Render a cJSON entity to text for transfer/storage. Free the char *when finished. */ +extern char *cJSON_Print(cJSON *item); +/* Render a cJSON entity to text for transfer/storage without any formatting. Free the char *when finished. */ +extern char *cJSON_PrintUnformatted(cJSON *item); +/* Render a cJSON entity to text using a buffered strategy. prebuffer is a guess at the final size. guessing well reduces reallocation. fmt=0 gives unformatted, =1 gives formatted */ +extern char *cJSON_PrintBuffered(cJSON *item,int prebuffer,int fmt); +/* Delete a cJSON entity and all subentities. */ +extern void cJSON_Delete(cJSON *c); + +/* Returns the number of items in an array (or object). */ +extern int cJSON_GetArraySize(cJSON *array); +/* Retrieve item number "item" from array "array". Returns NULL if unsuccessful. */ +extern cJSON *cJSON_GetArrayItem(cJSON *array,int item); +/* Get item "string" from object. Case insensitive. */ +extern cJSON *cJSON_GetObjectItem(cJSON *object,const char *string); + +/* For analysing failed parses. This returns a pointer to the parse error. You'll probably need to look a few chars back to make sense of it. Defined when cJSON_Parse() returns 0. 0 when cJSON_Parse() succeeds. */ +extern const char *cJSON_GetErrorPtr(void); + +/* These calls create a cJSON item of the appropriate type. */ +extern cJSON *cJSON_CreateNull(void); +extern cJSON *cJSON_CreateTrue(void); +extern cJSON *cJSON_CreateFalse(void); +extern cJSON *cJSON_CreateBool(int b); +extern cJSON *cJSON_CreateNumber(double num); +extern cJSON *cJSON_CreateString(const char *string); +extern cJSON *cJSON_CreateArray(void); +extern cJSON *cJSON_CreateObject(void); + +/* These utilities create an Array of count items. */ +extern cJSON *cJSON_CreateIntArray(const int *numbers,int count); +extern cJSON *cJSON_CreateFloatArray(const float *numbers,int count); +extern cJSON *cJSON_CreateDoubleArray(const double *numbers,int count); +extern cJSON *cJSON_CreateStringArray(const char **strings,int count); + +/* Append item to the specified array/object. */ +extern void cJSON_AddItemToArray(cJSON *array, cJSON *item); +extern void cJSON_AddItemToObject(cJSON *object,const char *string,cJSON *item); +extern void cJSON_AddItemToObjectCS(cJSON *object,const char *string,cJSON *item); /* Use this when string is definitely const (i.e. a literal, or as good as), and will definitely survive the cJSON object */ +/* Append reference to item to the specified array/object. Use this when you want to add an existing cJSON to a new cJSON, but don't want to corrupt your existing cJSON. */ +extern void cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item); +extern void cJSON_AddItemReferenceToObject(cJSON *object,const char *string,cJSON *item); + +/* Remove/Detatch items from Arrays/Objects. */ +extern cJSON *cJSON_DetachItemFromArray(cJSON *array,int which); +extern void cJSON_DeleteItemFromArray(cJSON *array,int which); +extern cJSON *cJSON_DetachItemFromObject(cJSON *object,const char *string); +extern void cJSON_DeleteItemFromObject(cJSON *object,const char *string); + +/* Update array items. */ +extern void cJSON_InsertItemInArray(cJSON *array,int which,cJSON *newitem); /* Shifts pre-existing items to the right. */ +extern void cJSON_ReplaceItemInArray(cJSON *array,int which,cJSON *newitem); +extern void cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem); + +/* Duplicate a cJSON item */ +extern cJSON *cJSON_Duplicate(cJSON *item,int recurse); +/* Duplicate will create a new, identical cJSON item to the one you pass, in new memory that will +need to be released. With recurse!=0, it will duplicate any children connected to the item. +The item->next and ->prev pointers are always zero on return from Duplicate. */ + +/* ParseWithOpts allows you to require (and check) that the JSON is null terminated, and to retrieve the pointer to the final byte parsed. */ +extern cJSON *cJSON_ParseWithOpts(const char *value,const char **return_parse_end,int require_null_terminated); + +extern void cJSON_Minify(char *json); + +/* Macros for creating things quickly. */ +#define cJSON_AddNullToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateNull()) +#define cJSON_AddTrueToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateTrue()) +#define cJSON_AddFalseToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateFalse()) +#define cJSON_AddBoolToObject(object,name,b) cJSON_AddItemToObject(object, name, cJSON_CreateBool(b)) +#define cJSON_AddNumberToObject(object,name,n) cJSON_AddItemToObject(object, name, cJSON_CreateNumber(n)) +#define cJSON_AddStringToObject(object,name,s) cJSON_AddItemToObject(object, name, cJSON_CreateString(s)) + +/* When assigning an integer value, it needs to be propagated to valuedouble too. */ +#define cJSON_SetIntValue(object,val) ((object)?(object)->valueint=(object)->valuedouble=(val):(val)) +#define cJSON_SetNumberValue(object,val) ((object)?(object)->valueint=(object)->valuedouble=(val):(val)) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/cetus-error.h b/src/cetus-error.h new file mode 100644 index 0000000..523c7f8 --- /dev/null +++ b/src/cetus-error.h @@ -0,0 +1,18 @@ +#ifndef _CETUS_ERROR_H_ +#define _CETUS_ERROR_H_ + +/** + * ERROR codes extends ER_xx from mysqld_error.h + */ + +enum { + ER_CETUS_UNKNOWN = 5001, + ER_CETUS_RESULT_MERGE, + ER_CETUS_LONG_RESP, + ER_CETUS_PARSE_SHARDING, + ER_CETUS_NOT_SUPPORTED, + ER_CETUS_SINGLE_NODE_FAIL, + ER_CETUS_NO_GROUP, +}; + +#endif /*_CETUS_ERROR_H_*/ diff --git a/src/cetus-log.c b/src/cetus-log.c new file mode 100644 index 0000000..7da61e9 --- /dev/null +++ b/src/cetus-log.c @@ -0,0 +1,259 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cetus-log.h" + +#define TC_PREFIX "/var/log/" +#define TC_ERROR_LOG_PATH "xa.log" +#define TC_ERR_LOG_TIME_LEN (sizeof("2012-07-31 12:35:00 +999") - 1) +#define TC_ERR_LOG_TIME_STR_LEN (TC_ERR_LOG_TIME_LEN + 1) + +#define tc_cpymem(d, s, l) (((char *) memcpy(d, (void *) s, l)) + (l)) + + +static int log_fd = -1; +static int last_hour = -1; +static const char *file_name_prefix = NULL; +static char tc_error_log_time[TC_ERR_LOG_TIME_STR_LEN]; + +static int update_time(); + +typedef struct { + char *level; + int len; +} tc_log_level_t; + +static tc_log_level_t tc_log_levels[] = { + { "[unknown]", 9 }, + { "[emerg]", 7 }, + { "[alert]", 7 }, + { "[crit]", 6 }, + { "[error]", 7 }, + { "[warn]", 6 }, + { "[notice]", 8}, + { "[info]", 6}, + { "[debug]", 7 } +}; + + +static int +tc_vscnprintf(char *buf, size_t size, const char *fmt, va_list args) +{ + int i; + + /* + * Attention for vsnprintf: http://lwn.net/Articles/69419/ + */ + i = vsnprintf(buf, size, fmt, args); + + if (i < (int) size) { + return i; + } + + if (size >= 1) { + return size - 1; + } else { + return 0; + } +} + + +static void +tc_localtime(time_t sec, struct tm *tm) +{ +#if (HAVE_LOCALTIME_R) + (void) localtime_r(&sec, tm); +#else + struct tm *t; + + t = localtime(&sec); + *tm = *t; +#endif + + tm->tm_mon++; + tm->tm_year += 1900; +} + +static int +tc_scnprintf(char *buf, size_t size, const char *fmt, ...) +{ + int i; + va_list args; + + va_start(args, fmt); + i = tc_vscnprintf(buf, size, fmt, args); + va_end(args); + + return i; +} + + +static int +tc_create_new_file(const char *file, int hour) +{ + int len; + char new_file_path[512], *p; + + if (file == NULL) { + len = strlen(TC_PREFIX); + if (len >= 256) { + fprintf(stderr, "file prefix too long: %s\n", TC_PREFIX); + return -1; + } + strncpy(new_file_path, TC_PREFIX, len); + p = new_file_path + len; + len += strlen(TC_ERROR_LOG_PATH); + if (len >= 256) { + fprintf(stderr, "file path too long: %s\n", TC_PREFIX); + return -1; + } + strcpy(p, TC_ERROR_LOG_PATH); + p = new_file_path + len; + + sprintf(p, "_%2d", hour); + } else { + p = new_file_path; + len = strlen(file); + strcpy(p, file); + p = p + len; + sprintf(p, "_%02d", hour); + } + + file = new_file_path; + +#if (PROXY_O_SYNC) + log_fd = open(file, O_RDWR|O_CREAT|O_APPEND|O_SYNC, 0644); +#else + log_fd = open(file, O_RDWR|O_CREAT|O_APPEND, 0644); +#endif + + if (log_fd == -1) { + fprintf(stderr, "Open log file:%s error\n", file); + } + + return log_fd; +} + +int +tc_log_init(const char *file) +{ + file_name_prefix = file; + update_time(); + tc_create_new_file(file, last_hour); + return log_fd; +} + +int +tc_get_log_hour() +{ + if (last_hour == -1) { + update_time(); + } + + return last_hour; +} + +void +tc_log_end(void) +{ + if (log_fd != -1) { + close(log_fd); + } + + log_fd = -1; +} + + +static int +update_time() { + int status; + time_t sec; + struct tm tm; + struct timeval tv; + + status = gettimeofday(&tv, NULL); + if (status >= 0) { + sec = tv.tv_sec; + long msec = tv.tv_usec / 1000; + + tc_localtime(sec, &tm); + + snprintf(tc_error_log_time, TC_ERR_LOG_TIME_STR_LEN, + "%4d/%02d/%02d %02d:%02d:%02d +%03d", + tm.tm_year, tm.tm_mon, + tm.tm_mday, tm.tm_hour, + tm.tm_min, tm.tm_sec, + (int) msec); + if (tm.tm_hour != last_hour) { + if (last_hour == -1) { + last_hour = tm.tm_hour; + } else { + last_hour = tm.tm_hour; + return 1; + } + } + } + + return 0; +} + + +void +tc_log_info(int level, int err, const char *fmt, ...) +{ + int n, len; + char buffer[LOG_MAX_LEN], *p; + va_list args; + tc_log_level_t *ll; + + if (log_fd == -1) { + return; + } + + if (update_time()) { + tc_log_end(); + tc_create_new_file(file_name_prefix, last_hour); + } + + ll = &tc_log_levels[level]; + + p = buffer; + + p = tc_cpymem(p, tc_error_log_time, TC_ERR_LOG_TIME_LEN); + *p++ = ' '; + + p = tc_cpymem(p, ll->level, ll->len); + *p++ = ' '; + + n = len = TC_ERR_LOG_TIME_LEN + ll->len + 2; + va_start(args, fmt); + len += tc_vscnprintf(p, LOG_MAX_LEN - n, fmt, args); + va_end(args); + + if (len < n) { + return; + } + + p = buffer + len; + + if (err > 0) { + len += tc_scnprintf(p, LOG_MAX_LEN - len, " (%s)", strerror(err)); + if (len < (p - buffer)) { + return; + } + + p = buffer + len; + } + + *p++ = '\n'; + + write(log_fd, buffer, p - buffer); +} + diff --git a/src/cetus-log.h b/src/cetus-log.h new file mode 100644 index 0000000..53fa4f6 --- /dev/null +++ b/src/cetus-log.h @@ -0,0 +1,70 @@ +#ifndef TC_LOG_INCLUDED +#define TC_LOG_INCLUDED + +#include "chassis-exports.h" + +#define LOG_STDERR 0 +#define LOG_EMERG 1 +#define LOG_ALERT 2 +#define LOG_CRIT 3 +#define LOG_ERR 4 +#define LOG_WARN 5 +#define LOG_NOTICE 6 +#define LOG_INFO 7 +#define LOG_DEBUG 8 + +#define LOG_MAX_LEN 2048 + +int tc_log_init(const char *); +int tc_get_log_hour(); +void tc_log_end(void); + +void tc_log_info(int level, int err, const char *fmt, ...); + +#if (TC_DEBUG) + +#define tc_log_debug0(level, err, fmt) \ + tc_log_info(level, err, (const char *) fmt) + +#define tc_log_debug1(level, err, fmt, a1) \ + tc_log_info(level, err, (const char *) fmt, a1) + +#define tc_log_debug2(level, err, fmt, a1, a2) \ + tc_log_info(level, err, (const char *) fmt, a1, a2) + +#define tc_log_debug3(level, err, fmt, a1, a2, a3) \ + tc_log_info(level, err, (const char *) fmt, a1, a2, a3) + +#define tc_log_debug4(level, err, fmt, a1, a2, a3, a4) \ + tc_log_info(level, err, (const char *) fmt, a1, a2, a3, a4) + +#define tc_log_debug5(level, err, fmt, a1, a2, a3, a4, a5) \ + tc_log_info(level, err, (const char *) fmt, a1, a2, a3, a4, a5) + +#define tc_log_debug6(level, err, fmt, a1, a2, a3, a4, a5, a6) \ + tc_log_info(level, err, (const char *) fmt, a1, a2, a3, a4, a5, a6) + +#define tc_log_debug7(level, err, fmt, a1, a2, a3, a4, a5, a6, a7) \ + tc_log_info(level, err, (const char *) fmt, a1, a2, a3, a4, a5, a6, a7) + +#define tc_log_debug8(level, err, fmt, a1, a2, a3, a4, a5, a6, a7, a8) \ + tc_log_info(level, err, (const char *) fmt, a1, a2, a3, a4, a5, a6, a7, a8) + +#else + +#define tc_log_debug0(level, err, fmt) +#define tc_log_debug1(level, err, fmt, a1) +#define tc_log_debug2(level, err, fmt, a1, a2) +#define tc_log_debug3(level, err, fmt, a1, a2, a3) +#define tc_log_debug4(level, err, fmt, a1, a2, a3, a4) +#define tc_log_debug5(level, err, fmt, a1, a2, a3, a4, a5) +#define tc_log_debug6(level, err, fmt, a1, a2, a3, a4, a5, a6) +#define tc_log_debug7(level, err, fmt, a1, a2, a3, a4, a5, a6, a7) +#define tc_log_debug8(level, err, fmt, a1, a2, a3, a4, a5, a6, a7, a8) +#define tc_log_debug_trace(level, err, flag, ip, tcp) + +#endif /* TC_DEBUG */ + +#endif /* TC_LOG_INCLUDED */ + + diff --git a/src/cetus-monitor.c b/src/cetus-monitor.c new file mode 100644 index 0000000..0685df4 --- /dev/null +++ b/src/cetus-monitor.c @@ -0,0 +1,509 @@ +#include "cetus-monitor.h" + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cetus-users.h" +#include "cetus-util.h" +#include "chassis-timings.h" +#include "chassis-event.h" +#include "glib-ext.h" +#include "sharding-config.h" + +#define CHECK_ALIVE_INTERVAL 3 +#define CHECK_ALIVE_TIMES 2 +#define CHECK_DELAY_INTERVAL 300 * 1000 /* 300ms */ + +/* Each backend should have db and table */ +#define HEARTBEAT_DB "proxy_heart_beat" + +struct cetus_monitor_t { + struct chassis *chas; + GThread *thread; + chassis_event_loop_t *evloop; + + struct event check_alive_timer; + struct event write_master_timer; + struct event read_slave_timer; + struct event check_config_timer; + + GString *db_passwd; + GHashTable *backend_conns; + + GList *registered_objects; + char *config_id; +}; + +static void mysql_conn_free(gpointer e) +{ + MYSQL *conn = e; + if (conn) { + mysql_close(conn); + } +} + +static char *get_current_sys_timestr(void) +{ + struct timeval tv; + gettimeofday(&tv, NULL); +#define BUFSIZE 32 + char *time_sec = g_malloc(BUFSIZE); + strftime(time_sec, BUFSIZE, "%Y-%m-%d %H:%M:%S", localtime(&tv.tv_sec)); + char *time_micro = g_strdup_printf("%s.%06ld", time_sec, tv.tv_usec); + g_free(time_sec); + return time_micro; +} + +static MYSQL *get_mysql_connection(cetus_monitor_t *monitor, char *addr) +{ + MYSQL *conn = g_hash_table_lookup(monitor->backend_conns, addr); + if (conn) { + if (mysql_ping(conn) == 0) { + return conn; + } else { + g_hash_table_remove(monitor->backend_conns, addr); + g_message("monitor: remove dead mysql conn of backend: %s", addr); + } + } + + conn = mysql_init(NULL); + if (!conn) + return NULL; + + unsigned int timeout = 2 * SECONDS; + mysql_options(conn, MYSQL_OPT_CONNECT_TIMEOUT, &timeout); + mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout); + mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); + + char **ip_port = g_strsplit(addr, ":", -1); + int port = atoi(ip_port[1]); + char *user = monitor->chas->default_username; + if (mysql_real_connect(conn, ip_port[0], user, monitor->db_passwd->str, + NULL, port, NULL, 0) == NULL) + { + g_critical("monitor thread cannot connect to backend: %s@%s", + monitor->chas->default_username, addr); + mysql_conn_free(conn); + g_strfreev(ip_port); + return NULL; + } + g_hash_table_insert(monitor->backend_conns, g_strdup(addr), conn); + g_message("monitor thread connected to backend: %s, cached %d conns", + addr, g_hash_table_size(monitor->backend_conns)); + g_strfreev(ip_port); + return conn; +} + +#define ADD_MONITOR_TIMER(ev_struct, ev_cb, timeout) \ + evtimer_set(&(monitor->ev_struct), ev_cb, monitor);\ + event_base_set(monitor->evloop, &(monitor->ev_struct));\ + evtimer_add(&(monitor->ev_struct), &timeout); + +static void check_backend_alive(int fd, short what, void *arg) +{ + cetus_monitor_t *monitor = arg; + chassis *chas = monitor->chas; + + int i; + network_backends_t *bs = chas->priv->backends; + for (i = 0; i < network_backends_count(bs); i++) { + network_backend_t *backend = network_backends_get(bs, i); + if (backend->state == BACKEND_STATE_DELETED || + backend->state == BACKEND_STATE_MAINTAINING) + continue; + + char *backend_addr = backend->addr->name->str; + int check_count = 0; + MYSQL *conn = NULL; + while (++check_count <= CHECK_ALIVE_TIMES) { + conn = get_mysql_connection(monitor, backend_addr); + if (conn) break; + } + + if (conn == NULL) { + if (backend->state != BACKEND_STATE_DOWN) { + network_backends_modify(bs, i, backend->type, BACKEND_STATE_DOWN); + g_critical("Backend %s is set to DOWN.", backend_addr); + } + g_debug("Backend %s is not ALIVE!", backend_addr); + } else { + if (backend->state != BACKEND_STATE_UP) { + network_backends_modify(bs, i, backend->type, BACKEND_STATE_UP); + g_message("Backend %s is set to UP.", backend_addr); + } + g_debug("Backend %s is ALIVE!", backend_addr); + } + } + + struct timeval timeout = {0}; + timeout.tv_sec = CHECK_ALIVE_INTERVAL; + ADD_MONITOR_TIMER(check_alive_timer, check_backend_alive, timeout); +} + +static void check_slave_timestamp(int fd, short what, void *arg); + +static void update_master_timestamp(int fd, short what, void *arg) +{ + cetus_monitor_t *monitor = arg; + chassis *chas = monitor->chas; + int i; + network_backends_t *bs = chas->priv->backends; + /* Catch RW time + * Need a table to write from master and read from slave. + * CREATE TABLE `tb_heartbeat` ( + * `p_id` varchar(128) NOT NULL, + * `p_ts` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + * PRIMARY KEY (`p_id`) + * ) ENGINE = InnoDB DEFAULT CHARSET = utf8; + */ + for (i = 0; i < network_backends_count(bs); i++) { + network_backend_t *backend = network_backends_get(bs, i); + if (backend->state == BACKEND_STATE_DELETED || + backend->state == BACKEND_STATE_MAINTAINING) + continue; + + if (backend->type == BACKEND_TYPE_RW) { + static char sql[1024]; + char *cur_time_str = get_current_sys_timestr(); + snprintf(sql, sizeof(sql), "INSERT INTO %s.tb_heartbeat (p_id, p_ts)" + " VALUES ('%s', '%s') ON DUPLICATE KEY UPDATE p_ts='%s'", + HEARTBEAT_DB, monitor->config_id, cur_time_str, cur_time_str); + + char *backend_addr = backend->addr->name->str; + MYSQL *conn = get_mysql_connection(monitor, backend_addr); + if (conn == NULL) { + if (backend->state != BACKEND_STATE_DOWN) { + network_backends_modify(bs, i, backend->type, BACKEND_STATE_DOWN); + g_critical("Backend %s is set to DOWN.", backend_addr); + } + } else { + if (backend->state != BACKEND_STATE_UP) { + network_backends_modify(bs, i, backend->type, BACKEND_STATE_UP); + g_message("Backend %s is set to UP.", backend_addr); + } + static int previous_result[256] = {0}; /* for each backend group */ + int result = mysql_real_query(conn, L(sql)); + if (result != previous_result[i] && result != 0) { + g_critical("Update heartbeat error: %d, text: %s, backend: %s", + mysql_errno(conn), mysql_error(conn), backend_addr); + } else if (result != previous_result[i] && result == 0) { + g_message("Update heartbeat success. backend: %s", backend_addr); + } + previous_result[i] = result; + } + g_free(cur_time_str); + } + } + + /* Wait 50ms for RO write data */ + struct timeval timeout = {0}; + timeout.tv_usec = 50 * 1000; + ADD_MONITOR_TIMER(read_slave_timer, check_slave_timestamp, timeout); +} + +static void check_slave_timestamp(int fd, short what, void *arg) +{ + cetus_monitor_t *monitor = arg; + chassis *chas = monitor->chas; + int i; + network_backends_t *bs = chas->priv->backends; + + /* Read delay sec and set slave UP/DOWN according to delay_secs */ + for (i = 0; i < network_backends_count(bs); i++) { + network_backend_t *backend = network_backends_get(bs, i); + if (backend->type == BACKEND_TYPE_RW ||backend->state == BACKEND_STATE_DELETED || + backend->state == BACKEND_STATE_MAINTAINING) + continue; + + char *backend_addr = backend->addr->name->str; + MYSQL *conn = get_mysql_connection(monitor, backend_addr); + if (conn == NULL) { + g_critical("Connection error when read delay from RO backend: %s", backend_addr); + if (backend->state != BACKEND_STATE_DOWN) { + network_backends_modify(bs, i, backend->type, BACKEND_STATE_DOWN); + g_critical("Backend %s is set to DOWN.", backend_addr); + } + continue; + } + + static char sql[512]; + snprintf(sql, sizeof(sql), "select p_ts from %s.tb_heartbeat where p_id='%s'", + HEARTBEAT_DB, monitor->config_id); + static int previous_result[256] = {0}; /* for each backend group */ + int result = mysql_real_query(conn, L(sql)); + if (result != previous_result[i] && result != 0) { + g_critical("Select heartbeat error: %d, text: %s, backend: %s", + mysql_errno(conn), mysql_error(conn), backend_addr); + } else if (result != previous_result[i] && result == 0) { + g_message("Select heartbeat success. backend: %s", backend_addr); + } + previous_result[i] = result; + if (result == 0) { + MYSQL_RES *rs_set = mysql_store_result(conn); + MYSQL_ROW row = mysql_fetch_row(rs_set); + double ts_slave; + if (row != NULL) { + if (strstr(row[0],".") != NULL) { + char **tms = g_strsplit(row[0], ".", -1); + glong ts_slave_sec = chassis_epoch_from_string(tms[0], NULL); + double ts_slave_msec = atof(tms[1]); + ts_slave = ts_slave_sec + ts_slave_msec/1000; + g_strfreev(tms); + } else { + ts_slave = chassis_epoch_from_string(row[0], NULL); + } + } else { + g_critical("Check slave delay no data:%s", sql); + ts_slave = (double)G_MAXINT32; + } + if (ts_slave != 0) { + struct timeval tv; + gettimeofday(&tv, NULL); + double ts_now = tv.tv_sec + ((double)tv.tv_usec)/1000000; + double delay_secs = ts_now - ts_slave; + backend->slave_delay_msec = (int)delay_secs * 1000; + if (delay_secs > chas->slave_delay_down_threshold_sec && + backend->state != BACKEND_STATE_DOWN) + { + network_backends_modify(bs, i, backend->type, BACKEND_STATE_DOWN); + g_critical("Slave delay %.3f seconds. Set slave to DOWN.", delay_secs); + } else if (delay_secs <= chas->slave_delay_recover_threshold_sec && + backend->state != BACKEND_STATE_UP) + { + network_backends_modify(bs, i, backend->type, BACKEND_STATE_UP); + g_message("Slave delay %.3f seconds. Recovered. Set slave to UP.", delay_secs); + } + } + mysql_free_result(rs_set); + } + } + struct timeval timeout = {0}; + timeout.tv_usec = CHECK_DELAY_INTERVAL; + ADD_MONITOR_TIMER(write_master_timer, update_master_timestamp, timeout); +} + +#define MON_MAX_NAME_LEN 128 +struct monitored_object_t { + char name[MON_MAX_NAME_LEN]; + monitor_callback_fn func; + void *arg; +}; + +struct event config_reload_timer; + +static void check_config_worker(int fd, short what, void *arg) +{ + cetus_monitor_t *monitor = arg; + chassis *chas = monitor->chas; + chassis_config_t *conf = chas->config_manager; + struct timeval timeout = {0}; + GList *l; + + for (l = monitor->registered_objects; l; l = l->next) { + struct monitored_object_t *ob = l->data; + if (chassis_config_is_object_outdated(conf, ob->name)) { + /* if (evtimer_pending(&config_reload_timer, NULL)) + break; */ + g_message("monitor: object `%s` is outdated, try updating...", ob->name); + + /* first read in object from remote in monitor thread */ + chassis_config_update_object_cache(conf, ob->name); + + /* then switch to main thread */ + evtimer_set(&config_reload_timer, ob->func, ob->arg); + event_base_set(chas->event_base, &config_reload_timer); + evtimer_add(&config_reload_timer, &timeout); + break; /* TODO: for now, only update one object each time */ + } + } + + timeout.tv_sec = 5; + ADD_MONITOR_TIMER(check_config_timer, check_config_worker, timeout); +} + +void cetus_monitor_open(cetus_monitor_t *monitor, monitor_type_t monitor_type) +{ + struct timeval timeout; + switch (monitor_type) { + case MONITOR_TYPE_CHECK_ALIVE: + timeout.tv_sec = CHECK_ALIVE_INTERVAL; + timeout.tv_usec = 0; + ADD_MONITOR_TIMER(check_alive_timer, check_backend_alive, timeout); + g_message("check_alive monitor open."); + break; + case MONITOR_TYPE_CHECK_DELAY: + timeout.tv_sec = 0; + timeout.tv_usec = CHECK_DELAY_INTERVAL; + ADD_MONITOR_TIMER(write_master_timer, update_master_timestamp, timeout); + g_message("check_slave monitor open."); + break; + case MONITOR_TYPE_CHECK_CONFIG: + timeout.tv_sec = 5; + timeout.tv_usec = 0; + ADD_MONITOR_TIMER(check_config_timer, check_config_worker, timeout); + g_message("check_config monitor open."); + break; + default: + break; + } +} + +void cetus_monitor_close(cetus_monitor_t *monitor, monitor_type_t monitor_type) +{ + switch (monitor_type) { + case MONITOR_TYPE_CHECK_ALIVE: + if (monitor->check_alive_timer.ev_base) { + evtimer_del(&monitor->check_alive_timer); + } + g_message("check_alive monitor close."); + break; + case MONITOR_TYPE_CHECK_DELAY: + if (monitor->write_master_timer.ev_base) { + evtimer_del(&monitor->write_master_timer); + } + if (monitor->read_slave_timer.ev_base) { + evtimer_del(&monitor->read_slave_timer); + } + g_message("check_slave monitor close."); + break; + case MONITOR_TYPE_CHECK_CONFIG: + if (monitor->check_config_timer.ev_base) { + evtimer_del(&monitor->check_config_timer); + } + g_message("check_config monitor close."); + break; + default: + break; + } +} + +static void *cetus_monitor_mainloop(void *data) +{ + cetus_monitor_t *monitor = data; + + chassis_event_loop_t *loop = chassis_event_loop_new(); + monitor->evloop = loop; + + chassis *chas = monitor->chas; + monitor->config_id = chassis_config_get_id(chas->config_manager); + if (!chas->default_username) { + g_warning("default-username not set, monitor will not work"); + return NULL; + } + + cetus_users_get_server_pwd(chas->priv->users, + chas->default_username, monitor->db_passwd); + if (monitor->db_passwd->len == 0) { /* TODO: retry */ + g_warning("no password for %s, monitor will not work", chas->default_username); + return NULL; + } + monitor->backend_conns = g_hash_table_new_full( + g_str_hash, g_str_equal, g_free, mysql_conn_free); + + if (!chas->check_slave_delay) { + cetus_monitor_open(monitor, MONITOR_TYPE_CHECK_ALIVE); + } + if (chas->check_slave_delay) { + cetus_monitor_open(monitor, MONITOR_TYPE_CHECK_DELAY); + } +#if 0 + cetus_monitor_open(monitor, MONITOR_TYPE_CHECK_CONFIG); +#endif + chassis_event_loop(loop); + + g_message("monitor thread closing %d mysql conns", + g_hash_table_size(monitor->backend_conns)); + g_hash_table_destroy(monitor->backend_conns); + mysql_thread_end(); + + g_debug("exiting monitor loop"); + chassis_event_loop_free(loop); + return NULL; +} + +void cetus_monitor_start_thread(cetus_monitor_t *monitor, chassis *chas) +{ + monitor->chas = chas; + if (chas->disable_threads) { + g_message("monitor thread is disabled"); + return; + } + + g_assert(monitor->thread == 0); + + GThread *new_thread = NULL; +#if !GLIB_CHECK_VERSION(2, 32, 0) + GError *error = NULL; + new_thread = g_thread_create(cetus_monitor_mainloop, + monitor, TRUE, &error); + if (new_thread == NULL && error != NULL) { + g_critical("Create thread error: %s", error->message); + } +#else + new_thread = g_thread_new("monitor-thread", + cetus_monitor_mainloop, monitor); + if (new_thread == NULL) { + g_critical("Create thread error."); + } +#endif + + monitor->thread = new_thread; + g_message("monitor thread started"); +} + +void cetus_monitor_stop_thread(cetus_monitor_t *monitor) +{ + if (monitor->thread) { + g_message("Waiting for monitor thread to quit ..."); + g_thread_join(monitor->thread); + g_message("Monitor thread stopped"); + } +} + +cetus_monitor_t *cetus_monitor_new() +{ + cetus_monitor_t *monitor = g_new0(cetus_monitor_t, 1); + + monitor->db_passwd = g_string_new(0); + return monitor; +} + +void cetus_monitor_free(cetus_monitor_t *monitor) +{ + /* backend_conns should be freed in its own thread, not here */ + g_string_free(monitor->db_passwd, TRUE); + g_list_free_full(monitor->registered_objects, g_free); + if (monitor->config_id) g_free(monitor->config_id); + g_free(monitor); +} + +void cetus_monitor_register_object(cetus_monitor_t *monitor, + const char *name, monitor_callback_fn func, void *arg) +{ + GList *l; + struct monitored_object_t *object = NULL; + for (l = monitor->registered_objects; l; l = l->next) { + struct monitored_object_t *ob = l->data; + if (strcmp(ob->name, name) == 0) { + object = ob; + break; + } + } + if (!object) { + object = g_new0(struct monitored_object_t, 1); + strncpy(object->name, name, MON_MAX_NAME_LEN - 1); + monitor->registered_objects = g_list_append(monitor->registered_objects, object); + } + object->func = func; + object->arg = arg; +} diff --git a/src/cetus-monitor.h b/src/cetus-monitor.h new file mode 100644 index 0000000..2ddfcfa --- /dev/null +++ b/src/cetus-monitor.h @@ -0,0 +1,34 @@ +#ifndef _CHASSIS_REMOTE_CONFIGS_H_ +#define _CHASSIS_REMOTE_CONFIGS_H_ + +#include /* GPtrArray */ +#include "chassis-mainloop.h" + +typedef struct cetus_monitor_t cetus_monitor_t; + +typedef enum { + MONITOR_TYPE_CHECK_ALIVE, + MONITOR_TYPE_CHECK_DELAY, + MONITOR_TYPE_CHECK_CONFIG +} monitor_type_t; + +typedef void (*monitor_callback_fn)(int, short, void *); + +cetus_monitor_t *cetus_monitor_new(); + +void cetus_monitor_free(cetus_monitor_t *); + +void cetus_monitor_open(cetus_monitor_t *, monitor_type_t); + +void cetus_monitor_close(cetus_monitor_t *, monitor_type_t); + +void cetus_monitor_open(cetus_monitor_t *, monitor_type_t); + +void cetus_monitor_start_thread(cetus_monitor_t *, chassis *data); + +void cetus_monitor_stop_thread(cetus_monitor_t *); + +void cetus_monitor_register_object(cetus_monitor_t *, + const char *, monitor_callback_fn, void *); + +#endif diff --git a/src/cetus-query-queue.c b/src/cetus-query-queue.c new file mode 100644 index 0000000..c535d07 --- /dev/null +++ b/src/cetus-query-queue.c @@ -0,0 +1,100 @@ +#include "cetus-query-queue.h" + +#include +#include +#include + +#include "glib-ext.h" +#include "network-mysqld-proto.h" + +struct query_entry_t { + enum enum_server_command command; + GString *sql; + time_t recv_time; +}; + +static void query_entry_free(struct query_entry_t *entry) +{ + if (entry->sql) { + g_string_free(entry->sql, TRUE); + } + g_free(entry); +} + +query_queue_t *query_queue_new(int len) +{ + query_queue_t *q = g_new0(struct query_queue_t, 1); + q->chunks = g_queue_new(); + q->max_len = len; + return q; +} + +void query_queue_free(query_queue_t *q) +{ + g_queue_foreach(q->chunks, (GFunc)query_entry_free, NULL); + g_queue_free(q->chunks); + g_free(q); +} + +void query_queue_append(query_queue_t *q, GString *data) +{ + network_packet packet = {data, 0}; + guint8 command = 0; + if (packet.data) { + network_mysqld_proto_skip_network_header(&packet); + if (network_mysqld_proto_get_int8(&packet, &command) != 0) { + return; + } + } else { + g_warning("%s: packet data is nill ", G_STRLOC); + return; + } + + struct query_entry_t *entry = g_new0(struct query_entry_t, 1); + entry->command = command; + if (command == COM_QUERY) { + gsize sql_len = packet.data->len - packet.offset; + entry->sql = g_string_sized_new(sql_len + 1); + network_mysqld_proto_get_gstr_len(&packet, sql_len, entry->sql); + } + entry->recv_time = time(0); + + if (g_queue_get_length(q->chunks) >= q->max_len) { /* TODO: slow */ + struct query_entry_t *old = g_queue_pop_head(q->chunks); + query_entry_free(old); + } + g_queue_push_tail(q->chunks, entry); +} + +const char *command_name(enum enum_server_command cmd) +{ + static char number[8]; + switch(cmd) { + case COM_QUERY:return "COM_QUERY"; + case COM_QUIT:return "COM_QUIT"; + case COM_INIT_DB:return "COM_INIT_DB"; + case COM_FIELD_LIST:return "COM_FIELD_LIST"; + default: + snprintf(number, sizeof(number), "COM_<%d>", cmd);/* TODO: only works for 1 cmd */ + return number; + } +} + +static void query_entry_dump(gpointer data, gpointer user_data) +{ + struct query_entry_t *entry = (struct query_entry_t *)data; + struct tm *tm = localtime(&entry->recv_time); + char tm_str[16] = {0}; + snprintf(tm_str, sizeof(tm_str), "%d:%d:%d", tm->tm_hour, tm->tm_min, tm->tm_sec); + g_message("%s %s %s", tm_str, command_name(entry->command), + entry->sql ? entry->sql->str:""); +} + +void query_queue_dump(query_queue_t *q) +{ + if (!g_queue_is_empty(q->chunks)) { + g_message("recent queries:"); + g_queue_foreach(q->chunks, query_entry_dump, NULL); + } +} + diff --git a/src/cetus-query-queue.h b/src/cetus-query-queue.h new file mode 100644 index 0000000..41cc944 --- /dev/null +++ b/src/cetus-query-queue.h @@ -0,0 +1,16 @@ +#ifndef _CETUS_QUERY_QUEUE_H_ +#define _CETUS_QUERY_QUEUE_H_ + +#include + +typedef struct query_queue_t { + GQueue *chunks; + int max_len; +} query_queue_t; + +query_queue_t *query_queue_new(int max_len); +void query_queue_free(query_queue_t *); +void query_queue_append(query_queue_t *, GString *); +void query_queue_dump(query_queue_t *); + +#endif /* _CETUS_QUERY_QUEUE_H_*/ diff --git a/src/cetus-users.c b/src/cetus-users.c new file mode 100644 index 0000000..b676d07 --- /dev/null +++ b/src/cetus-users.c @@ -0,0 +1,288 @@ +#include "cetus-users.h" +#include +#include +#include +#include "cJSON.h" +#include "cetus-util.h" +#include "chassis-config.h" + +struct pwd_pair_t { + char *client; + char *server; +}; + +static struct pwd_pair_t *pwd_pair_new(const char *c, const char *s) +{ + struct pwd_pair_t *pwd = g_new0(struct pwd_pair_t, 1); + pwd->client = g_strdup(c); + pwd->server = g_strdup(s); + return pwd; +} + +static void pwd_pair_set_pwd(struct pwd_pair_t *pwd, + const char *new_pass, enum cetus_pwd_type t) +{ + switch (t) { + case CETUS_CLIENT_PWD: + g_free(pwd->client); + pwd->client = g_strdup(new_pass); + break; + case CETUS_SERVER_PWD: + g_free(pwd->server); + pwd->server = g_strdup(new_pass); + break; + default: + g_assert(0); + } +} + +static gboolean pwd_pair_same_pwd(struct pwd_pair_t *pwd, + const char *new_pass, enum cetus_pwd_type t) +{ + switch (t) { + case CETUS_CLIENT_PWD: + return strcmp(pwd->client, new_pass) == 0; + case CETUS_SERVER_PWD: + return strcmp(pwd->server, new_pass) == 0; + default: + return FALSE; + } +} + +static void pwd_pair_free(struct pwd_pair_t *pwd) +{ + if (pwd) { + g_free(pwd->client); + g_free(pwd->server); + g_free(pwd); + } +} + +cetus_users_t *cetus_users_new() +{ + cetus_users_t *users = g_new0(cetus_users_t, 1); + users->records = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, + (GDestroyNotify)pwd_pair_free); + return users; +} + +void cetus_users_free(cetus_users_t *users) +{ + if (users) { + if (users->records) + g_hash_table_destroy(users->records); + g_free(users); + } +} + +static void cetus_users_set_records(cetus_users_t *users, GHashTable *new_records) +{ + if (users->records) { + g_hash_table_destroy(users->records); + } + users->records = new_records; +} + +static gboolean cetus_users_parse_json(cetus_users_t *users, char *buffer) +{ + cJSON *root = cJSON_Parse(buffer); + if (!root) { + g_critical(G_STRLOC ":json syntax error"); + return FALSE; + } + + gboolean success = FALSE; + + GHashTable *user_records = g_hash_table_new_full(g_str_hash, g_str_equal, + g_free, (GDestroyNotify)pwd_pair_free); + + cJSON *users_node = cJSON_GetObjectItem(root, "users"); + cJSON *user_node = users_node ? users_node->child : NULL; + for (; user_node; user_node = user_node->next) { + cJSON *username = cJSON_GetObjectItem(user_node, "user"); + if (!username) { + g_critical(G_STRLOC ":user conf error, no username"); + goto out; + } + cJSON *client_pwd = cJSON_GetObjectItem(user_node, "client_pwd"); + cJSON *server_pwd = cJSON_GetObjectItem(user_node, "server_pwd"); + if (!client_pwd && !server_pwd) { + g_critical(G_STRLOC ":user conf error, at least one of client/server is needed"); + goto out; + } + if (client_pwd && !server_pwd) server_pwd = client_pwd; + if (!client_pwd && server_pwd) client_pwd = server_pwd; + g_hash_table_insert(user_records, g_strdup(username->valuestring), + pwd_pair_new(client_pwd->valuestring, server_pwd->valuestring)); + } + + success = TRUE; +out: + cJSON_Delete(root); + if (success) { + cetus_users_set_records(users, user_records); + } else { + g_hash_table_destroy(user_records); + } + return success; +} + +gboolean cetus_users_read_json(cetus_users_t *users, chassis_config_t *conf) +{ + users->conf_manager = conf; + char *buffer = NULL; + chassis_config_query_object(conf, "users", &buffer); + if (!buffer) + return FALSE; + gboolean success = cetus_users_parse_json(users, buffer); + if (success) { + g_message("read %d users", g_hash_table_size(users->records)); + } + g_free(buffer); + + return success; +} + +gboolean cetus_users_update_record(cetus_users_t *users, const char *user, + const char *pass, enum cetus_pwd_type type) +{ + struct pwd_pair_t *pwd = g_hash_table_lookup(users->records, user); + if (pwd) { + if (pwd_pair_same_pwd(pwd, pass, type)) { + return FALSE; + } + pwd_pair_set_pwd(pwd, pass, type); + } else { + g_hash_table_insert(users->records, g_strdup(user), pwd_pair_new(pass, pass)); + } + g_message("update user: %s", user); + return TRUE; +} + +gboolean cetus_users_delete_record(cetus_users_t *users, const char *user) +{ + gboolean found = g_hash_table_remove(users->records, user); + if (found) + g_message("delete user: %s", user); + return found; +} + +gboolean cetus_users_write_json(cetus_users_t *users) +{ + cJSON *users_node = cJSON_CreateArray(); + if (users_node == NULL) { + g_warning(G_STRLOC ":users_node is nil"); + return FALSE; + } + + GHashTableIter iter; + char *username = NULL; + struct pwd_pair_t *pwd = NULL; + g_hash_table_iter_init(&iter, users->records); + while (g_hash_table_iter_next(&iter, (gpointer *)&username, (gpointer *)&pwd)) { + cJSON *node = cJSON_CreateObject(); + cJSON_AddStringToObject(node, "user", username); + cJSON_AddStringToObject(node, "client_pwd", pwd->client); + cJSON_AddStringToObject(node, "server_pwd", pwd->server); + cJSON_AddItemToArray(users_node, node); + } + + cJSON *root = cJSON_CreateObject(); + cJSON_AddItemToObject(root, "users", users_node); + char *json_str = cJSON_Print(root); + + chassis_config_write_object(users->conf_manager, "users", json_str); + cJSON_Delete(root); + g_free(json_str); + + return TRUE; +} + +gboolean cetus_users_authenticate_client(cetus_users_t *users, + network_mysqld_auth_challenge *challenge, + network_mysqld_auth_response *response) +{ + char *user_name = response->username->str; + + struct pwd_pair_t *pwd = g_hash_table_lookup(users->records, user_name); + if (pwd == NULL) { + return FALSE; + } + + /* term 2: user_name and password must in frontend users/passwords */ + if (pwd->client) { + GString *sha1_pwd = g_string_new(NULL); + network_mysqld_proto_password_hash(sha1_pwd, pwd->client, strlen(pwd->client)); + GString *expected_response = g_string_new(NULL); + network_mysqld_proto_password_scramble(expected_response, + S(challenge->auth_plugin_data), S(sha1_pwd)); + + if (g_string_equal(response->auth_plugin_data, expected_response)) { + g_string_free(expected_response, TRUE); + g_string_free(sha1_pwd, TRUE); + return TRUE; + } + g_string_free(expected_response, TRUE); + g_string_free(sha1_pwd, TRUE); + } + return FALSE; +} + +void cetus_users_get_hashed_pwd(cetus_users_t *users, const char *user_name, + enum cetus_pwd_type type, GString *sha1pwd) +{ + if (type == CETUS_CLIENT_PWD) { + cetus_users_get_hashed_client_pwd(users, user_name, sha1pwd); + } else if (type == CETUS_SERVER_PWD) { + cetus_users_get_hashed_server_pwd(users, user_name, sha1pwd); + } +} + +void cetus_users_get_hashed_client_pwd(cetus_users_t *users, + const char *user_name, GString *sha1_pwd) +{ + struct pwd_pair_t *pwd = g_hash_table_lookup(users->records, user_name); + if (pwd == NULL) { + return; + } + if (pwd->client) { + network_mysqld_proto_password_hash(sha1_pwd, pwd->client, strlen(pwd->client)); + } +} + +void cetus_users_get_hashed_server_pwd(cetus_users_t *users, + const char *user_name, GString *sha1_pwd) +{ + struct pwd_pair_t *pwd = g_hash_table_lookup(users->records, user_name); + if (pwd == NULL) { + return; + } + if (pwd->server) { + network_mysqld_proto_password_hash(sha1_pwd, pwd->server, strlen(pwd->server)); + } +} + +void cetus_users_get_server_pwd(cetus_users_t *users, const char *user_name, + GString *res_pwd) +{ + struct pwd_pair_t *pwd = g_hash_table_lookup(users->records, user_name); + if (pwd == NULL) { + return; + } + if (pwd->server) { + g_string_assign(res_pwd, pwd->server); + } +} + +gboolean cetus_users_contains(cetus_users_t *users, const char *user_name) +{ + return g_hash_table_lookup(users->records, user_name) ? TRUE : FALSE; +} + +void cetus_users_reload_callback(int fd, short what, void *arg) +{ + cetus_users_t *users = arg; + gboolean ok = cetus_users_read_json(users, users->conf_manager); + if (!ok) + g_warning("reloading users failed"); +} diff --git a/src/cetus-users.h b/src/cetus-users.h new file mode 100644 index 0000000..364faf4 --- /dev/null +++ b/src/cetus-users.h @@ -0,0 +1,48 @@ +#ifndef _CETUS_USERS_H_ +#define _CETUS_USERS_H_ + +#include "glib-ext.h" +#include "network-mysqld-packet.h" + +typedef struct cetus_users_t { + chassis_config_t *conf_manager; + GHashTable *records; /* */ +} cetus_users_t; + +enum cetus_pwd_type { + CETUS_UNKNOWN_PWD, + CETUS_CLIENT_PWD, + CETUS_SERVER_PWD, +}; + +cetus_users_t *cetus_users_new(); + +void cetus_users_free(cetus_users_t *users); + +gboolean cetus_users_read_json(cetus_users_t *users, chassis_config_t *); + +gboolean cetus_users_update_record(cetus_users_t *users, const char *user, + const char *pass, enum cetus_pwd_type type); + +gboolean cetus_users_delete_record(cetus_users_t *users, const char *user); + +/* write current users to disk */ +gboolean cetus_users_write_json(cetus_users_t *users); + +gboolean cetus_users_authenticate_client(cetus_users_t *users, network_mysqld_auth_challenge *, + network_mysqld_auth_response *); + +void cetus_users_get_hashed_pwd(cetus_users_t *, const char *user, + enum cetus_pwd_type, GString *sha1pwd); + +void cetus_users_get_hashed_client_pwd(cetus_users_t *, const char *user, GString *sha1pwd); + +void cetus_users_get_hashed_server_pwd(cetus_users_t *, const char *user, GString *sha1pwd); + +void cetus_users_get_server_pwd(cetus_users_t *, const char *user, GString *pwd); + +gboolean cetus_users_contains(cetus_users_t *, const char *user); + +void cetus_users_reload_callback(int fd, short what, void *arg); + +#endif /*_CETUS_USERS_H_*/ diff --git a/src/cetus-util.c b/src/cetus-util.c new file mode 100644 index 0000000..3957609 --- /dev/null +++ b/src/cetus-util.c @@ -0,0 +1,79 @@ +#include +#include +#include +#include +#include +#include + +#include "glib-ext.h" +#include "cetus-util.h" + +void cetus_string_dequote(char *z) +{ + int quote; + int i, j; + if (z == 0) return; + quote = z[0]; + switch (quote) { + case '\'': break; + case '"': break; + case '`': break; /* For MySQL compatibility */ + default: return; + } + for (i = 1, j = 0; z[i]; i++) { + if (z[i] == quote) { + if (z[i+1] == quote) { /* quote escape */ + z[j++] = quote; + i++; + } else { + z[j++] = 0; + break; + } + } else if (z[i] == '\\') { /* slash escape */ + i++; + z[j++] = z[i]; + } else { + z[j++] = z[i]; + } + } +} + + +gboolean read_file_to_buffer(const char *filename, char **buffer) +{ + FILE *fp = fopen(filename, "r"); + if (!fp) { + g_critical(G_STRLOC ":cannot open user conf: %s", filename); + return FALSE; + } + const int MAX_FILE_SIZE = 1024 * 1024; /* 1M */ + fseek(fp, 0, SEEK_END); + int len = ftell(fp); + if (len < 0) { + g_warning(G_STRLOC ":%s", g_strerror(errno)); + fclose(fp); + return FALSE; + } + + if (len > MAX_FILE_SIZE) { + g_warning(G_STRLOC ":file too large"); + fclose(fp); + return FALSE; + } + + rewind(fp); + + *buffer = g_new(char, len + 1); + if (fread((*buffer), 1, len, fp) != len) { + g_warning(G_STRLOC ":len is not consistant"); + fclose(fp); + return FALSE; + } + + (*buffer)[len] = 0; + + fclose(fp); + + return TRUE; +} + diff --git a/src/cetus-util.h b/src/cetus-util.h new file mode 100644 index 0000000..fa14c63 --- /dev/null +++ b/src/cetus-util.h @@ -0,0 +1,29 @@ +#ifndef _GLIB_EXT_STRING_LEN_H_ +#define _GLIB_EXT_STRING_LEN_H_ + +#include + +/** + * simple macros to get the data and length of a "string" + * + * C() is for constant strings like "foo" + * S() is for GString's + */ +#define C(x) x, sizeof(x) - 1 +#define S(x) (x) ? (x)->str : NULL, (x) ? (x)->len : 0 +#define L(x) x, strlen(x) + +typedef int32_t BitArray; +#define SetBit(A,k) (A[(k/32)] |= (1 << (k%32))) +#define ClearBit(A,k) (A[(k/32)] &= ~(1 << (k%32))) +#define TestBit(A,k) (A[(k/32)] & (1 << (k%32))) + +void cetus_string_dequote(char *z); + +#define KB 1024 +#define MB 1024 * KB +#define GB 1024 * MB + +gboolean read_file_to_buffer(const char *filename, char **buffer); + +#endif diff --git a/src/cetus-variable.c b/src/cetus-variable.c new file mode 100644 index 0000000..46b39e4 --- /dev/null +++ b/src/cetus-variable.c @@ -0,0 +1,50 @@ +#include "cetus-variable.h" +#include "chassis-mainloop.h" + +#include +#include + +void cetus_variables_init_stats(cetus_variable_t **vars, chassis *chas) +{ + query_stats_t *stats = &(chas->query_stats); + + cetus_variable_t stats_variables[] = { + {"Com_select", &stats->com_select, VAR_INT64}, + {"Com_insert", &stats->com_insert, VAR_INT64}, + {"Com_update", &stats->com_update, VAR_INT64}, + {"Com_delete", &stats->com_delete, VAR_INT64}, + {"Com_select_shard", &stats->com_select_shard, VAR_INT64}, + {"Com_insert_shard", &stats->com_insert_shard, VAR_INT64}, + {"Com_update_shard", &stats->com_update_shard, VAR_INT64}, + {"Com_delete_shard", &stats->com_delete_shard, VAR_INT64}, + {"Com_select_global", &stats->com_select_global, VAR_INT64}, + {"Com_select_bad_key", &stats->com_select_bad_key, VAR_INT64}, + {NULL, NULL, 0} + }; + int length = sizeof(stats_variables); + int count = length / sizeof(cetus_variable_t); + *vars = calloc(count, sizeof(cetus_variable_t)); + memcpy(*vars, stats_variables, length); +} + +char *cetus_variable_get_value_str(cetus_variable_t *var) +{ + char *value = NULL; + switch (var->type) { + case VAR_INT: + value = g_strdup_printf("%u", *(gint *)(var->value)); + break; + case VAR_INT64: + value = g_strdup_printf("%lu", *(guint64 *)(var->value)); + break; + case VAR_FLOAT: + value = g_strdup_printf("%f", *(double *)(var->value)); + break; + case VAR_STRING: + value = g_strdup(var->value); + break; + default: + value = g_strdup("error value"); + } + return value; +} diff --git a/src/cetus-variable.h b/src/cetus-variable.h new file mode 100644 index 0000000..33e60e8 --- /dev/null +++ b/src/cetus-variable.h @@ -0,0 +1,27 @@ +#ifndef CETUS_VARIABLE_H +#define CETUS_VARIABLE_H + +#include + +enum cetus_variable_type_t { + VAR_INT, + VAR_INT64, + VAR_FLOAT, + VAR_STRING, +}; + +struct chassis; +typedef struct cetus_variable_t { + char *name; + void *value; + enum cetus_variable_type_t type; +} cetus_variable_t; + +void cetus_variables_init_stats(/* out */ cetus_variable_t **vars, struct chassis *); + +/** + @return newly allocated string, must be freed + */ +char *cetus_variable_get_value_str(cetus_variable_t *var); + +#endif /* CETUS_VARIABLES_H */ diff --git a/src/character-set.c b/src/character-set.c new file mode 100644 index 0000000..9a1f858 --- /dev/null +++ b/src/character-set.c @@ -0,0 +1,45 @@ +#include "character-set.h" + +#include +#include + +#define DEFAULT_CHARSET '\x21' + +int charset_get_number(const char *name) +{ + if (!name) + return DEFAULT_CHARSET; + static struct _charset_number_t { + const char *name; + int number; + } map[] = { + {"latin1", 0x08}, + {"big5", 0x01}, + {"gb2312", 0x18}, + {"gbk", 0x1c}, + {"utf8", 0x21}, + {"utf8mb4", 0x2d}, + {"binary", 0x3f} + }; + int map_len = sizeof(map)/sizeof(struct _charset_number_t); + int i = 0; + while (i < map_len) { + if (strcmp(name, map[i].name) == 0) { + return map[i].number; + } + ++i; + } + return DEFAULT_CHARSET; +} + +const char *charset_get_name(int number) +{ + assert(number < 64); + static const char *charset[64] = { + 0, "big5", 0, 0, 0, 0, 0, 0, "latin1", 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, "gb2312", 0, 0, 0, "gbk", 0, 0, 0, + 0, "utf8", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "utf8mb4", 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "binary" + }; + return charset[number]; +} diff --git a/src/character-set.h b/src/character-set.h new file mode 100644 index 0000000..8577c8d --- /dev/null +++ b/src/character-set.h @@ -0,0 +1,9 @@ +#ifndef CHARACTER_SET_H +#define CHARACTER_SET_H + +#include "network-exports.h" + +NETWORK_API int charset_get_number(const char *name); +NETWORK_API const char *charset_get_name(int number); + +#endif // CHARACTER_SET_H diff --git a/src/chassis-config.c b/src/chassis-config.c new file mode 100644 index 0000000..804d170 --- /dev/null +++ b/src/chassis-config.c @@ -0,0 +1,688 @@ +#include "chassis-config.h" +#include "chassis-timings.h" +#include "chassis-options.h" +#include "cetus-util.h" + +#include +#include +#include +#include +#include /* for g_stat */ + +enum chassis_config_type_t { + CHASSIS_CONF_SSH, + CHASSIS_CONF_FTP, + CHASSIS_CONF_MYSQL, + CHASSIS_CONF_SQLITE, + CHASSIS_CONF_LOCAL, /* maybe unify local directory? */ +}; + +#define RF_MAX_NAME_LEN 128 +typedef struct chassis_config_object_t { + char name[RF_MAX_NAME_LEN]; + + /* TODO: cache lock */ + char *cache; + time_t mtime; +} config_object_t; + +static void config_object_free(config_object_t *ob) +{ + if (ob->cache) g_free(ob->cache); + g_free(ob); +} + +struct chassis_config_t +{ + enum chassis_config_type_t type; + char *user; + char *password; + char *host; + int port; + char *schema; + +/* this mysql conn might be used in 2 threads, + * on startup, the main thread use it to load config + * while running, the monitor thread use it periodically + * for now, it is guaranteed not used simutaneously, so it's not locked + */ + MYSQL *mysql_conn; + + char *options_table; + char *options_filter; + GHashTable *options; + + GList *objects; +}; + +static gboolean url_parse_user_pass(chassis_config_t *rconf, const char *userpass, int len) +{ + char *p = strndup(userpass, len); + char *sep = strchr(p, ':'); + if (sep) { + rconf->user = strndup(p, sep-p); + rconf->password = strdup(sep+1); + g_free(p); + } else { + rconf->user = p; + } + return TRUE; +} + +static gboolean url_parse_host_port(chassis_config_t *rconf, const char *hostport, int len) +{ + char *p = strndup(hostport, len); + char *sep = strchr(p, ':'); + if (sep) { + rconf->host = strndup(p, sep-p); + rconf->port = atoi(sep+1); + g_free(p); + } else { + rconf->host = p; + rconf->port = 3306; + } + return TRUE; +} + +static gboolean url_parse_parameter(chassis_config_t *rconf, const char *param, int len) +{ + char **params = g_strsplit(param, "&", -1); + int i = 0; + GString *filter = g_string_new(0); + for (i = 0; params[i]; ++i) { + if (strncasecmp(params[i], "table=", 6) == 0) { + rconf->options_table = strdup(params[i]+6); + } else { + g_string_append(filter, params[i]); + g_string_append(filter, " and "); + } + } + if (filter->len >= 4) { /* remove last 'and' */ + g_string_truncate(filter, filter->len - 4); + rconf->options_filter = strdup(filter->str); + } + g_string_free(filter, TRUE); + g_strfreev(params); + return TRUE; +} + +/* example mysql://user:pass@host:port/schema?table=xx&id=xx */ +static gboolean chassis_config_parse_mysql_url(chassis_config_t *rconf, + const char *url, int len) +{ +/* only "host" is required -> [dbuser[:[dbpassword]]@]host[:port][/schema] */ + char *param = strchr(url, '?'); + char *schema = strchr(url, '/'); + char *at = strchr(url, '@'); + + if (schema) { + if (param && param < schema) + return FALSE; + rconf->schema = param ? strndup(schema+1, param-schema-1) : strdup(schema+1); + } + if (param) + url_parse_parameter(rconf, param+1, url+len - param); + if (!rconf->options_table) + rconf->options_table = g_strdup("settings"); + + const char *hostend = schema ? schema : (param ? param : url+len); + gboolean ok = FALSE; + if (at) { + ok = url_parse_user_pass(rconf, url, at-url); + ok = ok && url_parse_host_port(rconf, at+1, hostend-at-1); + } else { + ok = url_parse_host_port(rconf, url, hostend-url); + } + return ok; +} + +static MYSQL *chassis_config_get_mysql_connection(chassis_config_t *conf) +{ + /* first try the cached connection */ + if (conf->mysql_conn) { + if (mysql_ping(conf->mysql_conn) == 0) { + return conf->mysql_conn; + } else { + mysql_close(conf->mysql_conn); + conf->mysql_conn = NULL; + } + } + + MYSQL *conn = mysql_init(NULL); + if (!conn) + return NULL; + + unsigned int timeout = 2 * SECONDS; + mysql_options(conn, MYSQL_OPT_CONNECT_TIMEOUT, &timeout); + mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout); + mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); + + if (mysql_real_connect(conn, conf->host, conf->user, conf->password, + NULL, conf->port, NULL, 0) == NULL) + { + g_critical("%s", mysql_error(conn)); + mysql_close(conn); + return NULL; + } + conf->mysql_conn = conn; /* cache the connection */ + return conn; +} + +static gboolean chassis_config_mysql_init_tables(chassis_config_t *conf) +{ + MYSQL *conn = chassis_config_get_mysql_connection(conf); + if (!conn) { + g_critical("%s", mysql_error(conn)); + return FALSE; + } + char sql[256] = {0}; + snprintf(sql, sizeof(sql), "CREATE DATABASE IF NOT EXISTS %s", conf->schema); + if (mysql_query(conn, sql)) { + g_critical("%s", mysql_error(conn)); + return FALSE; + } + snprintf(sql, sizeof(sql), "CREATE TABLE IF NOT EXISTS %s.objects(" + "object_name varchar(64) NOT NULL," + "object_value text NOT NULL," + "mtime timestamp NOT NULL," + "PRIMARY KEY(object_name))", conf->schema); + if (mysql_query(conn, sql)) { + g_critical("%s", mysql_error(conn)); + return FALSE; + } + snprintf(sql, sizeof(sql), "CREATE TABLE IF NOT EXISTS %s.%s(" + "option_key varchar(64) NOT NULL," + "option_value varchar(1024) NOT NULL," + "PRIMARY KEY(option_key))", conf->schema, conf->options_table); + if (mysql_query(conn, sql)) { + g_critical("%s", mysql_error(conn)); + return FALSE; + } + return TRUE; +} + +chassis_config_t *chassis_config_from_url(char *url) +{ + chassis_config_t *rconf = g_new0(chassis_config_t, 1); + gboolean ok = FALSE; + if (strncasecmp(url, "mysql://", 8) == 0) { + rconf->type = CHASSIS_CONF_MYSQL; + ok = chassis_config_parse_mysql_url(rconf, url+8, strlen(url)-8); + } else if (strncasecmp(url, "sqlite://", 9) == 0) { + ok = chassis_config_parse_mysql_url(rconf, url+8, strlen(url)-8); + } else { + g_warning("url not supported: %s", url); + } + if (ok) + ok = chassis_config_mysql_init_tables(rconf); + + if (!ok) { + chassis_config_free(rconf); + return NULL; + } + return rconf; +} + +/* TODO: */ +chassis_config_t *chassis_config_from_local_dir(char *conf_dir, char *conf_file) +{ + chassis_config_t *conf = g_new0(chassis_config_t, 1); + conf->type = CHASSIS_CONF_LOCAL; + conf->schema = g_strdup(conf_dir); + conf->options_table = g_strdup(conf_file); + return conf; +} + +void chassis_config_free(chassis_config_t *p) +{ + if (!p) return; + if (p->user) g_free(p->user); + if (p->password) g_free(p->password); + if (p->host) g_free(p->host); + if (p->schema) g_free(p->schema); + if (p->options_table) g_free(p->options_table); + if (p->options_filter) g_free(p->options_filter); + if (p->options) g_hash_table_destroy(p->options); + if (p->objects) g_list_free_full(p->objects, (GDestroyNotify)config_object_free); + if (p->mysql_conn) { + mysql_close(p->mysql_conn); + } + mysql_thread_end(); + g_free(p); +} + +static gboolean chassis_config_load_options_mysql(chassis_config_t *conf) +{ + MYSQL *conn = chassis_config_get_mysql_connection(conf); + if (!conn) { + g_warning("chassis_config can't get mysql conn"); + goto mysql_error; + } + char sql[1024] = {0}; + snprintf(sql, sizeof(sql), "SELECT option_key,option_value FROM %s.%s", + conf->schema, conf->options_table); + if (mysql_query(conn, sql)) { + g_warning("sql failed: %s", sql); + goto mysql_error; + } + MYSQL_RES *result = mysql_store_result(conn); + if (!result) goto mysql_error; + + if (!conf->options) + conf->options = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free); + else + g_hash_table_remove_all(conf->options); + + MYSQL_ROW row; + while ((row = mysql_fetch_row(result))) { + g_hash_table_insert(conf->options, g_strdup(row[0]), g_strdup(row[1])); + } + mysql_free_result(result); + return TRUE; + +mysql_error: + return FALSE; +} + +static gboolean chassis_config_load_options(chassis_config_t *conf) +{ + switch (conf->type) { + case CHASSIS_CONF_MYSQL: + return chassis_config_load_options_mysql(conf); + default: + /* TODO g_critical(G_STRLOC " not implemented"); */ + return FALSE; + } +} + +GHashTable *chassis_config_get_options(chassis_config_t *conf) +{ + if (!conf->options) + chassis_config_load_options(conf); + return conf->options; +} + + +static void chassis_config_object_set_cache(config_object_t *ob, + const char *str, time_t mt) +{ + if (ob->cache) { + g_free(ob->cache); + } + ob->cache = g_strdup(str); + ob->mtime = mt; +} + +static config_object_t *chassis_config_get_object(chassis_config_t *conf, + const char *name) +{ + GList *l = NULL; + for (l = conf->objects; l; l = l->next) { + config_object_t *ob = l->data; + if (strcmp(ob->name, name) == 0) + return ob; + } + return NULL; +} + +static gboolean chassis_config_mysql_query_object(chassis_config_t *conf, + config_object_t *object, const char *name, char **json_res) +{ + g_assert(conf->type == CHASSIS_CONF_MYSQL); + if (object->cache) { + *json_res = g_strdup(object->cache); + return TRUE; + } + + gboolean status = FALSE; + MYSQL *conn = chassis_config_get_mysql_connection(conf); + + if (!conn) { + g_warning("Cannot connect to mysql server."); + goto mysql_error; + } + char sql[256] = {0}; + snprintf(sql, sizeof(sql), + "SELECT object_value,mtime FROM %s.objects where object_name='%s'", + conf->schema, name); + if (mysql_query(conn, sql)) { + g_warning("sql failed: %s", sql); + goto mysql_error; + } + MYSQL_RES *result = mysql_store_result(conn); + if (!result) + goto mysql_error; + + MYSQL_ROW row; + row = mysql_fetch_row(result); + if (!row) { + mysql_free_result(result); + goto mysql_error; + } + + *json_res = g_strdup(row[0]); + time_t mt = chassis_epoch_from_string(row[1], NULL); + + chassis_config_object_set_cache(object, row[0], mt); + mysql_free_result(result); + status = TRUE; +mysql_error: + return status; +} + +static gboolean chassis_config_local_query_object(chassis_config_t *conf, + config_object_t *object, const char *name, char **json_res) +{ + if (object->cache) { + *json_res = g_strdup(object->cache); + return TRUE; + } + + char basename[128] = {0}; + snprintf(basename, sizeof(basename), "%s.%s", name, "json"); + char *object_file = g_build_filename(conf->schema, basename, NULL); + char *buffer = NULL; + + if (read_file_to_buffer(object_file, &buffer) == FALSE) { + g_free(object_file); + g_free(buffer); + return FALSE; + } + + *json_res = buffer; + + GStatBuf sta; + if (g_stat(object_file, &sta) == 0) { + chassis_config_object_set_cache(object, buffer, sta.st_mtime); + } + + g_free(object_file); + + return TRUE; +} + +/* select a table, make it into a json */ +gboolean chassis_config_query_object(chassis_config_t *conf, + const char *name, char **json_res) +{ + config_object_t *object = chassis_config_get_object(conf, name); + if (!object) { + object = g_new0(config_object_t, 1); + strncpy(object->name, name, RF_MAX_NAME_LEN - 1); + conf->objects = g_list_append(conf->objects, object); + } + + switch (conf->type) { + case CHASSIS_CONF_MYSQL: + return chassis_config_mysql_query_object(conf, object, name, json_res); + case CHASSIS_CONF_LOCAL: + return chassis_config_local_query_object(conf, object, name, json_res); + default: + return FALSE; + } +} + +static gboolean chassis_config_mysql_write_object(chassis_config_t *conf, + config_object_t *object, const char *name, const char *json) +{ + g_assert(conf->type == CHASSIS_CONF_MYSQL); + time_t now = time(0); + GString *sql = g_string_new(0); + g_string_printf(sql, "REPLACE INTO %s.objects(object_name,object_value,mtime)" + " VALUES('%s','%s', FROM_UNIXTIME(%ld))", conf->schema, name, json, now); + + gboolean status = TRUE; + MYSQL *conn = chassis_config_get_mysql_connection(conf); + if (!conn) { + g_warning("Cannot connect to mysql server."); + status = FALSE; + } + if (mysql_query(conn, sql->str)) { + g_warning("sql failed: %s", sql->str); + status = FALSE; + } + chassis_config_object_set_cache(object, json, now); + g_string_free(sql, TRUE); + return status; +} + +static gboolean chassis_config_local_write_object(chassis_config_t *conf, + config_object_t *object, const char *name, const char *json_str) +{ + /* conf->schema is an absolute path for local config */ + char basename[128] = {0}; + snprintf(basename, sizeof(basename), "%s.%s", name, "json"); + char *object_file = g_build_filename(conf->schema, basename, NULL); + FILE *fp = fopen(object_file, "w"); /* truncate and write */ + if (!fp) { + g_critical(G_STRLOC "can't open file: %s for write", object_file); + g_free(object_file); + return FALSE; + } + fwrite(json_str, 1, strlen(json_str), fp); + fclose(fp); + + GStatBuf sta; + if (g_stat(object_file, &sta) == 0) { + chassis_config_object_set_cache(object, json_str, sta.st_mtime); + } + g_free(object_file); + return TRUE; +} + +gboolean chassis_config_write_object(chassis_config_t *conf, + const char *name, const char *json) +{ + config_object_t *object = chassis_config_get_object(conf, name); + if (!object) { + object = g_new0(config_object_t, 1); + strncpy(object->name, name, RF_MAX_NAME_LEN - 1); + conf->objects = g_list_append(conf->objects, object); + } + switch (conf->type) { + case CHASSIS_CONF_MYSQL: + return chassis_config_mysql_write_object(conf, object, name, json); + case CHASSIS_CONF_LOCAL: + return chassis_config_local_write_object(conf, object, name, json); + default: + return FALSE; + } +} + +gboolean chassis_config_parse_options(chassis_config_t *conf, GList *entries) +{ + GHashTable *opts_table = chassis_config_get_options(conf); + if (!opts_table) + return FALSE; + + GList *l; + for (l = entries; l ; l = l->next) { + chassis_option_t *entry = l->data; + /* already set by cmdline or config file */ + if ((entry->flags & OPTION_FLAG_CMDLINE) || + (entry->flags & OPTION_FLAG_CONF_FILE)) + continue; + + char *entry_value = g_hash_table_lookup(opts_table, entry->long_name); + if (entry_value) { + switch (entry->arg) { + case OPTION_ARG_NONE: + case OPTION_ARG_INT: + if (entry->arg_data == NULL) break; + *(int *)(entry->arg_data) = atoi(entry_value); + break; + case OPTION_ARG_STRING: { + if (entry->arg_data == NULL || *(char **)entry->arg_data != NULL) break; + char *value = g_strdup(entry_value); + *(char **)(entry->arg_data) = value; + break; + } + case OPTION_ARG_STRING_ARRAY: { + if (entry->arg_data == NULL || *(char **)entry->arg_data != NULL) break; + char **values = g_strsplit(entry_value, ",", -1); + *(char ***)(entry->arg_data) = values; + break; + } + case OPTION_ARG_DOUBLE: + *(double *)(entry->arg_data) = atof(entry_value); + break; + default: + g_warning("Unhandled option arg type: %d", entry->arg); + break; + } + } + } + return TRUE; +} + +gboolean chassis_config_mysql_is_object_outdated(chassis_config_t *conf, + config_object_t *object, + const char *name) +{ + MYSQL *conn = chassis_config_get_mysql_connection(conf); + if (!conn) + return FALSE; + static char sql[128] = {0}; + snprintf(sql, sizeof(sql), "SELECT mtime FROM %s.objects where object_name='%s'", + conf->schema, name); + if (mysql_query(conn, sql)) { + g_warning("sql failed: %s", sql); + return FALSE; + } + MYSQL_RES *result = mysql_store_result(conn); + if (!result) + return FALSE; + MYSQL_ROW row; + row = mysql_fetch_row(result); + if (!row) + return FALSE; + time_t mt = chassis_epoch_from_string(row[0], NULL); + mysql_free_result(result); + return object->mtime < mt; +} + +gboolean chassis_config_local_is_object_outdated(chassis_config_t *conf, + config_object_t *object, + const char *name) +{ + GStatBuf sta; + char basename[128] = {0}; + snprintf(basename, sizeof(basename), "%s.%s", name, "json"); + char *object_file = g_build_filename(conf->schema, basename, NULL); + if (g_stat(object_file, &sta)) { + g_free(object_file); + return FALSE; + } + g_free(object_file); + return object->mtime < sta.st_mtime; +} + +gboolean chassis_config_is_object_outdated(chassis_config_t *conf, const char *name) +{ + config_object_t *object = chassis_config_get_object(conf, name); + if (!object) { + return FALSE; + } + switch (conf->type) { + case CHASSIS_CONF_MYSQL: + return chassis_config_mysql_is_object_outdated(conf, object, name); + case CHASSIS_CONF_LOCAL: + return chassis_config_local_is_object_outdated(conf, object, name); + default: + return FALSE; + } +} + +void chassis_config_update_object_cache(chassis_config_t *conf, const char *name) +{ + config_object_t *object = chassis_config_get_object(conf, name); + if (!object) + return; + if (object->cache) { + g_free(object->cache); + object->cache = NULL; + } + char *str; + chassis_config_query_object(conf, name, &str); + if (str) { /* we just want to trigger query&caching, result is not needed */ + g_free(str); + } +} + +#define MAX_ID_SIZE 127 +char *chassis_config_get_id(chassis_config_t *conf) +{ + GString *id = g_string_new(0); + switch (conf->type) { + case CHASSIS_CONF_MYSQL: + g_string_printf(id, "%s_%d_%s", conf->host, conf->port, conf->schema); + break; + case CHASSIS_CONF_LOCAL: + g_string_assign(id, conf->schema); + break; + default: + g_string_assign(id, "error-config-id"); + g_assert(0); + break; + } + g_string_append_printf(id, "_%u_%u", g_random_int(), g_random_int()); + if (id->len >= MAX_ID_SIZE) { + g_string_erase(id, 0, id->len - MAX_ID_SIZE); + g_warning("id truncated to: %s", id->str); + } + char *id_str = id->str; + g_string_free(id, FALSE); + return id_str; +} + +gboolean chassis_config_register_service(chassis_config_t *conf, char *id, char *data) +{ + if (conf->type != CHASSIS_CONF_MYSQL) + return FALSE; + + MYSQL *conn = chassis_config_get_mysql_connection(conf); + if (!conn) { + g_critical("%s", mysql_error(conn)); + return FALSE; + } + + char sql[512] = {0}; + snprintf(sql, sizeof(sql), "CREATE TABLE IF NOT EXISTS %s.services(" + "id varchar(64) NOT NULL," + "data varchar(64) NOT NULL," + "start_time timestamp, PRIMARY KEY(id))", conf->schema); + if (mysql_query(conn, sql)) { + g_critical("%s", mysql_error(conn)); + return FALSE; + } + time_t now = time(0); + snprintf(sql, sizeof(sql), "INSERT INTO %s.services(id, data, start_time)" + " VALUES('%s','%s',FROM_UNIXTIME(%ld)) ON DUPLICATE KEY UPDATE" + " start_time=FROM_UNIXTIME(%ld)", + conf->schema, id, data, now, now); + if (mysql_query(conn, sql)) { + g_critical("%s", mysql_error(conn)); + return FALSE; + } + return TRUE; +} + +void chassis_config_unregister_service(chassis_config_t *conf, char *id) +{ + if (conf->type != CHASSIS_CONF_MYSQL) + return; + + MYSQL *conn = chassis_config_get_mysql_connection(conf); + if (!conn) { + g_critical("%s", mysql_error(conn)); + return; + } + + char sql[512] = {0}; + snprintf(sql, sizeof(sql), "DELETE FROM %s.services WHERE id='%s'", + conf->schema, id); + if (mysql_query(conn, sql)) { + g_critical("%s", mysql_error(conn)); + return; + } +} diff --git a/src/chassis-config.h b/src/chassis-config.h new file mode 100644 index 0000000..4a70aa1 --- /dev/null +++ b/src/chassis-config.h @@ -0,0 +1,39 @@ +#ifndef CHASSIS_CONFIG_H +#define CHASSIS_CONFIG_H + +#include "glib-ext.h" + +/** + * The config manager module, it manages `options` and `object` + * currently support `mysql` and `local` media type + */ + +typedef struct chassis_config_t chassis_config_t; + +chassis_config_t *chassis_config_from_url(char *url); + +chassis_config_t *chassis_config_from_local_dir(char *dir, char *conf_file); + +void chassis_config_free(chassis_config_t *); + +GHashTable *chassis_config_get_options(chassis_config_t *); + +gboolean chassis_config_parse_options(chassis_config_t *, GList *entries); + +gboolean chassis_config_query_object(chassis_config_t *, + const char *name, char **json); + +gboolean chassis_config_write_object(chassis_config_t *, + const char *name, const char *json); + +gboolean chassis_config_is_object_outdated(chassis_config_t *, const char *name); + +void chassis_config_update_object_cache(chassis_config_t *, const char *name); + +char *chassis_config_get_id(chassis_config_t *); + +gboolean chassis_config_register_service(chassis_config_t *conf, char *id, char *data); + +void chassis_config_unregister_service(chassis_config_t *conf, char *id); + +#endif /* CHASSIS_CONFIG_H */ diff --git a/src/chassis-event.c b/src/chassis-event.c new file mode 100644 index 0000000..3e36593 --- /dev/null +++ b/src/chassis-event.c @@ -0,0 +1,92 @@ +#include +#include + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_UNISTD_H +#include /* for write() */ +#endif + +#include /* for SOCK_STREAM and AF_UNIX/AF_INET */ + +#include + +#include "chassis-event.h" +#include "glib-ext.h" +#include "cetus-util.h" + +#define E_NET_CONNRESET ECONNRESET +#define E_NET_CONNABORTED ECONNABORTED +#define E_NET_INPROGRESS EINPROGRESS +#if EWOULDBLOCK == EAGAIN +/** + * some system make EAGAIN == EWOULDBLOCK which would lead to a + * error in the case handling + * + * set it to -1 as this error should never happen + */ +#define E_NET_WOULDBLOCK -1 +#else +#define E_NET_WOULDBLOCK EWOULDBLOCK +#endif + +void chassis_event_add_with_timeout(chassis *chas, struct event *ev, struct timeval *tv) { + event_base_set(chas->event_base, ev); +#if NETWORK_DEBUG_TRACE_EVENT + CHECK_PENDING_EVENT(ev); +#endif + event_add(ev, tv); + g_debug("%s:event add ev:%p",G_STRLOC, ev); +} + +/** + * add a event asynchronously + * + * @see network_mysqld_con_handle() + */ +void chassis_event_add(chassis *chas, struct event *ev) { + chassis_event_add_with_timeout(chas, ev, NULL); +} + + +chassis_event_loop_t *chassis_event_loop_new() { + return event_base_new(); +} + +void chassis_event_loop_free(chassis_event_loop_t *event) { + event_base_free(event); +} + +void *chassis_event_loop(chassis_event_loop_t *loop) { + + /** + * check once a second if we shall shutdown the proxy + */ + while (!chassis_is_shutdown()) { + struct timeval timeout; + int r; + + timeout.tv_sec = 1; + timeout.tv_usec = 0; + + r = event_base_loopexit(loop, &timeout); + if (r == -1) { + g_critical("%s: leaving chassis_event_loop early. failed", G_STRLOC); + break; + } + + r = event_base_dispatch(loop); + + if (r == -1) { + if (errno == EINTR) continue; + g_critical("%s: leaving chassis_event_loop early, errno != EINTR was: %s (%d)", + G_STRLOC, g_strerror(errno), errno); + break; + } + } + + return NULL; +} + diff --git a/src/chassis-event.h b/src/chassis-event.h new file mode 100644 index 0000000..dfc9b43 --- /dev/null +++ b/src/chassis-event.h @@ -0,0 +1,26 @@ +#ifndef _CHASSIS_EVENT_H_ +#define _CHASSIS_EVENT_H_ + +#include /* GPtrArray */ + +#include "chassis-exports.h" +#include "chassis-mainloop.h" + +#define CHECK_PENDING_EVENT(ev) \ + if (event_pending((ev), EV_READ|EV_WRITE|EV_TIMEOUT, NULL)) { \ + g_warning(G_STRLOC ": pending ev:%p, flags:%d", (ev), (ev)->ev_flags); \ + event_del(ev); \ + } + +CHASSIS_API void chassis_event_add(chassis *chas, struct event *ev); +CHASSIS_API void chassis_event_add_with_timeout(chassis *chas, + struct event *ev, struct timeval *tv); + +typedef struct event_base chassis_event_loop_t; + +CHASSIS_API chassis_event_loop_t *chassis_event_loop_new(); +CHASSIS_API void chassis_event_loop_free(chassis_event_loop_t *e); +CHASSIS_API void chassis_event_set_event_base(chassis_event_loop_t *e, struct event_base *event_base); +CHASSIS_API void *chassis_event_loop(chassis_event_loop_t *); + +#endif diff --git a/src/chassis-exports.h b/src/chassis-exports.h new file mode 100644 index 0000000..fbcd2a1 --- /dev/null +++ b/src/chassis-exports.h @@ -0,0 +1,7 @@ +#ifndef _CHASSIS_EXPORTS_ +#define _CHASSIS_EXPORTS_ + +#define CHASSIS_API extern + +#endif + diff --git a/src/chassis-filemode.c b/src/chassis-filemode.c new file mode 100644 index 0000000..332ee04 --- /dev/null +++ b/src/chassis-filemode.c @@ -0,0 +1,47 @@ + +#include +#include +#include +#include + +#include + +#include "chassis-filemode.h" + +/* + * check whether the given filename points to a file the permissions + * of which are 0 for group and other (ie read/writable only by owner). + * return 0 for "OK", -1 of the file cannot be accessed or is the wrong + * type of file, and 1 if permissions are wrong + * + * + * FIXME? this function currently ignores ACLs + */ +int chassis_filemode_check_full(const gchar *filename, int required_filemask, GError **gerr) { + struct stat stbuf; + mode_t fmode; + + if (stat(filename, &stbuf) == -1) { + g_set_error(gerr, G_FILE_ERROR, g_file_error_from_errno(errno), + "cannot stat(%s): %s", filename, + g_strerror(errno)); + return -1; + } + + fmode = stbuf.st_mode; + if ((fmode & S_IFMT) != S_IFREG) { + g_set_error(gerr, G_FILE_ERROR, G_FILE_ERROR_INVAL, + "%s isn't a regular file", filename); + return -1; + } + + if ((fmode & required_filemask) != 0) { + g_set_error(gerr, G_FILE_ERROR, G_FILE_ERROR_PERM, + "permissions of %s aren't secure (0660 or stricter required)", filename); + return 1; + } + +#undef MASK + + return 0; +} diff --git a/src/chassis-filemode.h b/src/chassis-filemode.h new file mode 100644 index 0000000..8cfadab --- /dev/null +++ b/src/chassis-filemode.h @@ -0,0 +1,12 @@ +#ifndef _CHASSIS_PERM_H_ +#define _CHASSIS_PERM_H_ + +#include +#include "chassis-exports.h" + +#include +#define CHASSIS_FILEMODE_SECURE_MASK (S_IROTH|S_IWOTH|S_IXOTH) + +CHASSIS_API int chassis_filemode_check_full(const gchar *, int , GError **); + +#endif diff --git a/src/chassis-frontend.c b/src/chassis-frontend.c new file mode 100644 index 0000000..4e9ba1c --- /dev/null +++ b/src/chassis-frontend.c @@ -0,0 +1,334 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#ifdef HAVE_UNISTD_H +#include +#endif +#include + +#include +#include +#include + +#include "glib-ext.h" +#include "chassis-config.h" +#include "chassis-frontend.h" +#include "chassis-path.h" +#include "chassis-plugin.h" +#include "chassis-keyfile.h" +#include "chassis-filemode.h" +#include "chassis-options.h" +#include "cetus-util.h" + +/** + * initialize the basic components of the chassis + */ +int chassis_frontend_init_glib() { + const gchar *check_str = NULL; +#if 0 + g_mem_set_vtable(glib_mem_profiler_table); +#endif + + if (!GLIB_CHECK_VERSION(2, 6, 0)) { + g_critical("the glib header is too old, need at least 2.6.0, got: %d.%d.%d", + GLIB_MAJOR_VERSION, GLIB_MINOR_VERSION, GLIB_MICRO_VERSION); + + return -1; + } + + check_str = glib_check_version(GLIB_MAJOR_VERSION, + GLIB_MINOR_VERSION, GLIB_MICRO_VERSION); + + if (check_str) { + g_critical("%s, got: lib=%d.%d.%d, headers=%d.%d.%d", + check_str, + glib_major_version, glib_minor_version, glib_micro_version, + GLIB_MAJOR_VERSION, GLIB_MINOR_VERSION, GLIB_MICRO_VERSION); + + return -1; + } + + if (!g_module_supported()) { + g_critical("loading modules is not supported on this platform"); + return -1; + } + +#if !GLIB_CHECK_VERSION(2, 32, 0) + /* GLIB below 2.32 must call thread_init */ + g_thread_init(NULL); +#endif + + return 0; +} + + +/** + * setup and check the basedir if nessesary + */ +int chassis_frontend_init_basedir(const char *prg_name, char **_base_dir) { + char *base_dir = *_base_dir; + + if (base_dir) { /* basedir is already known, check if it is absolute */ + if (!g_path_is_absolute(base_dir)) { + g_critical("%s: --basedir option must be an absolute path, but was %s", + G_STRLOC, + base_dir); + return -1; + } else { + return 0; + } + } + + /* find our installation directory if no basedir was given + * this is necessary for finding files when we daemonize + */ + base_dir = chassis_get_basedir(prg_name); + if (!base_dir) { + g_critical("%s: Failed to get base directory", + G_STRLOC); + return -1; + } + + *_base_dir = base_dir; + + return 0; + +} + +int chassis_frontend_init_plugin_dir(char **_plugin_dir, const char *base_dir) { + char *plugin_dir = *_plugin_dir; + + if (plugin_dir) return 0; + + plugin_dir = g_build_filename(base_dir, "lib", PACKAGE, "plugins", NULL); + + *_plugin_dir = plugin_dir; + + return 0; +} + +int chassis_frontend_load_plugins(GPtrArray *plugins, + const gchar *plugin_dir, gchar **plugin_names) +{ + int i; + + /* load the plugins */ + for (i = 0; plugin_names && plugin_names[i]; i++) { + chassis_plugin *p; +#define G_MODULE_PREFIX "lib" + /* we have to hack around some glib distributions that + * don't set the correct G_MODULE_SUFFIX, notably MacPorts + */ +#ifndef SHARED_LIBRARY_SUFFIX +#define SHARED_LIBRARY_SUFFIX G_MODULE_SUFFIX +#endif + char *plugin_filename; + /* skip trying to load a plugin when the parameter was --plugins= + that will never work... + */ + if (!g_strcmp0("", plugin_names[i])) { + continue; + } + + plugin_filename = g_strdup_printf("%s%c%s%s.%s", + plugin_dir, + G_DIR_SEPARATOR, + G_MODULE_PREFIX, + plugin_names[i], + SHARED_LIBRARY_SUFFIX); + + p = chassis_plugin_load(plugin_filename); + g_free(plugin_filename); + + if (NULL == p) { + g_critical("setting --plugin-dir= might help"); + return -1; + } + p->option_grp_name = g_strdup(plugin_names[i]); + + g_ptr_array_add(plugins, p); + } + return 0; +} + +int chassis_frontend_init_plugins(GPtrArray *plugins, + chassis_options_t *opts, chassis_config_t *config_manager, + int *argc_p, char ***argv_p, + GKeyFile *keyfile, + const char *keyfile_section_name, + GError **gerr) +{ + guint i; + for (i = 0; i < plugins->len; i++) { + GList *config_entries = NULL; + chassis_plugin *p = g_ptr_array_index(plugins, i); + + if (NULL != (config_entries = chassis_plugin_get_options(p))) { + chassis_options_add_options(opts, config_entries); + + if (FALSE == chassis_options_parse_cmdline(opts, argc_p, argv_p, gerr)) { + return -1; + } + /* parse the new options */ + if (keyfile) { + if (FALSE == chassis_keyfile_to_options_with_error(keyfile, + keyfile_section_name, config_entries, gerr)) + { + return -1; + } + } + /* Load from remote config first */ + if (config_manager) { + chassis_config_parse_options(config_manager, config_entries); + } + } + } + + return 0; +} + + +int chassis_frontend_init_base_options(int *argc_p, char ***argv_p, + int *print_version, + char **config_file, + GError **gerr) +{ + int ret = 0; + chassis_options_t *opts = chassis_options_new(); + chassis_options_set_cmdline_only_options(opts, print_version, config_file); + opts->ignore_unknown = TRUE; + + if (FALSE == chassis_options_parse_cmdline(opts, argc_p, argv_p, gerr)) { + ret = -1; + } + + chassis_options_free(opts); + return ret; +} + +GKeyFile *chassis_frontend_open_config_file(const char *filename, GError **gerr) { + GKeyFile *keyfile; + + if (chassis_filemode_check_full(filename, CHASSIS_FILEMODE_SECURE_MASK, gerr) != 0) { + return NULL; + } + keyfile = g_key_file_new(); + g_key_file_set_list_separator(keyfile, ','); + + if (FALSE == g_key_file_load_from_file(keyfile, filename, G_KEY_FILE_NONE, gerr)) { + g_key_file_free(keyfile); + + return NULL; + } + + return keyfile; +} + +/** + * setup the options that can only appear on the command-line + */ +int chassis_options_set_cmdline_only_options(chassis_options_t *opts, + int *print_version, + char **config_file) { + + chassis_options_add(opts, + "version", 'V', 0, OPTION_ARG_NONE, print_version, + "Show version", NULL); + + chassis_options_add(opts, + "defaults-file", 0, 0, OPTION_ARG_STRING, config_file, + "configuration file", ""); + + return 0; +} + + +int chassis_frontend_print_version() { + /* + * allow to pass down a build-tag at build-time + * which gets hard-coded into the binary + */ + g_print(" %s\n", PACKAGE_STRING); +#ifdef CHASSIS_BUILD_TAG + g_print(" build: %s\n", CHASSIS_BUILD_TAG); +#endif + g_print(" glib2: %d.%d.%d\n", GLIB_MAJOR_VERSION,GLIB_MINOR_VERSION,GLIB_MICRO_VERSION); + g_print(" libevent: %s\n", event_get_version()); + return 0; +} + +int chassis_frontend_print_plugin_versions(GPtrArray *plugins) { + guint i; + + g_print("-- modules\n"); + + for (i = 0; i < plugins->len; i++) { + chassis_plugin *p = plugins->pdata[i]; + + g_print(" %s: %s\n", p->name, p->version); + } + + return 0; +} + +/** + * log the versions of the initialized plugins + */ +int chassis_frontend_log_plugin_versions(GPtrArray *plugins) { + guint i; + + for (i = 0; i < plugins->len; i++) { + chassis_plugin *p = plugins->pdata[i]; + + g_message("plugin %s %s started", p->name, p->version); + } + + return 0; +} + +int chassis_frontend_write_pidfile(const char *pid_file, GError **gerr) { + int fd; + int ret = 0; + + gchar *pid_str; + + /** + * write the PID file + */ + + if (-1 == (fd = open(pid_file, O_WRONLY|O_TRUNC|O_CREAT, 0600))) { + g_set_error(gerr, + G_FILE_ERROR, + g_file_error_from_errno(errno), + "%s: open(%s) failed: %s", + G_STRLOC, + pid_file, + g_strerror(errno)); + + return -1; + } + + pid_str = g_strdup_printf("%d", getpid()); + + if (write(fd, pid_str, strlen(pid_str)) < 0) { + g_set_error(gerr, + G_FILE_ERROR, + g_file_error_from_errno(errno), + "%s: write(%s) of %s failed: %s", + G_STRLOC, + pid_file, + pid_str, + g_strerror(errno)); + ret = -1; + } + g_free(pid_str); + + close(fd); + + return ret; +} + diff --git a/src/chassis-frontend.h b/src/chassis-frontend.h new file mode 100644 index 0000000..5062032 --- /dev/null +++ b/src/chassis-frontend.h @@ -0,0 +1,134 @@ +#ifndef __CHASSIS_FRONTEND_H__ +#define __CHASSIS_FRONTEND_H__ + +#include + +#include "chassis-exports.h" +#include "chassis-options.h" + +/** + * @file + * a collections of common functions used by chassis frontends + * + * take a look at mysql-proxy-cli.c on what sequence to call these functions + */ + +/** + * setup glib and gmodule + * + * may abort of glib headers and glib libs don't match + * + * @return 0 on success, -1 on error + */ +CHASSIS_API int chassis_frontend_init_glib(void); + +/** + * detect the basedir + * + * if *_basedir is not NULL, don't change it + * otherwise extract the basedir from the prg_name + * + * @param prg_name program-name (usually argv[0]) + * @param _basedir user-supplied basedir + * @return 0 on success, -1 on error + */ +CHASSIS_API int chassis_frontend_init_basedir(const char *prg_name, char **_base_dir); + +/** + * open the configfile + * + * @param filename name of the configfile + * @param gerr a pointer to a clean GError *or + * NULL in case the possible error should be ignored + * + * @see g_key_file_free + */ +CHASSIS_API GKeyFile *chassis_frontend_open_config_file(const char *filename, GError **gerr); + +CHASSIS_API int chassis_frontend_init_plugin_dir(char **, const char *); + +/** + * extract --version and --defaults-file from comandline options + * + * @param argc_p pointer to the number of args in argv_p + * @param argv_p pointer to arguments to parse + * @param print_version pointer to int to set if --version is specified + * @param config_file pointer to char *to store if --defaults-file is specified + * @param gerr a pointer to a clean GError *or + * NULL in case the possible error should be ignored + */ +CHASSIS_API int chassis_frontend_init_base_options(int *argc_p, char ***argv_p, + int *print_version, + char **config_file, + GError **gerr); + +/** + * load the plugins + * + * loads the plugins from 'plugin_names' from the 'plugin_dir' and + * store their chassis_plugin structs in 'plugins' + * + * the filename of the plugin is constructed based depending on the platform + * + * @param plugins empty array + * @param plugin_dir directory to load the plugins from + * @param plugin_names NULL terminated list of plugin names + * + * @see chassis_frontend_init_plugins + */ +CHASSIS_API int chassis_frontend_load_plugins(GPtrArray *plugins, + const gchar *plugin_dir, + gchar **plugin_names); + +/** + * init the loaded plugins and setup their config + * + * @param plugins array of chassis_plugin structs + * @param option_ctx a fresh GOptionContext + * @param argc_p pointer to the number of args in argv_p + * @param argv_p pointer to arguments to parse + * @param keyfile the configfile + * @param base_dir base directory + * @param gerr a pointer to a clean GError *or + * NULL in case the possible error should be ignored + * + * @see chassis_frontend_init_basedir, chassis_frontend_init_plugins + */ +struct remote_config_t; +CHASSIS_API int chassis_frontend_init_plugins(GPtrArray *plugins, + chassis_options_t *option_ctx, + struct chassis_config_t *config_manager, + int *argc_p, char ***argv_p, + GKeyFile *keyfile, + const char *keyfile_section_name, + GError **gerr); + +/** + * print the version of the program + */ +CHASSIS_API int chassis_frontend_print_version(void); + +/** + * print the versions of the initialized plugins + */ +CHASSIS_API int chassis_frontend_print_plugin_versions(GPtrArray *plugins); + +/** + * log the versions of the initialized plugins + */ +CHASSIS_API int chassis_frontend_log_plugin_versions(GPtrArray *plugins); + +/** + * write the PID to a file + * + * @param pid_file name of the PID file + * @param gerr GError + * @return 0 on success, -1 on error + */ +CHASSIS_API int chassis_frontend_write_pidfile(const char *pid_file, GError **gerr); + +CHASSIS_API int chassis_options_set_cmdline_only_options(chassis_options_t *opts, + int *print_version, + char **config_file); + +#endif diff --git a/src/chassis-keyfile.c b/src/chassis-keyfile.c new file mode 100644 index 0000000..4ceed00 --- /dev/null +++ b/src/chassis-keyfile.c @@ -0,0 +1,115 @@ +#include "chassis-path.h" +#include "chassis-keyfile.h" +#include "chassis-options.h" + +/** + * map options from the keyfile to the config-options + * + * @returns FALSE on error, TRUE on success + * @added in 0.8.3 + */ +gboolean chassis_keyfile_to_options_with_error(GKeyFile *keyfile, + const gchar *ini_group_name, GList *config_entries, + GError **_gerr) +{ + GError *gerr = NULL; + gboolean ret = TRUE; + int j; + + if (NULL == keyfile) { + g_set_error(_gerr, G_FILE_ERROR, G_FILE_ERROR_INVAL, + "keyfile has to be set"); + return FALSE; + } + + if (!g_key_file_has_group(keyfile, ini_group_name)) { + /* the group doesn't exist, no config-entries to map */ + g_warning("(keyfile) has no group [%s]", ini_group_name); + return TRUE; + } + + GList *l; + /* set the defaults */ + for (l = config_entries; l; l = l->next) { + chassis_option_t *entry = l->data; + gchar *arg_string; + gchar **arg_string_array; + gboolean arg_bool = 0; + gint arg_int = 0; + gdouble arg_double = 0; + gsize len = 0; + + /* already set by command line */ + if (entry->flags & OPTION_FLAG_CMDLINE) + continue; + + switch (entry->arg) { + case OPTION_ARG_STRING: + /* is this option set already */ + if (entry->arg_data == NULL || *(char **)entry->arg_data != NULL) break; + + arg_string = g_key_file_get_string(keyfile, ini_group_name, + entry->long_name, &gerr); + if (!gerr) { + /* strip trailing spaces */ + *(gchar **)(entry->arg_data) = g_strchomp(arg_string); + } + break; + case OPTION_ARG_STRING_ARRAY: + /* is this option set already */ + if (entry->arg_data == NULL || *(char **)entry->arg_data != NULL) break; + + arg_string_array = g_key_file_get_string_list(keyfile, + ini_group_name, entry->long_name, &len, &gerr); + if (!gerr) { + for (j = 0; arg_string_array[j]; j++) { + arg_string_array[j] = g_strstrip(arg_string_array[j]); + } + *(gchar ***)(entry->arg_data) = arg_string_array; + } + break; + case OPTION_ARG_NONE: + arg_bool = g_key_file_get_boolean(keyfile, ini_group_name, + entry->long_name, &gerr); + if (!gerr) { + *(gboolean *)(entry->arg_data) = arg_bool; + } + break; + case OPTION_ARG_INT: + arg_int = g_key_file_get_integer(keyfile, ini_group_name, + entry->long_name, &gerr); + if (!gerr) { + *(gint *)(entry->arg_data) = arg_int; + } + break; + case OPTION_ARG_DOUBLE: + arg_double = g_key_file_get_double(keyfile, ini_group_name, + entry->long_name, &gerr); + if (!gerr) { + *(gdouble *)(entry->arg_data) = arg_double; + } + break; + default: + g_error("%s: (keyfile) the option %d can't be handled", + G_STRLOC, entry->arg); + break; + } + + if (gerr) { + if (gerr->code == G_KEY_FILE_ERROR_KEY_NOT_FOUND) { + /* ignore if this key isn't set in the config-file */ + g_error_free(gerr); + } else { + /* otherwise propage the error the higher level */ + g_propagate_error(_gerr, gerr); + ret = FALSE; + break; + } + gerr = NULL; + } else { + entry->flags |= OPTION_FLAG_CONF_FILE; + } + } + return ret; +} + diff --git a/src/chassis-keyfile.h b/src/chassis-keyfile.h new file mode 100644 index 0000000..8cb12cc --- /dev/null +++ b/src/chassis-keyfile.h @@ -0,0 +1,21 @@ +#ifndef _CHASSIS_KEYFILE_H_ +#define _CHASSIS_KEYFILE_H_ + +#include + +#include "chassis-exports.h" + +/** @addtogroup chassis */ +/*@{*/ +/** + * parse the configfile options into option entries + * + */ +gboolean chassis_keyfile_to_options_with_error(GKeyFile *keyfile, + const gchar *groupname, GList *config_entries, GError **gerr); + + +/*@}*/ + +#endif + diff --git a/src/chassis-limits.c b/src/chassis-limits.c new file mode 100644 index 0000000..7859cb1 --- /dev/null +++ b/src/chassis-limits.c @@ -0,0 +1,60 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_SYS_RESOURCE_H +#include +#endif +#include +#include + +#include "chassis-limits.h" + +gint64 chassis_fdlimit_get() { + struct rlimit max_files_rlimit; + + if (-1 == getrlimit(RLIMIT_NOFILE, &max_files_rlimit)) { + return -1; + } else { + return max_files_rlimit.rlim_cur; + } +} + + +/** + * set the upper limit of open files + * + * @return -1 on error, 0 on success + */ +int chassis_fdlimit_set(gint64 max_files_number) { + struct rlimit max_files_rlimit; + rlim_t hard_limit; + + if (-1 == getrlimit(RLIMIT_NOFILE, &max_files_rlimit)) { + return -1; + } + + hard_limit = max_files_rlimit.rlim_max; + + max_files_rlimit.rlim_cur = max_files_number; + /* + * raise the hard-limit too in case it is smaller + * than the soft-limit, otherwise we get a EINVAL + */ + if (hard_limit < max_files_number) { + max_files_rlimit.rlim_max = max_files_number; + } + + if (-1 == setrlimit(RLIMIT_NOFILE, &max_files_rlimit)) { + return -1; + } + + return 0; +} + diff --git a/src/chassis-limits.h b/src/chassis-limits.h new file mode 100644 index 0000000..6b1d0ce --- /dev/null +++ b/src/chassis-limits.h @@ -0,0 +1,15 @@ +#ifndef _CHASSIS_LIMITS_H_ +#define _CHASSIS_LIMITS_H_ + +#include /* GPtrArray */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "chassis-exports.h" + +CHASSIS_API int chassis_fdlimit_set(gint64 max_files_number); +CHASSIS_API gint64 chassis_fdlimit_get(void); + +#endif diff --git a/src/chassis-log.c b/src/chassis-log.c new file mode 100644 index 0000000..8941dfb --- /dev/null +++ b/src/chassis-log.c @@ -0,0 +1,400 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include /* close */ +#define EVENTLOG_ERROR_TYPE 0x0001 +#define EVENTLOG_WARNING_TYPE 0x0002 +#define EVENTLOG_INFORMATION_TYPE 0x0004 +#include + +#ifdef HAVE_SYSLOG_H +#include +#else +/* placeholder values for platforms not having syslog support */ +#define LOG_USER 0 /* placeholder for user-level syslog facility */ +#define LOG_CRIT 0 +#define LOG_ERR 0 +#define LOG_WARNING 0 +#define LOG_NOTICE 0 +#define LOG_INFO 0 +#define LOG_DEBUG 0 +#endif + +#include "sys-pedantic.h" +#include "chassis-log.h" + +#define S(x) x->str, x->len + +/** + * the mapping of our internal log levels various log systems + * + * Attention: this needs to be adjusted should glib ever + * change its log level ordering + */ +#define G_LOG_ERROR_POSITION 3 +const struct { + char *name; + GLogLevelFlags lvl; + int syslog_lvl; + int win_evtype; +} log_lvl_map[] = { /* syslog levels are different to the glib ones */ + { "error", G_LOG_LEVEL_ERROR, LOG_CRIT, EVENTLOG_ERROR_TYPE}, + { "critical", G_LOG_LEVEL_CRITICAL, LOG_ERR, EVENTLOG_ERROR_TYPE}, + { "warning", G_LOG_LEVEL_WARNING, LOG_WARNING, EVENTLOG_WARNING_TYPE}, + { "message", G_LOG_LEVEL_MESSAGE, LOG_NOTICE, EVENTLOG_INFORMATION_TYPE}, + { "info", G_LOG_LEVEL_INFO, LOG_INFO, EVENTLOG_INFORMATION_TYPE}, + { "debug", G_LOG_LEVEL_DEBUG, LOG_DEBUG, EVENTLOG_INFORMATION_TYPE}, + + { NULL, 0, 0, 0 } +}; + +static gboolean +chassis_log_rotate_reopen(chassis_log *log, gpointer userdata, GError **gerr); + +chassis_log *chassis_log_new(void) { + chassis_log *log; + + log = g_new0(chassis_log, 1); + + log->log_file_fd = -1; + log->log_ts_str = g_string_sized_new(sizeof("2004-01-01T00:00:00.000Z")); + log->log_ts_resolution = CHASSIS_RESOLUTION_DEFAULT; + log->min_lvl = G_LOG_LEVEL_CRITICAL; + + log->last_msg = g_string_new(NULL); + log->last_msg_ts = 0; + log->last_msg_count = 0; + log->rotate_func = NULL; + + chassis_log_set_rotate_func(log, chassis_log_rotate_reopen, NULL, NULL); + + return log; +} + +int chassis_log_set_level(chassis_log *log, const gchar *level) { + gint i; + + for (i = 0; log_lvl_map[i].name; i++) { + if (0 == strcmp(log_lvl_map[i].name, level)) { + log->min_lvl = log_lvl_map[i].lvl; + return 0; + } + } + + return -1; +} + +/** + * open the log-file + * + * open the log-file set in log->log_filename + * if no log-filename is set, returns TRUE + * + * FIXME: the return value is not following 'unix'-style (0 on success, -1 on error), + * nor does it say it is a gboolean. Has to be fixed in 0.9.0 + * + * @return TRUE on success, FALSE on error + */ +int chassis_log_open(chassis_log *log) { + if (!log->log_filename) return TRUE; + + log->log_file_fd = open(log->log_filename, O_RDWR | O_CREAT | O_APPEND, 0660); + + return (log->log_file_fd != -1); +} + +/** + * close the log-file + * + * @return 0 on success + * + * @see chassis_log_open + */ +int chassis_log_close(chassis_log *log) { + if (log->log_file_fd == -1) return 0; + + close(log->log_file_fd); + + log->log_file_fd = -1; + + return 0; +} + +void chassis_log_free(chassis_log *log) { + if (!log) return; + + chassis_log_close(log); + g_string_free(log->log_ts_str, TRUE); + g_string_free(log->last_msg, TRUE); + + if (log->log_filename) g_free(log->log_filename); + + if (log->rotate_func_data_destroy && log->rotate_func_data) { + log->rotate_func_data_destroy(log->rotate_func_data); + } + + g_free(log); +} + +static int chassis_log_update_timestamp(chassis_log *log) { + struct tm *tm; + GTimeVal tv; + time_t t; + GString *s = log->log_ts_str; + + g_get_current_time(&tv); + t = (time_t) tv.tv_sec; + tm = localtime(&t); + + s->len = strftime(s->str, s->allocated_len, "%Y-%m-%d %H:%M:%S", tm); + if (log->log_ts_resolution == CHASSIS_RESOLUTION_MS) + g_string_append_printf(s, ".%.3d", (int) tv.tv_usec/1000); + + return 0; +} + +static int chassis_log_write(chassis_log *log, int log_level, GString *str) { + if (-1 != log->log_file_fd) { + /* prepend a timestamp */ + if (-1 == write(log->log_file_fd, S(str))) { + /* writing to the file failed (Disk Full, what ever ... */ + + if (write(STDERR_FILENO, S(str)) >= 0) { + if (write(STDERR_FILENO, "\n", 1) >= 0) { + } + } + } else { + if (write(log->log_file_fd, "\n", 1) >= 0) { + } + } + } else { + if (write(STDERR_FILENO, S(str)) >=0) { + if (write(STDERR_FILENO, "\n", 1) >= 0) { + } + } + } + + return 0; +} + +/** + * skip the 'top_srcdir' from a string starting with G_STRLOC or + * __FILE__ if it is absolute + * + * /src/chassis-log.c will become src/chassis-log.c + * + * NOTE: the code assumes it is located in src/ or src\. i + * If it gets moves somewhere else + * it won't crash, but strip too much of pathname + */ +const char *chassis_log_skip_topsrcdir(const char *message) { + const char *my_filename = __FILE__; + int ndx; + + /* + * we want to strip everything that is before the src/ + * in the above example. If we don't get the srcdir name passed down + * as part of the __FILE__, don't try to parse it out + */ + if (!g_path_is_absolute(__FILE__)) { + return message; + } + + /* usually the message start with G_STRLOC which may contain a rather + * long, absolute path. If + * it matches the TOP_SRCDIR, we filter it out + * + * - strip what is the same as our __FILE__ + * - don't strip our own sub-path 'src/' + */ + for (ndx = 0; message[ndx]; ndx++) { + if (0 == strncmp(message + ndx, "src" G_DIR_SEPARATOR_S, + sizeof("src" G_DIR_SEPARATOR_S) - 1)) + { + break; + } + if (message[ndx] != my_filename[ndx]) break; + } + + if (message[ndx] != '\0') { + message += ndx; + } + + return message; +} + +/* + * chassis_log_rotate_reopen: + * @log: a #chassis_log + * @userdata: ... + * @gerr: ... + * + * default log-rotate function + * + * assumes that: + * - log-file is moved to new name + * - SIGHUP is sent to process to announce log-file rotation + * - we reopen the log-file + * + * Returns: %TRUE + */ +static gboolean +chassis_log_rotate_reopen(chassis_log *log, gpointer userdata, GError **gerr) { + chassis_log_close(log); + chassis_log_open(log); + + return TRUE; +} + +/** + * chassis_log_rotate: + * @log: a #chassis_log + * + * rotate the logfile + * + * calls the rotation function provided by the user + * (or the default log-rotation function) + * + * Returns: %TRUE if the log-file was rotated, %FALSE on error + */ +gboolean +chassis_log_rotate(chassis_log *log, GError **gerr) { + return log->rotate_func(log, log->rotate_func_data, gerr); +} + +/** + * chassis_log_set_rotate_func: + * @log: a #chassis_log + * @rotate_func: (allow-none): log-file rotation function + * @userdata: (allow-none): userdata that is passed to @rotate_func + * @userdata_free: (allow-none): a #GDestroyNotify that is called + * when the rotate-function gets unset + * + * set a rotate function that is called when logfile rotate is requested + * + * if @rotate_func is %NULL, the default log-rotation function is set + */ +void + chassis_log_set_rotate_func(chassis_log *log, chassis_log_rotate_func rotate_func, + gpointer userdata, GDestroyNotify userdata_free) { + + if (log->rotate_func_data && log->rotate_func_data_destroy) { + log->rotate_func_data_destroy(log->rotate_func_data); + log->rotate_func_data = NULL; + } + + if (NULL == rotate_func) { + log->rotate_func = chassis_log_rotate_reopen; + } else { + log->rotate_func = rotate_func; + } + + log->rotate_func_data = userdata; + log->rotate_func_data_destroy = userdata_free; + + return; + + } + +static void +chassis_log_func_locked(const gchar G_GNUC_UNUSED *log_domain, GLogLevelFlags log_level, + const gchar *message, gpointer user_data) { + chassis_log *log = user_data; + int i; + gchar *log_lvl_name = "(error)"; + gboolean is_duplicate = FALSE; + const char *stripped_message = chassis_log_skip_topsrcdir(message); + + /** + * rotate logs straight away if log->rotate_logs is true + * we do this before ignoring any log levels, so that rotation + * happens straight away - see Bug#55711 + */ + if (-1 != log->log_file_fd) { + if (log->rotate_logs) { + gboolean is_rotated; + + is_rotated = chassis_log_rotate(log, NULL); + + log->rotate_logs = FALSE; + + /* ->is_rotated stays set until the first message is logged + * again to make sure we don't hide it as duplicate + */ + if (log->is_rotated == FALSE && is_rotated == TRUE) { + log->is_rotated = TRUE; + } + } + } + + /* ignore the verbose log-levels */ + if (log_level > log->min_lvl) { + return; + } + + for (i = 0; log_lvl_map[i].name; i++) { + if (log_lvl_map[i].lvl == log_level) { + log_lvl_name = log_lvl_map[i].name; + break; + } + } + + if (log->last_msg->len > 0 && + 0 == strcmp(log->last_msg->str, stripped_message)) { + is_duplicate = TRUE; + } + + /** + * if the log has been rotated, we always dump the last message even if it + * was a duplicate. Otherwise, do not print duplicates unless they have been + * ignored at least 100 times, or they were last printed greater than + * 30 seconds ago. + */ + if (log->is_rotated || + !is_duplicate || + log->last_msg_count > 100 || + time(NULL) - log->last_msg_ts > 30) { + + /* if we lave the last message repeating, log it */ + if (log->last_msg_count) { + chassis_log_update_timestamp(log); + g_string_append_printf(log->log_ts_str, ": (%s) last message repeated %d times", + log_lvl_name, + log->last_msg_count); + + chassis_log_write(log, log_level, log->log_ts_str); + } + chassis_log_update_timestamp(log); + g_string_append(log->log_ts_str, ": ("); + g_string_append(log->log_ts_str, log_lvl_name); + g_string_append(log->log_ts_str, ") "); + + g_string_append(log->log_ts_str, stripped_message); + + /* reset the last-logged message */ + g_string_assign(log->last_msg, stripped_message); + log->last_msg_count = 0; + log->last_msg_ts = time(NULL); + + chassis_log_write(log, log_level, log->log_ts_str); + } else { + log->last_msg_count++; + } + + log->is_rotated = FALSE; +} + +void chassis_log_func(const gchar *log_domain, GLogLevelFlags log_level, + const gchar *message, gpointer user_data) +{ + chassis_log_func_locked(log_domain, log_level, message, user_data); +} + +void chassis_log_set_logrotate(chassis_log *log) { + log->rotate_logs = TRUE; +} + diff --git a/src/chassis-log.h b/src/chassis-log.h new file mode 100644 index 0000000..33e228f --- /dev/null +++ b/src/chassis-log.h @@ -0,0 +1,75 @@ +#ifndef _CHASSIS_LOG_H_ +#define _CHASSIS_LOG_H_ + +#include + +#include "chassis-exports.h" + +#define CHASSIS_RESOLUTION_SEC 0x0 +#define CHASSIS_RESOLUTION_MS 0x1 + +#define CHASSIS_RESOLUTION_DEFAULT CHASSIS_RESOLUTION_SEC + +/** @addtogroup chassis */ +/*@{*/ + +typedef struct _chassis_log chassis_log; + +/** + * chassis_log_rotate_func: + * + * prototype of the log rotation function + * + * used by #chassis_log_set_rotate_func and #chassis_log_rotate() + * + * the function has to + * - return %TRUE if log-file rotation was successful, + * %FALSE otherwise and set the @gerr accordingly + * - @user_data is passed through from #chassis_log_set_rotate_func() + * - @gerr is passed in from #chassis_log_rotate() + * + */ +typedef gboolean (*chassis_log_rotate_func)(chassis_log *log, gpointer user_data, GError **gerr); + + +struct _chassis_log { + GLogLevelFlags min_lvl; + + gchar *log_filename; + gint log_file_fd; + + gboolean rotate_logs; + + GString *log_ts_str; + gint log_ts_resolution; /*<< timestamp resolution (sec, ms) */ + + GString *last_msg; + time_t last_msg_ts; + guint last_msg_count; + + /* private */ + chassis_log_rotate_func rotate_func; + gpointer rotate_func_data; + GDestroyNotify rotate_func_data_destroy; + + gboolean is_rotated; +}; + + +CHASSIS_API chassis_log *chassis_log_new(void); +CHASSIS_API int chassis_log_set_level(chassis_log *log, const gchar *level); +CHASSIS_API void chassis_log_free(chassis_log *log); +CHASSIS_API int chassis_log_open(chassis_log *log); +CHASSIS_API int chassis_log_close(chassis_log *log); +CHASSIS_API void chassis_log_func(const gchar *log_domain, GLogLevelFlags log_level, + const gchar *message, gpointer user_data); +CHASSIS_API void chassis_log_set_logrotate(chassis_log *log); +CHASSIS_API const char *chassis_log_skip_topsrcdir(const char *message); + +CHASSIS_API void +chassis_log_set_rotate_func(chassis_log *log, chassis_log_rotate_func rotate_func, + gpointer userdata, GDestroyNotify userdata_free); + +/*@}*/ + +#endif diff --git a/src/chassis-mainloop.c b/src/chassis-mainloop.c new file mode 100644 index 0000000..f0b3eae --- /dev/null +++ b/src/chassis-mainloop.c @@ -0,0 +1,395 @@ +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_SYS_TIME_H +#include /* event.h need struct timeval */ +#endif + +#ifdef HAVE_PWD_H +#include /* getpwnam() */ +#endif +#include /* for SOCK_STREAM and AF_UNIX/AF_INET */ + +#include + +#include "glib-ext.h" +#include "network-mysqld.h" +#include "chassis-plugin.h" +#include "chassis-mainloop.h" +#include "chassis-event.h" +#include "chassis-log.h" +#include "chassis-timings.h" + +static volatile sig_atomic_t signal_shutdown; + +/** + * check if the libevent headers we built against match the + * library we run against + */ +int chassis_check_version(const char *lib_version, const char *hdr_version) { + int lib_maj, lib_min, lib_pat; + int hdr_maj, hdr_min, hdr_pat; + int scanned_fields; + + if (3 != (scanned_fields = sscanf(lib_version, "%d.%d.%d%*s", &lib_maj, + &lib_min, &lib_pat))) + { + g_critical("%s: library version %s failed to parse: %d", + G_STRLOC, lib_version, scanned_fields); + return -1; + } + if (3 != (scanned_fields = sscanf(hdr_version, "%d.%d.%d%*s", &hdr_maj, + &hdr_min, &hdr_pat))) + { + g_critical("%s: header version %s failed to parse: %d", + G_STRLOC, hdr_version, scanned_fields); + return -1; + } + + if (lib_maj == hdr_maj && + lib_min == hdr_min && + lib_pat >= hdr_pat) { + return 0; + } + + return -1; +} + + +/** + * create a global context + */ +chassis *chassis_new() { + chassis *chas; + + if (0 != chassis_check_version(event_get_version(), _EVENT_VERSION)) { + g_critical("%s: chassis is built against libevent %s, but now runs against %s", + G_STRLOC, _EVENT_VERSION, event_get_version()); + return NULL; + } + + chas = g_new0(chassis, 1); + + chas->modules = g_ptr_array_new(); + + chas->event_hdr_version = g_strdup(_EVENT_VERSION); + + chas->shutdown_hooks = chassis_shutdown_hooks_new(); + + incremental_guid_init(&(chas->guid_state)); + + chas->startup_time = time(0); + + return chas; +} + +static void g_queue_free_cache_index(gpointer q) { + GQueue *queue = q; + + query_cache_index_item *index; + + while ((index = g_queue_pop_head(queue))) { + g_free(index->key); + g_free(index); + } + + g_queue_free(queue); +} + +/** + * free the global scope + * + * closes all open connections, cleans up all plugins + * + * @param chas global context + */ +void chassis_free(chassis *chas) { + guint i; +#ifdef HAVE_EVENT_BASE_FREE + const char *version; +#endif + + if (!chas) return; + + /* init the shutdown, without freeing share structures */ + if (chas->priv_shutdown) chas->priv_shutdown(chas, chas->priv); + + /* call the destructor for all plugins */ + for (i = 0; i < chas->modules->len; i++) { + chassis_plugin *p = g_ptr_array_index(chas->modules, i); + + g_assert(p->destroy); + p->destroy(chas, p->config); + } + + /* cleanup the global 3rd party stuff before we unload the modules */ + chassis_shutdown_hooks_call(chas->shutdown_hooks); + + for (i = 0; i < chas->modules->len; i++) { + chassis_plugin *p = chas->modules->pdata[i]; + + chassis_plugin_free(p); + } + + g_ptr_array_free(chas->modules, TRUE); + + + if (chas->base_dir) g_free(chas->base_dir); + if (chas->conf_dir) g_free(chas->conf_dir); + if (chas->plugin_dir) g_free(chas->plugin_dir); + if (chas->user) g_free(chas->user); + if (chas->default_db) g_free(chas->default_db); + if (chas->default_username) g_free(chas->default_username); + if (chas->default_hashed_pwd) g_free(chas->default_hashed_pwd); + if (chas->query_cache_table) g_hash_table_destroy(chas->query_cache_table); + if (chas->cache_index) g_queue_free_cache_index(chas->cache_index); + + g_free(chas->event_hdr_version); + + chassis_shutdown_hooks_free(chas->shutdown_hooks); + + if (chas->priv_finally_free_shared) chas->priv_finally_free_shared(chas, chas->priv); + + /* free the pointers _AFTER_ the modules are shutdown */ + if (chas->priv_free) chas->priv_free(chas, chas->priv); +#ifdef HAVE_EVENT_BASE_FREE + /* only recent versions have this call */ + + version = event_get_version(); + + /* libevent < 1.3e doesn't cleanup its own fds from the event-queue in signal_init() + * calling event_base_free() would cause a assert() on shutdown + */ + if (version && (strcmp(version, "1.3e") >= 0)) { + if (chas->event_base) { + event_base_free(chas->event_base); + } + } +#endif + if (chas->config_manager) chassis_config_free(chas->config_manager); + + g_free(chas); +} + +void chassis_set_shutdown_location(const gchar *location) { + if (signal_shutdown == 0) { + g_message("Initiating shutdown, requested from %s", + (location != NULL ? location : "signal handler")); + } + signal_shutdown = 1; +} + +gboolean chassis_is_shutdown() { + return signal_shutdown == 1; +} + +static void +sigterm_handler(int G_GNUC_UNUSED fd, short G_GNUC_UNUSED event_type, + void G_GNUC_UNUSED *_data) +{ + chassis_set_shutdown_location(NULL); +} + +static void +sighup_handler(int G_GNUC_UNUSED fd, short G_GNUC_UNUSED event_type, + void *_data) +{ + chassis *chas = _data; + + /* this should go into the old logfile */ + g_message("received a SIGHUP, closing log file"); + + chassis_log_set_logrotate(chas->log); + + /* ... and this into the new one */ + g_message("re-opened log file after SIGHUP"); +} + + +/** + * forward libevent messages to the glib error log + */ +static void event_log_use_glib(int libevent_log_level, const char *msg) { + /* map libevent to glib log-levels */ + + GLogLevelFlags glib_log_level = G_LOG_LEVEL_DEBUG; + + if (libevent_log_level == _EVENT_LOG_DEBUG) glib_log_level = G_LOG_LEVEL_DEBUG; + else if (libevent_log_level == _EVENT_LOG_MSG) glib_log_level = G_LOG_LEVEL_MESSAGE; + else if (libevent_log_level == _EVENT_LOG_WARN) glib_log_level = G_LOG_LEVEL_WARNING; + else if (libevent_log_level == _EVENT_LOG_ERR) glib_log_level = G_LOG_LEVEL_CRITICAL; + + g_log(G_LOG_DOMAIN, glib_log_level, "(libevent) %s", msg); +} + +int chassis_mainloop(void *_chas) { + chassis *chas = _chas; + guint i; + struct event ev_sigterm, ev_sigint; +#ifdef SIGHUP + struct event ev_sighup; +#endif + /* redirect logging from libevent to glib */ + event_set_log_callback(event_log_use_glib); + + /* add a event-handler for the "main" events */ + chassis_event_loop_t *mainloop = chassis_event_loop_new(); + chas->event_base = mainloop; + g_assert(chas->event_base); + + /* setup all plugins */ + for (i = 0; i < chas->modules->len; i++) { + chassis_plugin *p = chas->modules->pdata[i]; + + g_assert(p->apply_config); + if (0 != p->apply_config(chas, p->config)) { + g_critical("%s: applying config of plugin %s failed", + G_STRLOC, p->name); + return -1; + } + } + + chas->dist_tran_id = g_random_int_range(0, 100000000); + int srv_id = g_random_int_range(0, 10000); + if (chas->proxy_address) { + snprintf(chas->dist_tran_prefix, MAX_DIST_TRAN_PREFIX, + "clt-%s-%d", chas->proxy_address, srv_id); + } else { + snprintf(chas->dist_tran_prefix, MAX_DIST_TRAN_PREFIX, + "clt-%d", srv_id); + } + g_message("Initial dist_tran_id:%llu", chas->dist_tran_id); + g_message("dist_tran_prefix:%s", chas->dist_tran_prefix); + + /* + * drop root privileges if requested + */ + if (chas->user) { + struct passwd *user_info; + uid_t user_id = geteuid(); + + /* Don't bother if we aren't superuser */ + if (user_id) { + g_critical("can only use the --user switch if running as root"); + return -1; + } + + if (NULL == (user_info = getpwnam(chas->user))) { + g_critical("unknown user: %s", chas->user); + return -1; + } + + if (chas->log->log_filename) { + /* chown logfile */ + if (-1 == chown(chas->log->log_filename, user_info->pw_uid, + user_info->pw_gid)) + { + g_critical("%s: chown(%s) failed: %s", + G_STRLOC, + chas->log->log_filename, + g_strerror(errno)); + + return -1; + } + } + + if (setgid(user_info->pw_gid) == 0) { + if (setuid(user_info->pw_uid) == 0) { + } + } + g_debug("now running as user: %s (%d/%d)", + chas->user, + user_info->pw_uid, + user_info->pw_gid); + } + + signal_set(&ev_sigterm, SIGTERM, sigterm_handler, NULL); + event_base_set(chas->event_base, &ev_sigterm); + signal_add(&ev_sigterm, NULL); + + signal_set(&ev_sigint, SIGINT, sigterm_handler, NULL); + event_base_set(chas->event_base, &ev_sigint); + signal_add(&ev_sigint, NULL); + +#ifdef SIGHUP + signal_set(&ev_sighup, SIGHUP, sighup_handler, chas); + event_base_set(chas->event_base, &ev_sighup); + if (signal_add(&ev_sighup, NULL)) { + g_critical("%s: signal_add(SIGHUP) failed", G_STRLOC); + } +#endif + +#if !GLIB_CHECK_VERSION(2, 32, 0) + /* GLIB below 2.32 must call thread_init if multi threads */ + if (!chas->disable_threads) { + g_thread_init(NULL); + } else { + g_debug("Disable threads creation."); + } +#endif + + /** + * block until we are asked to shutdown + */ + chassis_event_loop(mainloop); + + signal_del(&ev_sigterm); + signal_del(&ev_sigint); +#ifdef SIGHUP + signal_del(&ev_sighup); +#endif + return 0; +} + +uint64_t incremental_guid_get_next(struct incremental_guid_state_t *s) +{ + uint64_t uniq_id = 0; + uint64_t cur_time = time(0); + static uint64_t SEQ_MASK = (-1L ^ (-1L << 16L)); + + uniq_id = cur_time << 32; + uniq_id |= (s->worker_id & 0xff) << 24; + + if (cur_time == s->last_sec) { + s->seq_id = (s->seq_id + 1) & SEQ_MASK; + if (s->seq_id == 0) { + s->rand_id = (s->rand_id + 1) & 0x3ff; + g_message("%s:rand id changed:%llu", G_STRLOC, + (unsigned long long) s->rand_id); + } + } else { + s->seq_id = 0; + s->rand_id = s->init_rand_id; + } + + s->last_sec = cur_time; + uniq_id |= s->rand_id << 16; + uniq_id |= s->seq_id; + + return uniq_id; +} + +void incremental_guid_init(struct incremental_guid_state_t *s) +{ + struct timeval tp; + gettimeofday(&tp, NULL); + unsigned int seed = tp.tv_usec; + + if (s->worker_id == 0) { + s->worker_id = (int) ((rand_r(&seed) / (RAND_MAX + 1.0)) * 64); + } + + s->rand_id = (int) ((rand_r(&seed) / (RAND_MAX + 1.0)) * 1024); + s->init_rand_id = s->rand_id; + s->last_sec = time(0); + s->seq_id = 0; +} diff --git a/src/chassis-mainloop.h b/src/chassis-mainloop.h new file mode 100644 index 0000000..9e3616a --- /dev/null +++ b/src/chassis-mainloop.h @@ -0,0 +1,186 @@ +#ifndef _CHASSIS_MAINLOOP_H_ +#define _CHASSIS_MAINLOOP_H_ + +#include /* GPtrArray */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_SYS_TIME_H +#include /* event.h needs struct tm */ +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#include +#include /* struct event_base */ + +#include "chassis-exports.h" +#include "chassis-log.h" +#include "chassis-shutdown-hooks.h" +#include "cetus-util.h" +#include "chassis-config.h" + +/** @defgroup chassis Chassis + * + * the chassis contains the set of functions that are used by all programs + * + * */ +/*@{*/ +typedef struct chassis_private chassis_private; +typedef struct chassis chassis; + +#define MAX_SERVER_NUM 64 +#define MAX_QUERY_TIME 1000 +#define MAX_WAIT_TIME 1024 +#define MAX_DIST_TRAN_PREFIX 32 + +#define MAX_ALLOWED_PACKET_CEIL (1 * GB) +#define MAX_ALLOWED_PACKET_DEFAULT (32 * MB) +#define MAX_ALLOWED_PACKET_FLOOR (1 * KB) + +typedef struct cached_sql_info_t { + time_t last_visit_time; + int visited_cnt; + int parsed_txLevel; + int parsed_rv; + int parsed_join_op_unsupported; + GPtrArray *groups; +} cached_sql_info_t; + +typedef struct rw_op_t { + uint64_t ro; + uint64_t rw; +} rw_op_t; + +typedef struct query_stats_t { + rw_op_t client_query; + rw_op_t proxyed_query; + uint64_t query_time_table[MAX_QUERY_TIME]; + uint64_t query_wait_table[MAX_WAIT_TIME]; + rw_op_t server_query_details[MAX_SERVER_NUM]; + uint64_t com_select; + uint64_t com_insert; + uint64_t com_update; + uint64_t com_delete; + uint64_t com_select_shard; + uint64_t com_insert_shard; + uint64_t com_update_shard; + uint64_t com_delete_shard; + uint64_t com_select_global; + uint64_t com_select_bad_key; + uint64_t xa_count; +} query_stats_t; + +/* For generating unique global ids for MySQL */ +struct incremental_guid_state_t { + unsigned int last_sec; + int worker_id; + int rand_id; + int init_rand_id; + int seq_id; +}; + +void incremental_guid_init(struct incremental_guid_state_t *s); +uint64_t incremental_guid_get_next(struct incremental_guid_state_t *s); + +struct chassis { + struct event_base *event_base; + gchar *event_hdr_version; + + /**< array(chassis_plugin) */ + GPtrArray *modules; + + /**< base directory for all relative paths referenced */ + gchar *base_dir; + /**< plugin dir for save settings */ + gchar *plugin_dir; + gchar *conf_dir; + /**< user to run as */ + gchar *user; + + char *proxy_address; + char *default_db; + char *default_username; + char *default_charset; + char *default_hashed_pwd; + + unsigned int auto_key; + unsigned int sess_key; + unsigned int maintain_close_mode; + unsigned int sharding_mode; + unsigned int config_remote; + unsigned int disable_threads; + unsigned int is_tcp_stream_enabled; + unsigned int query_cache_enabled; + unsigned int is_back_compressed; + unsigned int compress_support; + unsigned int client_found_rows; + unsigned int master_preferred; + unsigned int is_reduce_conns; + unsigned int xa_log_detailed; + unsigned int is_reset_conn_enabled; + unsigned int sharding_reload; + unsigned int check_slave_delay; + int complement_conn_cnt; + int default_query_cache_timeout; + double slave_delay_down_threshold_sec; + double slave_delay_recover_threshold_sec; + unsigned int long_query_time; + unsigned int min_req_time_for_cache; + int cetus_max_allowed_packet; + int disable_dns_cache; + + int max_resp_len; + int merged_output_size; + int max_header_size; + int compressed_merged_output_size; + + /* Conn-pool initialize settings */ + int max_idle_connections; + int mid_idle_connections; + + unsigned long long dist_tran_id; + + char dist_tran_prefix[MAX_DIST_TRAN_PREFIX]; + + chassis_private *priv; + void (*priv_shutdown)(chassis *chas, chassis_private *priv); + void (*priv_finally_free_shared)(chassis *chas, chassis_private *priv); + void (*priv_free)(chassis *chas, chassis_private *priv); + + chassis_log *log; + + chassis_shutdown_hooks_t *shutdown_hooks; + + query_stats_t query_stats; + + struct incremental_guid_state_t guid_state; + time_t startup_time; + struct chassis_options_t *options; + chassis_config_t *config_manager; + GHashTable *query_cache_table; + GQueue *cache_index; + unsigned long long last_cache_purge_time; + gboolean allow_new_conns; +}; + +CHASSIS_API chassis *chassis_new(void); +CHASSIS_API void chassis_free(chassis *chas); +CHASSIS_API int chassis_check_version(const char *lib_version, const char *hdr_version); + +/** + * the mainloop for all chassis apps + * + */ +CHASSIS_API int chassis_mainloop(void *user_data); + +CHASSIS_API void chassis_set_shutdown_location(const gchar *location); +CHASSIS_API gboolean chassis_is_shutdown(void); + +#define chassis_set_shutdown() chassis_set_shutdown_location(G_STRLOC) + +/*@}*/ + +#endif diff --git a/src/chassis-options.c b/src/chassis-options.c new file mode 100644 index 0000000..739e617 --- /dev/null +++ b/src/chassis-options.c @@ -0,0 +1,874 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#ifdef HAVE_UNISTD_H +#include +#endif +#include + +#include + +#include "chassis-options.h" + +/** + * create a command-line option + */ +chassis_option_t *chassis_option_new() { + chassis_option_t *opt; + + opt = g_slice_new0(chassis_option_t); + + return opt; +} + +/** + * free the option + */ +void chassis_option_free(chassis_option_t *opt) { + if (!opt) return; + + g_slice_free(chassis_option_t, opt); +} + +/** + * add a option + * + * GOptionEntry + */ +int chassis_option_set(chassis_option_t *opt, + const char *long_name, + gchar short_name, + gint flags, + enum option_type arg, + gpointer arg_data, + const char *description, + const char *arg_desc) { + opt->long_name = long_name; + opt->short_name = short_name; + opt->flags = flags; + opt->arg = arg; + opt->arg_data = arg_data; + opt->description = description; + opt->arg_description = arg_desc; + return 0; +} + +char *chassis_option_get_value_str(chassis_option_t *opt) +{ + gchar *value = NULL; + switch (opt->arg) { + case OPTION_ARG_NONE: + value = g_strdup((*(gint *)opt->arg_data) ? "true" : "false"); + break; + case OPTION_ARG_INT: + value = g_strdup_printf("%u", *(gint *)(opt->arg_data)); + break; + case OPTION_ARG_INT64: + value = g_strdup_printf("%lu", *(gint64 *)(opt->arg_data)); + break; + case OPTION_ARG_DOUBLE: + value = g_strdup_printf("%f", *(double *)(opt->arg_data)); + break; + case OPTION_ARG_STRING: + value = g_strdup(*(char **)opt->arg_data); + break; + default: + value = g_strdup("error value"); + } + return value; +} + +gboolean chassis_option_set_value(chassis_option_t *opt, const char *value) +{ + switch (opt->arg) { + case OPTION_ARG_NONE: + if (strcasecmp(value, "true") == 0 || strcasecmp(value, "1") == 0) { + *(gint *)opt->arg_data = 1; + return TRUE; + } else if (strcasecmp(value, "false") == 0 || strcasecmp(value, "0") == 0) { + *(gint *)opt->arg_data = 0; + return TRUE; + } else { + return FALSE; + } + case OPTION_ARG_INT: { + char *endptr = NULL; + int num = strtol(value, &endptr, 0); + if (endptr[0] == '\0') { + *(gint *)opt->arg_data = num; /* TODO: apply lower/upper bounds */ + return TRUE; + } else { + return FALSE; + } + } + case OPTION_ARG_INT64: { + char *endptr = NULL; + gint64 num = strtoll(value, &endptr, 0); + if (endptr[0] == '\0') { + *(gint64 *)opt->arg_data = num; /* TODO: apply lower/upper bounds */ + return TRUE; + } else { + return FALSE; + } + } + default: + return FALSE; + } +} + +/** + * create a command-line option + */ +chassis_options_t *chassis_options_new() { + chassis_options_t *opt; + + opt = g_slice_new0(chassis_options_t); + + return opt; +} + + +/** + * add a option + * + * GOptionEntry + */ +int chassis_options_add_option(chassis_options_t *opts, + chassis_option_t *opt) { + + opts->options = g_list_append(opts->options, opt); + + return 0; +} + +void chassis_options_add_options(chassis_options_t *opts, GList *list) +{ + opts->options = g_list_concat(opts->options, list); +} + +int chassis_options_add(chassis_options_t *opts, + const char *long_name, + gchar short_name, + int flags, + enum option_type arg, + gpointer arg_data, + const char *description, + const char *arg_desc) +{ + chassis_option_t *opt = chassis_option_new(); + if (0 != chassis_option_set(opt, + long_name, + short_name, + flags, + arg, + arg_data, + description, + arg_desc) || + 0 != chassis_options_add_option(opts, opt)) { + chassis_option_free(opt); + return -1; + } else { + return 0; + } +} + +chassis_option_t *chassis_options_get(GList *opts, const char *long_name) +{ + GList *l = opts; + for (l = opts; l; l = l->next) { + chassis_option_t *opt = l->data; + if (strcmp(opt->long_name, long_name) == 0) { + return opt; + } + } + return NULL; +} + + +#define NO_ARG(entry) ((entry)->arg == OPTION_ARG_NONE) + +#define OPTIONAL_ARG(entry) FALSE + +typedef struct +{ + enum option_type arg_type; + gpointer arg_data; + union + { + gboolean bool; + gint integer; + gchar *str; + gchar **array; + gdouble dbl; + gint64 int64; + } prev; + union + { + gchar *str; + struct + { + gint len; + gchar **data; + } array; + } allocated; +} Change; + +typedef struct +{ + gchar **ptr; + gchar *value; +} PendingNull; + +static gboolean context_has_h_entry(chassis_options_t *context) +{ + GList *l; + for (l = context->options; l; l = l->next) { + chassis_option_t *entry = l->data; + if (entry->short_name == 'h') + return TRUE; + } + return FALSE; +} + +static void print_help(chassis_options_t *context) +{ + int max_length = 50; + g_print("%s\n %s", "Usage:", g_get_prgname()); + if (context->options) + g_print(" %s", "[OPTION...]"); + + g_print("\n\nHelp Options:\n"); + char token = context_has_h_entry (context) ? '?' : 'h'; + g_print(" -%c, --%-*s %s\n", token, max_length-12, "help", "Show help options"); + + g_print("\n\nApplication Options:\n"); + GList *l; + for (l = context->options; l; l = l->next) { + chassis_option_t *entry = l->data; + int len = 0; + if (entry->short_name) { + g_print(" -%c, --%s", entry->short_name, entry->long_name); + len = 8 + strlen(entry->long_name); + } else { + g_print(" --%s", entry->long_name); + len = 4 + strlen(entry->long_name); + } + if (entry->arg_description) { + g_print("=%s", entry->arg_description); + len += 1+strlen(entry->arg_description); + } + g_print("%*s %s\n", max_length - len, "", + entry->description ? entry->description : ""); + } + exit(0); +} + +static gboolean +parse_int(const gchar *arg_name, + const gchar *arg, + gint *result, + GError **error) +{ + gchar *end; + glong tmp; + + errno = 0; + tmp = strtol(arg, &end, 0); + + if (*arg == '\0' || *end != '\0') { + g_set_error(error, G_OPTION_ERROR, G_OPTION_ERROR_BAD_VALUE, + "Cannot parse integer value '%s' for %s", + arg, arg_name); + return FALSE; + } + + *result = tmp; + if (*result != tmp || errno == ERANGE) { + g_set_error(error, G_OPTION_ERROR, G_OPTION_ERROR_BAD_VALUE, + "Integer value '%s' for %s out of range", + arg, arg_name); + return FALSE; + } + return TRUE; +} + +static gboolean +parse_double(const gchar *arg_name, + const gchar *arg, + gdouble *result, + GError **error) +{ + gchar *end; + gdouble tmp; + + errno = 0; + tmp = g_strtod(arg, &end); + + if (*arg == '\0' || *end != '\0') { + g_set_error(error, G_OPTION_ERROR, G_OPTION_ERROR_BAD_VALUE, + "Cannot parse double value '%s' for %s", + arg, arg_name); + return FALSE; + } + if (errno == ERANGE) { + g_set_error(error, G_OPTION_ERROR, G_OPTION_ERROR_BAD_VALUE, + "Double value '%s' for %s out of range", + arg, arg_name); + return FALSE; + } + *result = tmp; + return TRUE; +} + +static gboolean +parse_int64(const gchar *arg_name, + const gchar *arg, + gint64 *result, + GError **error) +{ + gchar *end; + gint64 tmp; + + errno = 0; + tmp = g_ascii_strtoll(arg, &end, 0); + + if (*arg == '\0' || *end != '\0') { + g_set_error(error, G_OPTION_ERROR, G_OPTION_ERROR_BAD_VALUE, + "Cannot parse integer value '%s' for %s", + arg, arg_name); + return FALSE; + } + if (errno == ERANGE) { + g_set_error(error, G_OPTION_ERROR, G_OPTION_ERROR_BAD_VALUE, + "Integer value '%s' for %s out of range", + arg, arg_name); + return FALSE; + } + *result = tmp; + return TRUE; +} + +static Change * +get_change(chassis_options_t *context, + enum option_type arg_type, + gpointer arg_data) +{ + GList *list; + Change *change = NULL; + + for(list = context->changes; list != NULL; list = list->next) { + change = list->data; + + if (change->arg_data == arg_data) + goto found; + } + + change = g_new0(Change, 1); + change->arg_type = arg_type; + change->arg_data = arg_data; + + context->changes = g_list_prepend(context->changes, change); + +found: + return change; +} + +static void +add_pending_null(chassis_options_t *context, + gchar **ptr, + gchar *value) +{ + PendingNull *n; + + n = g_new0(PendingNull, 1); + n->ptr = ptr; + n->value = value; + + context->pending_nulls = g_list_prepend(context->pending_nulls, n); +} + +static gboolean +parse_arg(chassis_options_t *context, + chassis_option_t *entry, + const gchar *value, + const gchar *option_name, + GError **error) + +{ + Change *change; + + g_assert(value || OPTIONAL_ARG(entry) || NO_ARG(entry)); + + switch (entry->arg) { + case OPTION_ARG_NONE: { + (void) get_change(context, OPTION_ARG_NONE, entry->arg_data); + + *(gboolean *)entry->arg_data = !(entry->flags & OPTION_FLAG_REVERSE); + break; + } + case OPTION_ARG_STRING: { + gchar *data = g_locale_to_utf8(value, -1, NULL, NULL, error); + if (!data) + return FALSE; + + change = get_change(context, OPTION_ARG_STRING, entry->arg_data); + g_free(change->allocated.str); + + change->prev.str = *(gchar **)entry->arg_data; + change->allocated.str = data; + + *(gchar **)entry->arg_data = data; + break; + } + case OPTION_ARG_STRING_ARRAY: { + gchar *data = g_locale_to_utf8(value, -1, NULL, NULL, error); + if (!data) + return FALSE; + + change = get_change(context, OPTION_ARG_STRING_ARRAY, entry->arg_data); + + if (change->allocated.array.len == 0) { + change->prev.array = *(gchar ***)entry->arg_data; + change->allocated.array.data = g_new(gchar *, 2); + } else { + change->allocated.array.data = + g_renew(gchar *, change->allocated.array.data, + change->allocated.array.len + 2); + } + change->allocated.array.data[change->allocated.array.len] = data; + change->allocated.array.data[change->allocated.array.len + 1] = NULL; + + change->allocated.array.len ++; + + *(gchar ***)entry->arg_data = change->allocated.array.data; + + break; + } + case OPTION_ARG_INT: { + gint data; + if (!parse_int(option_name, value, &data, error)) + return FALSE; + + change = get_change(context, OPTION_ARG_INT, entry->arg_data); + change->prev.integer = *(gint *)entry->arg_data; + *(gint *)entry->arg_data = data; + break; + } + case OPTION_ARG_DOUBLE: { + gdouble data; + if (!parse_double(option_name, value, &data, error)) { + return FALSE; + } + change = get_change(context, OPTION_ARG_DOUBLE, entry->arg_data); + change->prev.dbl = *(gdouble *)entry->arg_data; + *(gdouble *)entry->arg_data = data; + break; + } + case OPTION_ARG_INT64: { + gint64 data; + if (!parse_int64(option_name, value, &data, error)) { + return FALSE; + } + change = get_change(context, OPTION_ARG_INT64, entry->arg_data); + change->prev.int64 = *(gint64 *)entry->arg_data; + *(gint64 *)entry->arg_data = data; + break; + } + default: + g_assert_not_reached(); + } + entry->flags |= OPTION_FLAG_CMDLINE; + return TRUE; +} + +static gboolean +parse_short_option(chassis_options_t *context, + gint idx, + gint *new_idx, + gchar arg, + gint *argc, + gchar ***argv, + GError **error, + gboolean *parsed) +{ + GList *l; + for (l = context->options; l; l = l->next) { + chassis_option_t *entry = l->data; + + if (arg == entry->short_name) { + gchar *option_name = g_strdup_printf ("-%c", entry->short_name); + gchar *value = NULL; + if (NO_ARG(entry)) { + value = NULL; + } else { + if (*new_idx > idx) { + g_set_error(error, G_OPTION_ERROR, G_OPTION_ERROR_FAILED, + "Error parsing option %s", option_name); + g_free(option_name); + return FALSE; + } + + if (idx < *argc - 1) { + if (!OPTIONAL_ARG(entry)) { + value = (*argv)[idx + 1]; + add_pending_null(context, &((*argv)[idx + 1]), NULL); + *new_idx = idx + 1; + } else { + if ((*argv)[idx + 1][0] == '-') { + value = NULL; + } else { + value = (*argv)[idx + 1]; + add_pending_null(context, &((*argv)[idx + 1]), NULL); + *new_idx = idx + 1; + } + } + } else if (idx >= *argc - 1 && OPTIONAL_ARG(entry)) { + value = NULL; + } else { + g_set_error(error, G_OPTION_ERROR, G_OPTION_ERROR_BAD_VALUE, + "Missing argument for %s", option_name); + g_free(option_name); + return FALSE; + } + } + + if (!parse_arg(context, entry, value, option_name, error)) { + g_free(option_name); + return FALSE; + } + g_free(option_name); + *parsed = TRUE; + } + } + return TRUE; +} + +static gboolean +parse_long_option(chassis_options_t *context, + gint *idx, + gchar *arg, + gboolean aliased, + gint *argc, + gchar ***argv, + GError **error, + gboolean *parsed) +{ + GList *l; + for (l = context->options; l; l = l->next) { + chassis_option_t *entry = l->data; + if (*idx >= *argc) + return TRUE; + + if (NO_ARG(entry) && strcmp(arg, entry->long_name) == 0) { + gchar *option_name = g_strconcat("--", entry->long_name, NULL); + gboolean retval = parse_arg(context, entry, NULL, option_name, error); + g_free(option_name); + + add_pending_null(context, &((*argv)[*idx]), NULL); + *parsed = TRUE; + + return retval; + } else { + gint len = strlen(entry->long_name); + + if (strncmp(arg, entry->long_name, len) == 0 && + (arg[len] == '=' || arg[len] == 0)) + { + gchar *value = NULL; + gchar *option_name; + + add_pending_null(context, &((*argv)[*idx]), NULL); + option_name = g_strconcat("--", entry->long_name, NULL); + + if (arg[len] == '=') { + value = arg + len + 1; + } else if (*idx < *argc - 1) { + if (!OPTIONAL_ARG(entry)) { + value = (*argv)[*idx + 1]; + add_pending_null(context, &((*argv)[*idx + 1]), NULL); + (*idx)++; + } else { + if ((*argv)[*idx + 1][0] == '-') { + gboolean retval = parse_arg(context, entry, + NULL, option_name, error); + *parsed = TRUE; + g_free(option_name); + return retval; + } else { + value = (*argv)[*idx + 1]; + add_pending_null(context, &((*argv)[*idx + 1]), NULL); + (*idx)++; + } + } + } else if (*idx >= *argc - 1 && OPTIONAL_ARG(entry)) { + gboolean retval = parse_arg(context, entry, NULL, option_name, error); + *parsed = TRUE; + g_free(option_name); + return retval; + } else { + g_set_error(error, G_OPTION_ERROR, G_OPTION_ERROR_BAD_VALUE, + "Missing argument for %s", option_name); + g_free(option_name); + return FALSE; + } + if (!parse_arg(context, entry, value, option_name, error)) { + g_free(option_name); + return FALSE; + } + g_free(option_name); + *parsed = TRUE; + } + } + } + return TRUE; +} + +static gboolean +parse_remaining_arg(chassis_options_t *context, + gint *idx, + gint *argc, + gchar ***argv, + GError **error, + gboolean *parsed) +{ + GList *l; + for (l = context->options; l; l = l->next) { + chassis_option_t *entry = l->data; + if (*idx >= *argc) + return TRUE; + + if (entry->long_name[0]) + continue; + + g_return_val_if_fail(entry->arg == OPTION_ARG_STRING_ARRAY, FALSE); + + add_pending_null(context, &((*argv)[*idx]), NULL); + + if (!parse_arg(context, entry, (*argv)[*idx], "", error)) + return FALSE; + + *parsed = TRUE; + return TRUE; + } + return TRUE; +} + +static void free_changes_list(chassis_options_t *context, gboolean revert) +{ + GList *list; + for (list = context->changes; list != NULL; list = list->next) { + Change *change = list->data; + + if (revert) { + switch (change->arg_type) { + case OPTION_ARG_NONE: + *(gboolean *)change->arg_data = change->prev.bool; + break; + case OPTION_ARG_INT: + *(gint *)change->arg_data = change->prev.integer; + break; + case OPTION_ARG_STRING: + g_free(change->allocated.str); + *(gchar **)change->arg_data = change->prev.str; + break; + case OPTION_ARG_STRING_ARRAY: + g_strfreev (change->allocated.array.data); + *(gchar ***)change->arg_data = change->prev.array; + break; + case OPTION_ARG_DOUBLE: + *(gdouble *)change->arg_data = change->prev.dbl; + break; + case OPTION_ARG_INT64: + *(gint64 *)change->arg_data = change->prev.int64; + break; + default: + g_assert_not_reached(); + } + } + g_free(change); + } + g_list_free(context->changes); + context->changes = NULL; +} + +static void free_pending_nulls(chassis_options_t *context, gboolean perform_nulls) +{ + GList *list; + for (list = context->pending_nulls; list != NULL; list = list->next) { + PendingNull *n = list->data; + + if (perform_nulls) { + if (n->value) { + /* Copy back the short options */ + *(n->ptr)[0] = '-'; + strcpy(*n->ptr + 1, n->value); + } else { + *n->ptr = NULL; + } + } + g_free(n->value); + g_free(n); + } + g_list_free(context->pending_nulls); + context->pending_nulls = NULL; +} + +/** + * free the option context + */ +void chassis_options_free(chassis_options_t *context) +{ + if (!context) return; + + g_list_free_full(context->options, (GDestroyNotify)chassis_option_free); + + free_changes_list(context, FALSE); + free_pending_nulls(context, FALSE); + + g_slice_free(chassis_options_t, context); +} + + +gboolean chassis_options_parse_cmdline(chassis_options_t *context, + int *argc, char ***argv, GError **error) +{ + if (!argc || !argv) + return FALSE; + /* Set program name */ + if (!g_get_prgname()) { + if (*argc) { + gchar *prgname = g_path_get_basename((*argv)[0]); + g_set_prgname(prgname?prgname:""); + g_free(prgname); + } + } + + gboolean stop_parsing = FALSE; + gboolean has_unknown = FALSE; + gint separator_pos = 0; + int i, j, k; + for (i = 1; i < *argc; i++) { + gchar *arg; + gboolean parsed = FALSE; + + if ((*argv)[i][0] == '-' && (*argv)[i][1] != '\0' && !stop_parsing) { + if ((*argv)[i][1] == '-') { + /* -- option */ + + arg = (*argv)[i] + 2; + + /* '--' terminates list of arguments */ + if (*arg == 0) { + separator_pos = i; + stop_parsing = TRUE; + continue; + } + + /* Handle help options */ + if (context->help_enabled && strcmp(arg, "help") == 0) + print_help(context); + + if (!parse_long_option(context, &i, arg, + FALSE, argc, argv, error, &parsed)) + goto fail; + + if (parsed) + continue; + + if (context->ignore_unknown) + continue; + } else { /* short option */ + gint new_i = i, arg_length; + gboolean *nulled_out = NULL; + gboolean has_h_entry = context_has_h_entry(context); + arg = (*argv)[i] + 1; + arg_length = strlen(arg); + nulled_out = g_newa (gboolean, arg_length); + memset(nulled_out, 0, arg_length * sizeof (gboolean)); + + for (j = 0; j < arg_length; j++) { + if (context->help_enabled && + (arg[j] == '?' || (arg[j] == 'h' && !has_h_entry))) + print_help(context); + parsed = FALSE; + if (!parse_short_option(context, i, &new_i, arg[j], + argc, argv, error, &parsed)) + goto fail; + + if (context->ignore_unknown && parsed) + nulled_out[j] = TRUE; + else if (context->ignore_unknown) + continue; + else if (!parsed) + break; + /* !context->ignore_unknown && parsed */ + } + if (context->ignore_unknown) { + gchar *new_arg = NULL; + gint arg_index = 0; + for (j = 0; j < arg_length; j++) { + if (!nulled_out[j]) { + if (!new_arg) + new_arg = g_malloc(arg_length + 1); + new_arg[arg_index++] = arg[j]; + } + } + if (new_arg) + new_arg[arg_index] = '\0'; + add_pending_null(context, &((*argv)[i]), new_arg); + i = new_i; + } else if (parsed) { + add_pending_null(context, &((*argv)[i]), NULL); + i = new_i; + } + } + + if (!parsed) + has_unknown = TRUE; + + if (!parsed && !context->ignore_unknown) { + g_set_error(error, G_OPTION_ERROR, G_OPTION_ERROR_UNKNOWN_OPTION, + "Unknown option %s", (*argv)[i]); + goto fail; + } + } else { + //if (context->strict_posix) + stop_parsing = TRUE; + + /* Collect remaining args */ + if (!parse_remaining_arg(context, &i, argc, argv, error, &parsed)) + goto fail; + if (!parsed && (has_unknown || (*argv)[i][0] == '-')) + separator_pos = 0; + } + } + + if (separator_pos > 0) + add_pending_null(context, &((*argv)[separator_pos]), NULL); + if (argc && argv) { + free_pending_nulls(context, TRUE); + for (i = 1; i < *argc; i++) { + for (k = i; k < *argc; k++) + if ((*argv)[k] != NULL) + break; + + if (k > i) { + k -= i; + for (j = i + k; j < *argc; j++) { + (*argv)[j-k] = (*argv)[j]; + (*argv)[j] = NULL; + } + *argc -= k; + } + } + } + return TRUE; + +fail: + free_changes_list(context, TRUE); + free_pending_nulls(context, FALSE); + return FALSE; +} diff --git a/src/chassis-options.h b/src/chassis-options.h new file mode 100644 index 0000000..0b49bcd --- /dev/null +++ b/src/chassis-options.h @@ -0,0 +1,94 @@ +#ifndef __CHASSIS_OPTIONS_H__ +#define __CHASSIS_OPTIONS_H__ + +#include + +enum option_type { // arg_data type + OPTION_ARG_NONE, // bool * + OPTION_ARG_INT, // int * + OPTION_ARG_INT64, // int64 * + OPTION_ARG_DOUBLE, // double * + OPTION_ARG_STRING, // char ** + OPTION_ARG_STRING_ARRAY, // char ** +}; + +enum option_flags { + OPTION_FLAG_REVERSE = 0x04, + + OPTION_FLAG_CMDLINE = 0x10, + OPTION_FLAG_CONF_FILE = 0x20, + OPTION_FLAG_REMOTE_CONF = 0x40, +}; +/** + * @file + * + * a _new()/_free()-able version of GOptionEntry + */ + +/** + * 'const'-free version of GOptionEntry + */ +typedef struct { + const char *long_name; + const char *description; + const char *arg_description; + gpointer arg_data; + enum option_type arg; + enum option_flags flags; + gchar short_name; +} chassis_option_t; + +/** + * create a chassis_option_t + */ +chassis_option_t *chassis_option_new(void); +void chassis_option_free(chassis_option_t *opt); +int chassis_option_set(chassis_option_t *opt, + const char *long_name, + gchar short_name, + gint flags, + enum option_type arg, + gpointer arg_data, + const char *description, + const char *arg_desc); +/** + * @return newly allocated string, need to be freed + */ +char *chassis_option_get_value_str(chassis_option_t *opt); + +gboolean chassis_option_set_value(chassis_option_t *opt, const char *); + +typedef struct chassis_options_t{ + GList *options; /* List of chassis_option_t */ + + /* We keep a list of change so we can revert them */ + GList *changes; + + /* We also keep track of all argv elements + * that should be NULLed or modified. + */ + GList *pending_nulls; + + gboolean ignore_unknown; + gboolean help_enabled; +} chassis_options_t; + +chassis_options_t *chassis_options_new(void); +void chassis_options_free(chassis_options_t *opts); +int chassis_options_add_option(chassis_options_t *opts, chassis_option_t *opt); +void chassis_options_add_options(chassis_options_t *opts, GList *list); +int chassis_options_add(chassis_options_t *opts, + const char *long_name, + gchar short_name, + int flags, + enum option_type arg, + gpointer arg_data, + const char *description, + const char *arg_desc); + +chassis_option_t *chassis_options_get(GList *opts, const char *long_name); + +gboolean chassis_options_parse_cmdline(chassis_options_t *context, + int *argc, char ***argv, GError **error); + +#endif diff --git a/src/chassis-path.c b/src/chassis-path.c new file mode 100644 index 0000000..f24aa53 --- /dev/null +++ b/src/chassis-path.c @@ -0,0 +1,78 @@ +#include + +#include +#include /* for realpath */ +#include + +#include "glib-ext.h" +#include "chassis-path.h" + +gchar *chassis_get_basedir(const gchar *prgname) { + gchar *absolute_path; + gchar *bin_dir; + gchar r_path[PATH_MAX]; + gchar *base_dir; + + if (g_path_is_absolute(prgname)) { + /* No need to dup, just to get free right */ + absolute_path = g_strdup(prgname); + } else { + /** + * the path wasn't absolute + * + * Either it is + * - in the $PATH + * - relative like ./bin/... or + */ + + absolute_path = g_find_program_in_path(prgname); + if (absolute_path == NULL) { + g_critical("can't find myself (%s) in PATH", prgname); + + return NULL; + } + + if (!g_path_is_absolute(absolute_path)) { + gchar *cwd = g_get_current_dir(); + + g_free(absolute_path); + + absolute_path = g_build_filename(cwd, prgname, NULL); + + g_free(cwd); + } + } + + /* assume that the binary is in ./s?bin/ and + * that the the basedir is right above it + * + * to get this working we need a "clean" basedir, no .../foo/./bin/ + */ + if (NULL == realpath(absolute_path, r_path)) { + g_critical("%s: realpath(%s) failed: %s", + G_STRLOC, + absolute_path, + g_strerror(errno)); + + return NULL; + } + bin_dir = g_path_get_dirname(r_path); + base_dir = g_path_get_dirname(bin_dir); + + /* don't free base_dir, because we need it later */ + g_free(absolute_path); + g_free(bin_dir); + + return base_dir; +} + +gchar *chassis_resolve_path(const char *base_dir, gchar *filename) { + + if (!base_dir || !filename) + return NULL; + + if (g_path_is_absolute(filename)) return filename; + + return g_build_filename(base_dir, G_DIR_SEPARATOR_S, filename, NULL); +} + diff --git a/src/chassis-path.h b/src/chassis-path.h new file mode 100644 index 0000000..b3b5d98 --- /dev/null +++ b/src/chassis-path.h @@ -0,0 +1,12 @@ +#ifndef __CHASSIS_PATH_H__ +#define __CHASSIS_PATH_H__ + +#include + +#include "chassis-exports.h" + +CHASSIS_API gchar *chassis_resolve_path(const char *base_dir, gchar *filename); +CHASSIS_API gchar *chassis_get_basedir(const gchar *prgname); + +#endif + diff --git a/src/chassis-plugin.c b/src/chassis-plugin.c new file mode 100644 index 0000000..5a01827 --- /dev/null +++ b/src/chassis-plugin.c @@ -0,0 +1,108 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +#ifdef HAVE_SIGNAL_H +#include +#endif + +#include "chassis-plugin.h" +#include "chassis-options.h" + + +chassis_plugin *chassis_plugin_new(void) { + chassis_plugin *p; + + p = g_new0(chassis_plugin, 1); + + return p; +} + +void chassis_plugin_free(chassis_plugin *p) { + if (p->option_grp_name) g_free(p->option_grp_name); + if (p->name) g_free(p->name); + if (p->version) g_free(p->version); + + if (p->stats && p->free_stats) p->free_stats(p->stats); + + /* closing the plugin has to be the last call to make sure + * we don't free/call/... stuff that is already unmapped + * from memory + */ +#if (!VALGRIND_SUPPORT) + if (p->module) g_module_close(p->module); +#endif + + g_free(p); +} + +chassis_plugin *chassis_plugin_load(const gchar *name) { + int (*plugin_init)(chassis_plugin *p); + chassis_plugin *p = chassis_plugin_new(); + + p->module = g_module_open(name, G_MODULE_BIND_LOCAL); + + if (!p->module) { + g_critical("loading module '%s' failed: %s", name, g_module_error()); + + chassis_plugin_free(p); + + return NULL; + } + + /* each module has to have a plugin_init function */ + if (!g_module_symbol(p->module, "plugin_init", (gpointer) &plugin_init)) { + g_critical("module '%s' doesn't have a init-function: %s", name, g_module_error()); + chassis_plugin_free(p); + return NULL; + } + + if (0 != plugin_init(p)) { + g_critical("init-function for module '%s' failed", name); + chassis_plugin_free(p); + return NULL; + } + + if (p->magic != CHASSIS_PLUGIN_MAGIC) { + g_critical("'%s' doesn't match the current interface (plugin is %lx, chassis is %lx)", + name, p->magic, CHASSIS_PLUGIN_MAGIC); + chassis_plugin_free(p); + return NULL; + } + + if (p->init) { + p->config = p->init(); + } + + /* if the plugins haven't set p->name provide our own name */ + if (!p->name) p->name = g_strdup(name); + /* set dummy version number if the plugin doesn't provide a real one */ + if (!p->version) { + g_critical("plugin '%s' doesn't set a version num, refusing to load this plugin", + name); + chassis_plugin_free(p); + return NULL; + } + + if (p->new_stats) { + p->stats = p->new_stats(); + } + + return p; +} + +GList *chassis_plugin_get_options(chassis_plugin *p) { + GList *options = NULL; + + if (!p->get_options) return NULL; + + if (NULL == (options = p->get_options(p->config))) { + g_critical("adding config options for module '%s' failed", p->name); + } + return options; +} + diff --git a/src/chassis-plugin.h b/src/chassis-plugin.h new file mode 100644 index 0000000..76c9579 --- /dev/null +++ b/src/chassis-plugin.h @@ -0,0 +1,102 @@ +#ifndef _CHASSIS_PLUGIN_H_ +#define _CHASSIS_PLUGIN_H_ + +#include +#include + +#include "chassis-mainloop.h" +#include "chassis-exports.h" + +/* current magic is 0.8.0-1 */ +#define CHASSIS_PLUGIN_MAGIC 0x00080001L + +/** + * The private stats structure of a plugin. This is opaque to the rest of the code, + * we can only get a copy of it in a hash. + * @see chassis_plugin_stats.get_stats() + */ +typedef struct chassis_plugin_stats chassis_plugin_stats_t; +typedef struct chassis_plugin_config chassis_plugin_config; + +typedef struct chassis_plugin { + /**< a magic token to verify that the plugin API matches */ + long magic; + + /**< name of the option group (used in --help- */ + gchar *option_grp_name; + + /**< user visible name of this plugin */ + gchar *name; + + /**< the plugin's version number */ + gchar *version; + + /**< the plugin handle when loaded */ + GModule *module; + + /**< contains the plugin-specific statistics */ + chassis_plugin_stats_t *stats; + + /**< handler function to initialize the plugin-specific stats */ + chassis_plugin_stats_t *(*new_stats)(void); + + /**< handler function to dealloc the plugin-specific stats */ + void (*free_stats)(chassis_plugin_stats_t *user_data); + + /**< handler function to retrieve the plugin-specific stats */ + GHashTable *(*get_stats)(chassis_plugin_stats_t *user_data); + + /**< contains the plugin-specific config data */ + chassis_plugin_config *config; + + /**< handler function to allocate/initialize a chassis_plugin_config struct */ + chassis_plugin_config *(*init)(void); + + /**< handler function used to deallocate the chassis_plugin_config */ + void (*destroy)(chassis *chas, chassis_plugin_config *user_data); + + /**< handler function to obtain the command line argument information */ + GList *(*get_options)(chassis_plugin_config *user_data); + + /**< handler function to set the argument values in the plugin's config */ + int (*apply_config)(chassis *chas, chassis_plugin_config *user_data); + + /**< handler function to retrieve the plugin's global state */ + void *(*get_global_state)(chassis_plugin_config *user_data, const char *member); + + /**< handler function used to get allow ip list */ + GList *(*allow_ip_get)(chassis_plugin_config *user_data); + + /**< handler function used to add IP addr to allow_ip_table */ + gboolean (*allow_ip_add)(chassis_plugin_config *user_data, char *addr); + + /**< handler function used to delete IP addr to allow_ip_table */ + gboolean (*allow_ip_del)(chassis_plugin_config *user_data, char *addr); + + /**< handler function used to get deny ip list */ + GList *(*deny_ip_get)(chassis_plugin_config *user_data); + + /**< handler function used to add IP addr to deny_ip_table */ + gboolean (*deny_ip_add)(chassis_plugin_config *user_data, char *addr); + + /**< handler function used to delete IP addr to deny_ip_table */ + gboolean (*deny_ip_del)(chassis_plugin_config *user_data, char *addr); + +} chassis_plugin; + +CHASSIS_API chassis_plugin *chassis_plugin_new(void); +CHASSIS_API chassis_plugin *chassis_plugin_load(const gchar *name); +CHASSIS_API void chassis_plugin_free(chassis_plugin *p); +CHASSIS_API GList *chassis_plugin_get_options(chassis_plugin *p); + +/** + * Retrieve the chassis plugin for a particular name. + * + * @param chas a pointer to the chassis + * @param plugin_name The name of the plugin to look up. + * @return A pointer to a chassis_plugin structure + * @retval NULL if there is no loaded chassis with this name + */ +CHASSIS_API chassis_plugin *chassis_plugin_for_name(chassis *chas, gchar *plugin_name); + +#endif diff --git a/src/chassis-shutdown-hooks.c b/src/chassis-shutdown-hooks.c new file mode 100644 index 0000000..b4ae353 --- /dev/null +++ b/src/chassis-shutdown-hooks.c @@ -0,0 +1,42 @@ +#include +#include + +#include "chassis-shutdown-hooks.h" +#include "glib-ext.h" + +void chassis_shutdown_hook_free(chassis_shutdown_hook_t *hook) { + g_slice_free(chassis_shutdown_hook_t, hook); +} + +chassis_shutdown_hooks_t *chassis_shutdown_hooks_new() { + chassis_shutdown_hooks_t *hooks; + + hooks = g_slice_new0(chassis_shutdown_hooks_t); + hooks->hooks = g_hash_table_new_full( + (GHashFunc)g_string_hash, + (GEqualFunc)g_string_equal, + g_string_true_free, + (GDestroyNotify)chassis_shutdown_hook_free); + + return hooks; +} + +void chassis_shutdown_hooks_free(chassis_shutdown_hooks_t *hooks) { + g_hash_table_destroy(hooks->hooks); + + g_slice_free(chassis_shutdown_hooks_t, hooks); +} + + +void chassis_shutdown_hooks_call(chassis_shutdown_hooks_t *hooks) { + GHashTableIter iter; + GString *key; + chassis_shutdown_hook_t *hook; + + g_hash_table_iter_init(&iter, hooks->hooks); + while (g_hash_table_iter_next(&iter, (void **)&key, (void **)&hook)) { + if (hook->func && !hook->is_called) hook->func(hook->udata); + hook->is_called = TRUE; + } +} + diff --git a/src/chassis-shutdown-hooks.h b/src/chassis-shutdown-hooks.h new file mode 100644 index 0000000..ba4cbde --- /dev/null +++ b/src/chassis-shutdown-hooks.h @@ -0,0 +1,28 @@ +#ifndef _CHASSIS_SHUTDOWN_HOOKS_H_ +#define _CHASSIS_SHUTDOWN_HOOKS_H_ + +#include /* GPtrArray */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "chassis-exports.h" + +typedef struct { + void (*func)(gpointer _udata); + gpointer udata; + gboolean is_called; +} chassis_shutdown_hook_t; + +CHASSIS_API void chassis_shutdown_hook_free(chassis_shutdown_hook_t *); + +typedef struct { + GHashTable *hooks; +} chassis_shutdown_hooks_t; + +CHASSIS_API chassis_shutdown_hooks_t *chassis_shutdown_hooks_new(void); +CHASSIS_API void chassis_shutdown_hooks_free(chassis_shutdown_hooks_t *); +CHASSIS_API void chassis_shutdown_hooks_call(chassis_shutdown_hooks_t *hooks); + +#endif diff --git a/src/chassis-timings.c b/src/chassis-timings.c new file mode 100644 index 0000000..caa57ec --- /dev/null +++ b/src/chassis-timings.c @@ -0,0 +1,48 @@ +#include +#include +#include + +#include "chassis-timings.h" +#include "glib-ext.h" + +int chassis_epoch_from_string(const char *str, gboolean *ok) +{ + if (!str) { + *ok = FALSE; + return 0; + } + struct tm t = {0}; + if (strptime(str, "%Y-%m-%d %H:%M:%S", &t)) {/* %Y-%m-%d will fail */ + if (ok) + *ok = TRUE; + return mktime(&t); + } + struct tm d = {0}; + if (strptime(str, "%Y-%m-%d", &d)) { /* %Y-%m-%d %H:%M:%S will also pass, with 'd' set wrong */ + if (ok) + *ok = TRUE; + return mktime(&d); + } + if (ok) + *ok = FALSE; + return 0; +} + +gboolean chassis_timeval_from_double(struct timeval *dst, double t) +{ + g_return_val_if_fail(dst != NULL, FALSE); + g_return_val_if_fail(t >= 0, FALSE); + + dst->tv_sec = floor(t); + dst->tv_usec = floor((t - dst->tv_sec) * 1000000); + + return TRUE; +} + +void chassis_epoch_to_string(time_t *epoch, char *str, int len) +{ + struct tm *local = localtime(epoch); + if (local) { + strftime(str, len, "%Y-%m-%d %H:%M:%S", local); + } +} diff --git a/src/chassis-timings.h b/src/chassis-timings.h new file mode 100644 index 0000000..cf1eb85 --- /dev/null +++ b/src/chassis-timings.h @@ -0,0 +1,16 @@ +#ifndef __CHASSIS_TIMINGS_H__ +#define __CHASSIS_TIMINGS_H__ + +#include /* struct timeval */ +#include +#include "chassis-exports.h" + +#define SECONDS ( 1) +#define MINUTES ( 60 * SECONDS) +#define HOURS ( 60 * MINUTES) + +CHASSIS_API int chassis_epoch_from_string(const char *str, gboolean *ok); +CHASSIS_API gboolean chassis_timeval_from_double(struct timeval *dst, double t); +void chassis_epoch_to_string(time_t *epoch, char *str, int len); + +#endif diff --git a/src/chassis-unix-daemon.c b/src/chassis-unix-daemon.c new file mode 100644 index 0000000..5786089 --- /dev/null +++ b/src/chassis-unix-daemon.c @@ -0,0 +1,216 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_SYS_WAIT_H +#include /* wait4 */ +#endif +#include +#ifdef HAVE_SYS_RESOURCE_H +#include /* getrusage */ +#endif + +#include +#include +#ifdef HAVE_UNISTD_H +#include +#endif +#include +#include +#include + +#include + +#include "glib-ext.h" +#include "chassis-unix-daemon.h" + +/** + * start the app in the background + * + * UNIX-version + */ +void chassis_unix_daemonize(void) { +#ifdef SIGTTOU + signal(SIGTTOU, SIG_IGN); +#endif +#ifdef SIGTTIN + signal(SIGTTIN, SIG_IGN); +#endif +#ifdef SIGTSTP + signal(SIGTSTP, SIG_IGN); +#endif + if (fork() != 0) exit(0); + + if (setsid() == -1) exit(0); + + signal(SIGHUP, SIG_IGN); + + if (fork() != 0) exit(0); + + umask(0); +} + + +/** + * forward the signal to the process group, but not us + */ +static void chassis_unix_signal_forward(int sig) { + signal(sig, SIG_IGN); /* we don't want to create a loop here */ + + kill(0, sig); +} + +/** + * keep the ourself alive + * + * if we or the child gets a SIGTERM, we quit too + * on everything else we restart it + */ +int chassis_unix_proc_keepalive(int *child_exit_status) { + int nprocs = 0; + pid_t child_pid = -1; + + /* + * we ignore SIGINT and SIGTERM and + * just let it be forwarded to the child instead + * as we want to collect its PID before we shutdown too + * + * the child will have to set its own signal handlers for this + */ + + for (;;) { + /* try to start the children */ + while (nprocs < 1) { + pid_t pid = fork(); + + if (pid == 0) { + /* child */ + + g_debug("%s: we are the child: %d, nprocs:%d", + G_STRLOC, + getpid(), + nprocs); + return 0; + } else if (pid < 0) { + /* fork() failed */ + + g_critical("%s: fork() failed: %s (%d), nprocs:%d", + G_STRLOC, + g_strerror(errno), + errno, + nprocs); + + return -1; + } else { + /* we are the angel, let's see what the child did */ + g_message("%s: [angel] we try to keep PID=%d alive, nprocs:%d", + G_STRLOC, + pid, + nprocs); + + /* forward a few signals that are sent to us to the child instead */ + signal(SIGINT, chassis_unix_signal_forward); + signal(SIGTERM, chassis_unix_signal_forward); + signal(SIGHUP, chassis_unix_signal_forward); + signal(SIGUSR1, chassis_unix_signal_forward); + signal(SIGUSR2, chassis_unix_signal_forward); + + child_pid = pid; + nprocs++; + } + } + + if (child_pid != -1) { + struct rusage rusage; + int exit_status; + pid_t exit_pid; + + g_debug("%s: waiting for %d", + G_STRLOC, + child_pid); +#ifdef HAVE_WAIT4 + exit_pid = wait4(child_pid, &exit_status, 0, &rusage); +#else + /* make sure everything is zero'ed out */ + memset(&rusage, 0, sizeof(rusage)); + exit_pid = waitpid(child_pid, &exit_status, 0); +#endif + g_debug("%s: %d returned: %d", + G_STRLOC, + child_pid, + exit_pid); + + if (exit_pid == child_pid) { + /* our child returned, let's see how it went */ + if (WIFEXITED(exit_status)) { + g_message("%s: PID=%d exited with %d (%ld kBytes max)", + G_STRLOC, + child_pid, + WEXITSTATUS(exit_status), + rusage.ru_maxrss / 1024); + if (child_exit_status) { + *child_exit_status = WEXITSTATUS(exit_status); + } + + if (WEXITSTATUS(exit_status) != EXIT_SUCCESS && + WEXITSTATUS(exit_status) != EXIT_FAILURE) + { + int time_towait = 2; + signal(SIGINT, SIG_DFL); + signal(SIGTERM, SIG_DFL); + signal(SIGHUP, SIG_DFL); + while (time_towait > 0) time_towait = sleep(time_towait); + nprocs--; + child_pid = -1; + } else { + return 1; + } + + return 1; + } else if (WIFSIGNALED(exit_status)) { + int time_towait = 2; + /* our child died on a signal + * + * log it and restart */ + + g_critical("%s: PID=%d died on signal=%d (%ld kBytes max)", + G_STRLOC, + child_pid, + WTERMSIG(exit_status), + rusage.ru_maxrss / 1024); + + /** + * to make sure we don't loop as fast as we can, + * sleep a bit between restarts + */ + + signal(SIGINT, SIG_DFL); + signal(SIGTERM, SIG_DFL); + signal(SIGHUP, SIG_DFL); + while (time_towait > 0) time_towait = sleep(time_towait); + + nprocs--; + child_pid = -1; + } else if (WIFSTOPPED(exit_status)) { + } else { + g_assert_not_reached(); + } + } else if (-1 == exit_pid) { + /* EINTR is ok, all others bad */ + if (EINTR != errno) { + /* how can this happen ? */ + g_critical("%s: wait4(%d, ...) failed: %s (%d)", + G_STRLOC, + child_pid, + g_strerror(errno), + errno); + + return -1; + } + } else { + g_assert_not_reached(); + } + } + } +} + diff --git a/src/chassis-unix-daemon.h b/src/chassis-unix-daemon.h new file mode 100644 index 0000000..e2dda0b --- /dev/null +++ b/src/chassis-unix-daemon.h @@ -0,0 +1,7 @@ +#ifndef __CHASSIS_UNIX_DAEMON_H__ +#define __CHASSIS_UNIX_DAEMON_H__ + +int chassis_unix_proc_keepalive(int *child_exit_status); +void chassis_unix_daemonize(void); + +#endif diff --git a/src/glib-ext.c b/src/glib-ext.c new file mode 100644 index 0000000..404d5d7 --- /dev/null +++ b/src/glib-ext.c @@ -0,0 +1,130 @@ +#include + +#include "glib-ext.h" +#include "sys-pedantic.h" +#include +#include "cetus-util.h" + +/** + * free function for GStrings in a GHashTable + */ +void g_hash_table_string_free(gpointer data) { + g_string_free(data, TRUE); +} + +/** + * common GString free func as a hook + */ +void g_string_true_free(gpointer data) { + g_string_free(data, TRUE); +} + + +/** + * hash function for GString + */ +guint g_hash_table_string_hash(gconstpointer _key) { + return g_string_hash(_key); +} + +/** + * compare function for GString + */ +gboolean g_hash_table_string_equal(gconstpointer _a, gconstpointer _b) { + return g_string_equal(_a, _b); +} + +/** + * true-function for g_hash_table_foreach + */ + gboolean +g_hash_table_true(gpointer UNUSED_PARAM(key), gpointer UNUSED_PARAM(value), + gpointer UNUSED_PARAM(u)) +{ + return TRUE; +} + + +/** + * duplicate a GString + */ +GString *g_string_dup(GString *src) { + GString *dst = g_string_sized_new(src->len+1); + + g_string_assign(dst, src->str); + + return dst; +} + +/** + * compare two strings (gchar arrays), whose lengths are known + */ +gboolean strleq(const gchar *a, gsize a_len, const gchar *b, gsize b_len) { + if (a_len != b_len) return FALSE; + return (0 == memcmp(a, b, a_len)); +} + +/** + * calculate the difference between two GTimeVal values, in usec + * positive return value in *tdiff means *told is indeed "earlier" than *tnew, + * negative return value means the reverse + * Caller is responsible for passing valid pointers + */ +void ge_gtimeval_diff(GTimeVal *told, GTimeVal *tnew, gint64 *tdiff) { + *tdiff = (gint64) tnew->tv_sec - told->tv_sec; + *tdiff *= G_USEC_PER_SEC; + *tdiff += (gint64) tnew->tv_usec - told->tv_usec; +} + +GString *g_string_assign_len(GString *s, const char *str, gsize str_len) { + g_string_truncate(s, 0); + return g_string_append_len(s, str, str_len); +} + +void g_debug_hexdump(const char *msg, const void *_s, size_t len) { + GString *hex; + size_t i; + const unsigned char *s = _s; + + hex = g_string_new(NULL); + + for (i = 0; i < len; i++) { + if (i % 16 == 0) { + g_string_append_printf(hex, "[%04"G_GSIZE_MODIFIER"x] ", i); + } + g_string_append_printf(hex, "%02x", s[i]); + + if ((i + 1) % 16 == 0) { + size_t j; + g_string_append_len(hex, C(" ")); + for (j = i - 15; j <= i; j++) { + g_string_append_c(hex, g_ascii_isprint(s[j]) ? s[j] : '.'); + } + g_string_append_len(hex, C("\n ")); + } else { + g_string_append_c(hex, ' '); + } + } + + if (i % 16 != 0) { + /* fill up the line */ + size_t j; + + for (j = 0; j < 16 - (i % 16); j++) { + g_string_append_len(hex, C(" ")); + } + + g_string_append_len(hex, C(" ")); + for (j = i - (len % 16); j < i; j++) { + g_string_append_c(hex, g_ascii_isprint(s[j]) ? s[j] : '.'); + } + } + + g_debug("(%s) %"G_GSIZE_FORMAT" bytes:\n %s", + msg, + len, + hex->str); + + g_string_free(hex, TRUE); +} + diff --git a/src/glib-ext.h b/src/glib-ext.h new file mode 100644 index 0000000..9c134f3 --- /dev/null +++ b/src/glib-ext.h @@ -0,0 +1,33 @@ +#ifndef _GLIB_EXT_H_ +#define _GLIB_EXT_H_ + +#include +#include "config.h" +#include "chassis-exports.h" + +#ifndef USE_GLIB_DEBUG_LOG + +/* default: define g_debug() to nothing */ +#ifdef g_debug +#undef g_debug +#define g_debug(...) +#endif /* g_debug */ + +#endif /* USE_G_DEBUG_LOG */ + +CHASSIS_API void g_string_true_free(gpointer data); + +CHASSIS_API gboolean g_hash_table_true(gpointer key, gpointer value, gpointer user_data); +CHASSIS_API guint g_hash_table_string_hash(gconstpointer _key); +CHASSIS_API gboolean g_hash_table_string_equal(gconstpointer _a, gconstpointer _b); +CHASSIS_API void g_hash_table_string_free(gpointer data); + +CHASSIS_API GString *g_string_dup(GString *); + +CHASSIS_API gboolean strleq(const gchar *a, gsize a_len, const gchar *b, gsize b_len); + +CHASSIS_API void ge_gtimeval_diff(GTimeVal *old, GTimeVal *new, gint64 *delay); +CHASSIS_API GString *g_string_assign_len(GString *s, const char *, gsize); +CHASSIS_API void g_debug_hexdump(const char *msg, const void *s, size_t len); + +#endif diff --git a/src/mysql-proxy-cli.c b/src/mysql-proxy-cli.c new file mode 100644 index 0000000..8769510 --- /dev/null +++ b/src/mysql-proxy-cli.c @@ -0,0 +1,1023 @@ +/** @file + * the user-interface for the cetus @see main() + * + * - command-line handling + * - config-file parsing + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +#ifdef HAVE_SIGNAL_H +#include +#endif +#include +#include +#include +#include + +#include +#include +#include /* for rusage in wait() */ + +#include +#include + +#ifdef HAVE_VALGRIND_VALGRIND_H +#include +#endif + +#ifdef HAVE_SIGACTION +#include +#endif + +#ifndef HAVE_VALGRIND_VALGRIND_H +#define RUNNING_ON_VALGRIND 0 +#endif + +#include "glib-ext.h" +#include "network-mysqld.h" +#include "network-mysqld-proto.h" +#include "sys-pedantic.h" + +#include "cetus-log.h" +#include "chassis-log.h" +#include "chassis-keyfile.h" +#include "chassis-mainloop.h" +#include "chassis-path.h" +#include "chassis-limits.h" +#include "chassis-filemode.h" +#include "chassis-unix-daemon.h" +#include "chassis-frontend.h" +#include "chassis-options.h" +#include "cetus-monitor.h" + +#define GETTEXT_PACKAGE "cetus" + +/** + * options of the cetus frontend + */ +typedef struct { + int print_version; + int verbose_shutdown; + + int daemon_mode; + int set_client_found_rows; + int default_pool_size; + int max_pool_size; + int merged_output_size; + int max_header_size; + int max_resp_len; + int master_preferred; + int worker_id; + int config_port; + int disable_threads; + int is_tcp_stream_enabled; + int is_back_compressed; + int is_client_compress_support; + int check_slave_delay; + int is_reduce_conns; + int is_reset_conn_enabled; + int long_query_time; + int xa_log_detailed; + int cetus_max_allowed_packet; + int default_query_cache_timeout; + int query_cache_enabled; + int disable_dns_cache; + double slave_delay_down_threshold_sec; + double slave_delay_recover_threshold_sec; + + guint invoke_dbg_on_crash; + /* the --keepalive option isn't available on Unix */ + guint auto_restart; + gint max_files_number; + + gchar *user; + + gchar *base_dir; + gchar *conf_dir; + + gchar *default_file; + GKeyFile *keyfile; + + chassis_plugin *p; + GOptionEntry *config_entries; + + gchar *pid_file; + + gchar *plugin_dir; + gchar **plugin_names; + + gchar *log_level; + gchar *log_filename; + gchar *log_xa_filename; + char *default_username; + char *default_charset; + char *default_db; + + char *remote_config_url; +} chassis_frontend_t; + +/** + * create a new the frontend for the chassis + */ +chassis_frontend_t *chassis_frontend_new(void) { + chassis_frontend_t *frontend; + + frontend = g_slice_new0(chassis_frontend_t); + frontend->max_files_number = 0; + frontend->disable_threads = 0; + frontend->is_back_compressed = 0; + frontend->is_client_compress_support = 0; + frontend->xa_log_detailed = 0; + + frontend->default_pool_size = 100; + frontend->max_resp_len = 10 * 1024 * 1024; /* 10M */ + frontend->merged_output_size = 8192; + frontend->max_header_size = 65536; + frontend->config_port = 3306; + + frontend->slave_delay_down_threshold_sec = 60.0; + frontend->default_query_cache_timeout = 100; + frontend->long_query_time = MAX_QUERY_TIME; + frontend->cetus_max_allowed_packet = MAX_ALLOWED_PACKET_DEFAULT; + frontend->disable_dns_cache = 0; + return frontend; +} + +/** + * free the frontend of the chassis + */ +void chassis_frontend_free(chassis_frontend_t *frontend) { + if (!frontend) return; + + if (frontend->keyfile) g_key_file_free(frontend->keyfile); + g_free(frontend->default_file); + g_free(frontend->log_xa_filename); + g_free(frontend->log_filename); + + g_free(frontend->base_dir); + g_free(frontend->conf_dir); + g_free(frontend->user); + g_free(frontend->pid_file); + g_free(frontend->log_level); + g_free(frontend->plugin_dir); + g_free(frontend->default_username); + g_free(frontend->default_db); + g_free(frontend->default_charset); + + if (frontend->plugin_names) { + g_strfreev(frontend->plugin_names); + } + + g_free(frontend->remote_config_url); + + g_slice_free(chassis_frontend_t, frontend); +} + + +/** + * setup the options of the chassis + */ +int chassis_frontend_set_chassis_options(chassis_frontend_t *frontend, chassis_options_t *opts) { + chassis_options_add(opts, + "verbose-shutdown", + 0, 0, OPTION_ARG_NONE, &(frontend->verbose_shutdown), + "Always log the exit code when shutting down", NULL); + + chassis_options_add(opts, + "daemon", + 0, 0, OPTION_ARG_NONE, &(frontend->daemon_mode), + "Start in daemon-mode", NULL); + + chassis_options_add(opts, + "user", + 0, 0, OPTION_ARG_STRING, &(frontend->user), + "Run cetus as user", ""); + + chassis_options_add(opts, + "basedir", + 0, 0, OPTION_ARG_STRING, &(frontend->base_dir), + "Base directory to prepend to relative paths in the config", + ""); + + chassis_options_add(opts, + "conf-dir", + 0, 0, OPTION_ARG_STRING, &(frontend->conf_dir), + "Configuration directory", + ""); + + chassis_options_add(opts, + "pid-file", + 0, 0, OPTION_ARG_STRING, &(frontend->pid_file), + "PID file in case we are started as daemon", ""); + + chassis_options_add(opts, + "plugin-dir", + 0, 0, OPTION_ARG_STRING, &(frontend->plugin_dir), + "Path to the plugins", ""); + + chassis_options_add(opts, + "plugins", + 0, 0, OPTION_ARG_STRING_ARRAY, &(frontend->plugin_names), + "Plugins to load", ""); + + chassis_options_add(opts, + "log-level", + 0, 0, OPTION_ARG_STRING, &(frontend->log_level), + "Log all messages of level ... or higher", + "(error|warning|info|message|debug)"); + + chassis_options_add(opts, + "log-file", + 0, 0, OPTION_ARG_STRING, &(frontend->log_filename), + "Log all messages in a file", ""); + + chassis_options_add(opts, + "log-xa-file", + 0, 0, OPTION_ARG_STRING, &(frontend->log_xa_filename), + "Log all xa messages in a file", ""); + + chassis_options_add(opts, + "log-backtrace-on-crash", + 0, 0, OPTION_ARG_NONE, &(frontend->invoke_dbg_on_crash), + "Try to invoke debugger on crash", NULL); + + chassis_options_add(opts, + "keepalive", + 0, 0, OPTION_ARG_NONE, &(frontend->auto_restart), + "Try to restart the proxy if it crashed", NULL); + + chassis_options_add(opts, + "max-open-files", + 0, 0, OPTION_ARG_INT, &(frontend->max_files_number), + "Maximum number of open files (ulimit -n)", NULL); + + chassis_options_add(opts, + "default-charset", + 0, 0, OPTION_ARG_STRING, &(frontend->default_charset), + "Set the default character set for backends", ""); + + chassis_options_add(opts, + "default-username", + 0, 0, OPTION_ARG_STRING, &(frontend->default_username), + "Set the default username for visiting backends", ""); + + chassis_options_add(opts, + "default-db", + 0, 0, OPTION_ARG_STRING, &(frontend->default_db), + "Set the default db for visiting backends", ""); + + chassis_options_add(opts, + "default-pool-size", + 0, 0, OPTION_ARG_INT, &(frontend->default_pool_size), + "Set the default pool szie for visiting backends", ""); + + chassis_options_add(opts, + "max-pool-size", + 0, 0, OPTION_ARG_INT, &(frontend->max_pool_size), + "Set the max pool szie for visiting backends", ""); + + chassis_options_add(opts, + "max-resp-size", + 0, 0, OPTION_ARG_INT, &(frontend->max_resp_len), + "Set the max response size for one backend", ""); + + chassis_options_add(opts, + "merged-output-size", + 0, 0, OPTION_ARG_INT, &(frontend->merged_output_size), + "set the merged output size for tcp streaming", ""); + + chassis_options_add(opts, + "max-header-size", + 0, 0, OPTION_ARG_INT, &(frontend->max_header_size), + "set the max header size for tcp streaming", ""); + + chassis_options_add(opts, + "worker_id", + 0, 0, OPTION_ARG_INT, &(frontend->worker_id), + "Set the worker id and the maximum value allowed is 63 and the min value is 1", + ""); + + chassis_options_add(opts, + "disable-threads", + 0, 0, OPTION_ARG_NONE, &(frontend->disable_threads), + "Disable all threads creation", NULL); + + chassis_options_add(opts, + "enable-back-compress", + 0, 0, OPTION_ARG_NONE, &(frontend->is_back_compressed), + "enable compression for backend interactions", NULL); + + chassis_options_add(opts, + "enable-client-compress", + 0, 0, OPTION_ARG_NONE, &(frontend->is_client_compress_support), + "enable compression for client interactions", NULL); + + chassis_options_add(opts, + "check-slave-delay", + 0, 0, OPTION_ARG_NONE, &(frontend->check_slave_delay), + "Check ro backends with heartbeat", NULL); + + chassis_options_add(opts, + "slave-delay-down", + 0, 0, OPTION_ARG_DOUBLE, &(frontend->slave_delay_down_threshold_sec), + "Slave will be set down after reach this delay secondes", ""); + + chassis_options_add(opts, + "slave-delay-recover", + 0, 0, OPTION_ARG_DOUBLE, &(frontend->slave_delay_recover_threshold_sec), + "Slave will recover after below this delay secondes", ""); + + chassis_options_add(opts, + "default-query-cache-timeout", + 0, 0, OPTION_ARG_INT, &(frontend->default_query_cache_timeout), + "timeout when proxy connect to backends", ""); + + chassis_options_add(opts, + "long-query-time", + 0, 0, OPTION_ARG_INT, &(frontend->long_query_time), + "Long query time in ms", ""); + + chassis_options_add(opts, + "enable-client-found-rows", + 0, 0, OPTION_ARG_NONE, &(frontend->set_client_found_rows), + "Set client found rows flag", NULL); + + chassis_options_add(opts, + "reduce-connections", + 0, 0, OPTION_ARG_NONE, &(frontend->is_reduce_conns), + "Reduce connections when idle connection num is too high", NULL); + + chassis_options_add(opts, + "enable-reset-connection", + 0, 0, OPTION_ARG_NONE, &(frontend->is_reset_conn_enabled), + "Restart connections when feature changed", NULL); + + chassis_options_add(opts, + "enable-query-cache", + 0, 0, OPTION_ARG_NONE, &(frontend->query_cache_enabled), + "", NULL); + + chassis_options_add(opts, + "enable-tcp-stream", + 0, 0, OPTION_ARG_NONE, &(frontend->is_tcp_stream_enabled), + "", NULL); + + chassis_options_add(opts, + "log-xa-in-detail", + 0, 0, OPTION_ARG_NONE, &(frontend->xa_log_detailed), + "log xa in detail", NULL); + + chassis_options_add(opts, + "disable-dns-cache", + 0, 0, OPTION_ARG_NONE, &(frontend->disable_dns_cache), + "Every new connection to backends will resolve domain name", NULL); + + chassis_options_add(opts, + "master-preferred", + 0, 0, OPTION_ARG_NONE, &(frontend->master_preferred), + "Access to master preferentially", NULL); + chassis_options_add(opts, + "max-allowed-packet", + 0, 0, OPTION_ARG_INT, &(frontend->cetus_max_allowed_packet), + "Max allowed packet as in mysql", ""); + chassis_options_add(opts, + "remote-conf-url", + 0, 0, OPTION_ARG_STRING, &(frontend->remote_config_url), + "Remote config url, mysql://xx", ""); + + return 0; +} + + +#ifdef HAVE_SIGACTION +static void +log_backtrace(void) +{ + void *array[16]; + int size, i; + char **strings; + + size = backtrace(array, 16); + strings = backtrace_symbols(array, size); + g_warning("Obtained %d stack frames.", size); + + for (i = 0; i < size; i++) { + g_warning("%s", strings[i]); + } + + free (strings); +} + +static void sigsegv_handler(int G_GNUC_UNUSED signum) { + log_backtrace(); + + abort(); /* trigger a SIGABRT instead of just exiting */ +} +#endif + +static gboolean check_plugin_mode_valid(chassis_frontend_t *frontend, chassis *srv) +{ + int i, proxy_mode = 0; + + for (i = 0; frontend->plugin_names[i]; ++i) { + char *name = frontend->plugin_names[i]; + if (strcmp(name, "shard") == 0) { + srv->sharding_mode = 1; + g_message("set sharding mode true"); + } + if (strcmp(name, "proxy") == 0) { + proxy_mode = 1; + } + } + + if (srv->sharding_mode && proxy_mode) { + g_critical("shard & proxy is mutual exclusive"); + return FALSE; + } + + return TRUE; +} + +static void g_query_cache_item_free(gpointer q) { + query_cache_item *item = q; + network_queue_free(item->queue); + g_free(item); +} + +/* strdup with 1) default value & 2) NULL check */ +#define DUP_STRING(STR, DEFAULT) \ + (STR) ? g_strdup(STR) : ((DEFAULT) ? g_strdup(DEFAULT) : NULL) + +static void init_parameters(chassis_frontend_t *frontend, chassis *srv) +{ + srv->default_username = DUP_STRING(frontend->default_username, NULL); + srv->default_charset = DUP_STRING(frontend->default_charset, NULL); + srv->default_db = DUP_STRING(frontend->default_db, NULL); + + srv->mid_idle_connections = frontend->default_pool_size; + g_message("set default pool size:%d", srv->mid_idle_connections); + + if (frontend->max_pool_size >= srv->mid_idle_connections) { + srv->max_idle_connections = frontend->max_pool_size; + } else { + srv->max_idle_connections = srv->mid_idle_connections << 1; + } + g_message("set max pool size:%d", srv->max_idle_connections); + + srv->max_resp_len = frontend->max_resp_len; + g_message("set max resp len:%d", srv->max_resp_len); + + srv->merged_output_size = frontend->merged_output_size; + srv->compressed_merged_output_size = srv->merged_output_size << 3; + g_message("%s:set merged output size:%d", G_STRLOC, srv->merged_output_size); + + srv->max_header_size = frontend->max_header_size; + g_message("%s:set max header size:%d", G_STRLOC, srv->max_header_size); + + if (frontend->worker_id > 0) { + srv->guid_state.worker_id = frontend->worker_id & 0x3f; + } + +#undef DUP_STRING + + srv->client_found_rows = frontend->set_client_found_rows; + g_message("set client_found_rows %s", srv->client_found_rows?"true":"false"); + + srv->xa_log_detailed = frontend->xa_log_detailed; + if (srv->xa_log_detailed) { + g_message("%s:xa_log_detailed true", G_STRLOC); + } else { + g_message("%s:xa_log_detailed false", G_STRLOC); + } + srv->is_reset_conn_enabled = frontend->is_reset_conn_enabled; + srv->query_cache_enabled = frontend->query_cache_enabled; + if (srv->query_cache_enabled) { + srv->query_cache_table = g_hash_table_new_full(g_str_hash, + g_str_equal, g_free, g_query_cache_item_free); + srv->cache_index = g_queue_new(); + } + srv->is_tcp_stream_enabled = frontend->is_tcp_stream_enabled; + if (srv->is_tcp_stream_enabled) { + g_message("%s:tcp stream enabled", G_STRLOC); + } + srv->disable_threads = frontend->disable_threads; + srv->is_back_compressed = frontend->is_back_compressed; + srv->compress_support = frontend->is_client_compress_support; + srv->check_slave_delay = frontend->check_slave_delay; + srv->slave_delay_down_threshold_sec = frontend->slave_delay_down_threshold_sec; + srv->master_preferred = frontend->master_preferred; + srv->disable_dns_cache = frontend->disable_dns_cache; + if (frontend->slave_delay_recover_threshold_sec > 0) { + srv->slave_delay_recover_threshold_sec = frontend->slave_delay_recover_threshold_sec; + if (frontend->slave_delay_recover_threshold_sec > srv->slave_delay_down_threshold_sec) { + srv->slave_delay_recover_threshold_sec = srv->slave_delay_down_threshold_sec; + g_warning("`slave-delay-recover` should be lower than `slave-delay-down`."); + g_warning("Set slave-delay-recover=%.3f", srv->slave_delay_down_threshold_sec); + } + } else { + srv->slave_delay_recover_threshold_sec = srv->slave_delay_down_threshold_sec / 2; + } + + srv->default_query_cache_timeout = MAX(frontend->default_query_cache_timeout, 1); + srv->long_query_time = MIN(frontend->long_query_time, MAX_QUERY_TIME); + srv->cetus_max_allowed_packet = CLAMP(frontend->cetus_max_allowed_packet, + MAX_ALLOWED_PACKET_FLOOR, MAX_ALLOWED_PACKET_CEIL); +} + + +static void +release_resouces_when_exit(chassis_frontend_t *frontend, chassis *srv, GError *gerr, + chassis_options_t *opts, chassis_log *log) +{ + if (gerr) g_error_free(gerr); + if (srv) chassis_free(srv); + g_debug("%s: call chassis_options_free", G_STRLOC); + if (opts) chassis_options_free(opts); + g_debug("%s: call g_hash_table_destroy", G_STRLOC); + g_debug("%s: call chassis_log_free", G_STRLOC); + chassis_log_free(log); + tc_log_end(); + + chassis_frontend_free(frontend); +} + +static void +resolve_path(chassis *srv, chassis_frontend_t *frontend) +{ + /* + * these are used before we gathered all the options + * from the plugins, thus we need to fix them up before + * dealing with all the rest. + */ + char *new_path; + new_path = chassis_resolve_path(srv->base_dir, frontend->log_filename); + if (new_path && new_path != frontend->log_filename) { + g_free(frontend->log_filename); + frontend->log_filename = new_path; + } + + new_path = chassis_resolve_path(srv->base_dir, frontend->pid_file); + if (new_path && new_path != frontend->pid_file) { + g_free(frontend->pid_file); + frontend->pid_file = new_path; + } + + new_path = chassis_resolve_path(srv->base_dir, frontend->plugin_dir); + if (new_path && new_path != frontend->plugin_dir) { + g_free(frontend->plugin_dir); + frontend->plugin_dir = new_path; + } + + new_path = chassis_resolve_path(srv->base_dir, frontend->conf_dir); + if (new_path && new_path != frontend->conf_dir) { + g_free(frontend->conf_dir); + frontend->conf_dir = new_path; + } +} + +static void slow_query_log_handler(const gchar *log_domain, GLogLevelFlags log_level, + const gchar *message, gpointer user_data) +{ + FILE* fp = user_data; + time_t t = time(0); + struct tm *tm = localtime(&t); + char timestr[32] = {0}; + const int len = 20; + strftime(timestr, sizeof(timestr), "%Y-%m-%d %H:%M:%S ", tm); + fwrite(timestr, 1, len, fp); + fwrite(message, 1, strlen(message), fp); + fwrite("\n", 1, 1, fp); +} + +static FILE* init_slow_query_log(const char* main_log) +{ + if (!main_log) { + return NULL; + } + GString* log_name = g_string_new(main_log); + g_string_append(log_name, ".slowquery.log"); + + FILE* fp = fopen(log_name->str, "a"); + if (fp) { + g_log_set_handler("slowquery", G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL + | G_LOG_FLAG_RECURSION, slow_query_log_handler, fp); + } + g_string_free(log_name, TRUE); + return fp; +} + +/** + * This is the "real" main which is called on UNIX platforms. + */ +int main_cmdline(int argc, char **argv) { + chassis *srv = NULL; +#ifdef HAVE_SIGACTION + static struct sigaction sigsegv_sa; +#endif + /* read the command-line options */ + chassis_frontend_t *frontend = NULL; + chassis_options_t *opts = NULL; + + GError *gerr = NULL; + chassis_log *log = NULL; + + /* + * a little helper macro to set the src-location that + * we stepped out at to exit + */ +#define GOTO_EXIT(status) \ + exit_code = status; \ + exit_location = G_STRLOC; \ + goto exit_nicely; + + int exit_code = EXIT_SUCCESS; + const gchar *exit_location = G_STRLOC; + + /* init module, ... system */ + if (chassis_frontend_init_glib()) { + GOTO_EXIT(EXIT_FAILURE); + } + + /* start the logging ... to stderr */ + log = chassis_log_new(); + /* display messages while parsing or loading plugins */ + log->min_lvl = G_LOG_LEVEL_MESSAGE; + g_log_set_default_handler(chassis_log_func, log); + + /* may fail on library mismatch */ + if (NULL == (srv = chassis_new())) { + GOTO_EXIT(EXIT_FAILURE); + } + + /* we need the log structure for the log-rotation */ + srv->log = log; + + frontend = chassis_frontend_new(); + + if (frontend == NULL) { + GOTO_EXIT(EXIT_FAILURE); + } + + /** + * parse once to get the basic options like --default-file and --version + * + * leave the unknown options in the list + */ + if (chassis_frontend_init_base_options(&argc, &argv, + &(frontend->print_version), + &(frontend->default_file), + &gerr)) + { + g_critical("%s: %s", G_STRLOC, gerr->message); + g_clear_error(&gerr); + + GOTO_EXIT(EXIT_FAILURE); + } + + if (frontend->default_file) { + if (!(frontend->keyfile = + chassis_frontend_open_config_file(frontend->default_file, &gerr))) + { + g_critical("%s: loading config from '%s' failed: %s", + G_STRLOC, + frontend->default_file, + gerr->message); + g_clear_error(&gerr); + GOTO_EXIT(EXIT_FAILURE); + } + } + + /* print the main version number here, but don't exit + * we check for print_version again, after loading the plugins (if any) + * and print their version numbers, too. then we exit cleanly. + */ + if (frontend->print_version) { + chassis_frontend_print_version(); + } + + /* add the other options which can also appear in the config file */ + opts = chassis_options_new(); + opts->ignore_unknown = TRUE; + srv->options = opts; + + chassis_frontend_set_chassis_options(frontend, opts); + + if (FALSE == chassis_options_parse_cmdline(opts, &argc, &argv, &gerr)) { + g_critical("%s", gerr->message); + GOTO_EXIT(EXIT_FAILURE); + } + + if (frontend->keyfile) { + if (FALSE == chassis_keyfile_to_options_with_error(frontend->keyfile, + "cetus", opts->options, &gerr)) + { + g_critical("%s", gerr->message); + GOTO_EXIT(EXIT_FAILURE); + } + } + + if (frontend->remote_config_url) { + srv->config_manager = chassis_config_from_url(frontend->remote_config_url); + if (!srv->config_manager) { + g_critical("remote config init error"); + GOTO_EXIT(EXIT_FAILURE); + } + if (!chassis_config_parse_options(srv->config_manager, opts->options)) { + g_critical("remote_config parse error"); + GOTO_EXIT(EXIT_FAILURE); + } + } + + if (chassis_frontend_init_basedir(argv[0], &(frontend->base_dir))) { + GOTO_EXIT(EXIT_FAILURE); + } + +#ifdef HAVE_SIGACTION + /* register the sigsegv interceptor */ + + memset(&sigsegv_sa, 0, sizeof(sigsegv_sa)); + sigsegv_sa.sa_handler = sigsegv_handler; + sigemptyset(&sigsegv_sa.sa_mask); + + frontend->invoke_dbg_on_crash = 1; + if (frontend->invoke_dbg_on_crash && !(RUNNING_ON_VALGRIND)) { + sigaction(SIGSEGV, &sigsegv_sa, NULL); + } +#endif + + /* + * some plugins cannot see the chassis struct from the point + * where they open files, hence we must make it available + */ + srv->base_dir = g_strdup(frontend->base_dir); + srv->plugin_dir = g_strdup(frontend->plugin_dir); + chassis_frontend_init_plugin_dir(&frontend->plugin_dir, srv->base_dir); + + if (!frontend->conf_dir) + frontend->conf_dir = g_strdup("conf"); + + resolve_path(srv, frontend); + + srv->conf_dir = g_strdup(frontend->conf_dir); + + if (!srv->config_manager) { /* if no remote-config-url, we use local config */ + srv->config_manager = + chassis_config_from_local_dir(srv->conf_dir, frontend->default_file); + } + + /* + * start the logging + */ + if (frontend->log_filename) { + log->log_filename = g_strdup(frontend->log_filename); + } + + if (log->log_filename && FALSE == chassis_log_open(log)) { + g_critical("can't open log-file '%s': %s", log->log_filename, g_strerror(errno)); + + GOTO_EXIT(EXIT_FAILURE); + } + FILE* slow_query_log_fp = init_slow_query_log(log->log_filename); + if (!slow_query_log_fp) { + g_warning("cannot open slow-query log"); + } + + /* + * handle log-level after the config-file is read, + * just in case it is specified in the file + */ + if (frontend->log_level) { + if (0 != chassis_log_set_level(log, frontend->log_level)) { + g_critical("--log-level=... failed, level '%s' is unknown ", + frontend->log_level); + + GOTO_EXIT(EXIT_FAILURE); + } + } else { + /* if it is not set, use "critical" as default */ + log->min_lvl = G_LOG_LEVEL_CRITICAL; + } + g_message("starting " PACKAGE_STRING); +#ifdef CHASSIS_BUILD_TAG + g_message("build revision: " CHASSIS_BUILD_TAG); +#endif + + g_message("glib version: %d.%d.%d", + GLIB_MAJOR_VERSION, GLIB_MINOR_VERSION, GLIB_MICRO_VERSION); + g_message("libevent version: %s", event_get_version()); + g_message("config dir: %s", frontend->conf_dir); + + if (network_mysqld_init(srv) == -1) { + g_print("network_mysqld_init failed\n"); + GOTO_EXIT(EXIT_FAILURE); + } + + if (!frontend->plugin_names) { + frontend->plugin_names = g_new(char *, 2); +#ifdef DEFAULT_PLUGIN + frontend->plugin_names[0] = g_strdup(DEFAULT_PLUGIN); +#else + frontend->plugin_names[0] = g_strdup("proxy"); +#endif + frontend->plugin_names[1] = NULL; + } + + if (chassis_frontend_load_plugins(srv->modules, + frontend->plugin_dir, + frontend->plugin_names)) { + GOTO_EXIT(EXIT_FAILURE); + } + + if (chassis_frontend_init_plugins(srv->modules, + opts, + srv->config_manager, + &argc, &argv, + frontend->keyfile, + "cetus", + &gerr)) { + g_critical("%s: %s", + G_STRLOC, + gerr->message); + g_clear_error(&gerr); + + GOTO_EXIT(EXIT_FAILURE); + } + + + /* if we only print the version numbers, exit and don't do any more work */ + if (frontend->print_version) { + chassis_frontend_print_plugin_versions(srv->modules); + GOTO_EXIT(EXIT_SUCCESS); + } + + /* we know about the options now, lets parse them */ + opts->ignore_unknown = FALSE; + opts->help_enabled = TRUE; + + /* handle unknown options */ + if (FALSE == chassis_options_parse_cmdline(opts, &argc, &argv, &gerr)) { + if (gerr->domain == G_OPTION_ERROR && + gerr->code == G_OPTION_ERROR_UNKNOWN_OPTION) { + g_critical("%s: %s (use --help to show all options)", + G_STRLOC, + gerr->message); + } else { + g_critical("%s: %s (code = %d, domain = %s)", + G_STRLOC, + gerr->message, + gerr->code, + g_quark_to_string(gerr->domain) + ); + } + + GOTO_EXIT(EXIT_FAILURE); + } + + /* after parsing the options we should only have the program name left */ + if (argc > 1) { + g_critical("unknown option: %s", argv[1]); + + GOTO_EXIT(EXIT_FAILURE); + } + + signal(SIGPIPE, SIG_IGN); + + if (frontend->daemon_mode) { + chassis_unix_daemonize(); + } + + if (frontend->auto_restart) { + int child_exit_status = EXIT_SUCCESS; /* forward the exit-status of the child */ + int ret = chassis_unix_proc_keepalive(&child_exit_status); + + if (ret > 0) { + /* the agent stopped */ + + exit_code = child_exit_status; + goto exit_nicely; + } else if (ret < 0) { + GOTO_EXIT(EXIT_FAILURE); + } else { + /* we are the child, go on */ + } + } + if (frontend->pid_file) { + if (0 != chassis_frontend_write_pidfile(frontend->pid_file, &gerr)) { + g_critical("%s", gerr->message); + g_clear_error(&gerr); + + GOTO_EXIT(EXIT_FAILURE); + } + } + + /* + * log the versions of all loaded plugins + */ + chassis_frontend_log_plugin_versions(srv->modules); + + /* + * we have to drop root privileges in chassis_mainloop() after + * the plugins opened the ports, so we need the user there + */ + srv->user = g_strdup(frontend->user); + + if (check_plugin_mode_valid(frontend, srv) == FALSE) { + GOTO_EXIT(EXIT_FAILURE); + } + + if (frontend->default_username == NULL) { + g_critical("proxy needs default username"); + GOTO_EXIT(EXIT_FAILURE); + } + + init_parameters(frontend, srv); + + if (srv->sharding_mode) { + if (!frontend->log_xa_filename) + frontend->log_xa_filename = g_strdup("logs/xa.log"); + + char *new_path = chassis_resolve_path(srv->base_dir, frontend->log_xa_filename); + if (new_path && new_path != frontend->log_xa_filename) { + g_free(frontend->log_xa_filename); + frontend->log_xa_filename = new_path; + } + + g_message("XA log file: %s", frontend->log_xa_filename); + + if (tc_log_init(frontend->log_xa_filename) == -1) { + GOTO_EXIT(EXIT_FAILURE); + } + } + + if (frontend->max_files_number) { + if (0 != chassis_fdlimit_set(frontend->max_files_number)) { + g_critical("%s: setting fdlimit = %d failed: %s (%d)", + G_STRLOC, + frontend->max_files_number, + g_strerror(errno), + errno); + GOTO_EXIT(EXIT_FAILURE); + } + } + g_debug("max open file-descriptors = %"G_GINT64_FORMAT, + chassis_fdlimit_get()); + + + cetus_monitor_start_thread(srv->priv->monitor, srv); + + if (chassis_mainloop(srv)) { + /* looks like we failed */ + g_critical("%s: Failure from chassis_mainloop. Shutting down.", G_STRLOC); + GOTO_EXIT(EXIT_FAILURE); + } + + cetus_monitor_stop_thread(srv->priv->monitor); + +exit_nicely: + /* necessary to set the shutdown flag, because the monitor will continue + * to schedule timers otherwise, causing an infinite loop in cleanup + */ + if (!exit_code) { + exit_location = G_STRLOC; + } + + struct mallinfo m = mallinfo(); + + g_message("Total allocated space (bytes): %d", m.uordblks); + g_message("Total free space (bytes): %d", m.fordblks); + g_message("Top-most, releasable space (bytes): %d", m.keepcost); + + chassis_set_shutdown_location(exit_location); + + if (frontend && !frontend->print_version) { + /* add a tag to the logfile */ + g_log(G_LOG_DOMAIN, + (frontend->verbose_shutdown ? G_LOG_LEVEL_CRITICAL : G_LOG_LEVEL_MESSAGE), + "shutting down normally, exit code is: %d", exit_code); + } + +#ifdef HAVE_SIGACTION + /* reset the handler */ + sigsegv_sa.sa_handler = SIG_DFL; + if (frontend && frontend->invoke_dbg_on_crash && !(RUNNING_ON_VALGRIND)) { + sigaction(SIGSEGV, &sigsegv_sa, NULL); + } +#endif + + release_resouces_when_exit(frontend, srv, gerr, opts, log); + if (slow_query_log_fp) fclose(slow_query_log_fp); + + return exit_code; +} + +int main(int argc, char **argv) { + return main_cmdline(argc, argv); +} + diff --git a/src/network-address.c b/src/network-address.c new file mode 100644 index 0000000..da69873 --- /dev/null +++ b/src/network-address.c @@ -0,0 +1,524 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#ifdef HAVE_SYS_TYPES_H +#include +#endif + +#include /** inet_ntoa */ +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "network-address.h" +#include "glib-ext.h" +#include "cetus-util.h" + +network_address *network_address_new() { + network_address *addr; + + addr = g_new0(network_address, 1); + addr->len = sizeof(addr->addr); + addr->name = g_string_new(NULL); + g_debug("%s: create socket name %p successful for add:%p", + G_STRLOC, addr->name, addr); + + return addr; +} + +void network_address_free(network_address *addr) { + + if (!addr) return; + + /* + * if the name we're freeing starts with a '/', we're + * looking at a unix socket which needs to be removed + */ + if (addr->can_unlink_socket == TRUE && addr->name != NULL && + addr->name->str != NULL) { + gchar *name = addr->name->str; + if (name[0] == '/') { + int ret = g_remove(name); + if (ret == 0) { + g_debug("%s: removing socket %s successful", + G_STRLOC, name); + } else { + if (errno != EPERM && errno != EACCES) { + g_critical("%s: removing socket %s failed: %s (%d)", + G_STRLOC, name, strerror(errno), errno); + } + } + } + } + + g_string_free(addr->name, TRUE); + g_free(addr); +} + +void network_address_reset(network_address *addr) { + addr->len = sizeof(addr->addr.common); +} + +static gint +network_address_set_address_ip(network_address *addr, + const gchar *address, guint port) +{ + g_return_val_if_fail(addr, -1); + + if (port > 65535) { + g_critical("%s: illegal value %u for port, only 1 ... 65535 allowed", + G_STRLOC, port); + return -1; + } + + if (NULL == address || + address[0] == '\0') { + /* no ip */ +#if 0 + /* disabled as it breaks the default behaviour on FreeBSD and windows + * + * FreeBSD doesn't do IPv6+IPv4 sockets by default, other unixes do. + * while we could change that to with setsockopt(..., IPV6_V6ONLY, ...) + * it should be fixed by adding support for multiple sockets instead. + */ + struct in6_addr addr6 = IN6ADDR_ANY_INIT; + + memset(&addr->addr.ipv6, 0, sizeof(struct sockaddr_in6)); + + addr->addr.ipv6.sin6_addr = addr6; + addr->addr.ipv6.sin6_family = AF_INET6; + addr->addr.ipv6.sin6_port = htons(port); + addr->len = sizeof(struct sockaddr_in6); + +#else + memset(&addr->addr.ipv4, 0, sizeof(struct sockaddr_in)); + + addr->addr.ipv4.sin_addr.s_addr = htonl(INADDR_ANY); + addr->addr.ipv4.sin_family = AF_INET; /* "default" family */ + addr->addr.ipv4.sin_port = htons(port); + addr->len = sizeof(struct sockaddr_in); +#endif + } else if (0 == strcmp("0.0.0.0", address)) { + /* that's any IPv4 address, so bind to IPv4-any only */ + memset(&addr->addr.ipv4, 0, sizeof(struct sockaddr_in)); + + addr->addr.ipv4.sin_addr.s_addr = htonl(INADDR_ANY); + addr->addr.ipv4.sin_family = AF_INET; /* "default" family */ + addr->addr.ipv4.sin_port = htons(port); + addr->len = sizeof(struct sockaddr_in); + } else { +#ifdef HAVE_GETADDRINFO + struct addrinfo *first_ai = NULL; + struct addrinfo hint; + struct addrinfo *ai; + int ret; + + /* AI_ADDRCONFIG filters out ::1 and link-local addresses if there is no + * global IPv6 address assigned to the local interfaces + * + * 1) try to resolve with ADDRCONFIG + * 2) if 1) fails, try without ADDRCONFIG + * + * this should handle DNS problems where + * - only IPv4 is configured, but DNS returns IPv6 records + IPv4 and + * we would pick IPv6 (and fail) + * - + */ + + memset(&hint, 0, sizeof(hint)); + hint.ai_family = PF_UNSPEC; + hint.ai_socktype = SOCK_STREAM; + hint.ai_protocol = 0; + hint.ai_flags = AI_ADDRCONFIG; + if ((ret = getaddrinfo(address, NULL, &hint, &first_ai)) != 0) { + if ( + /* FreeBSD doesn't provide EAI_ADDRFAMILY (since 2003) */ +#ifdef EAI_ADDRFAMILY + EAI_ADDRFAMILY == ret || /* AI_ADDRCONFIG with a non-global address */ +#endif + EAI_BADFLAGS == ret) { /* AI_ADDRCONFIG isn't supported */ + if (first_ai) freeaddrinfo(first_ai); + first_ai = NULL; + + hint.ai_flags &= ~AI_ADDRCONFIG; + + ret = getaddrinfo(address, NULL, &hint, &first_ai); + } + + if (ret != 0) { + g_critical("getaddrinfo(\"%s\") failed: %s (%d)", address, + gai_strerror(ret), + ret); + return -1; + } + } + + ret = 0; /* bogus, just to make it explicit */ + + for (ai = first_ai; ai; ai = ai->ai_next) { + int family = ai->ai_family; + + if (family == PF_INET6) { + memcpy(&addr->addr.ipv6, + (struct sockaddr_in6 *) ai->ai_addr, + sizeof (addr->addr.ipv6)); + addr->addr.ipv6.sin6_port = htons(port); + addr->len = sizeof(struct sockaddr_in6); + + break; + } else if (family == PF_INET) { + memcpy(&addr->addr.ipv4, + (struct sockaddr_in *) ai->ai_addr, + sizeof (addr->addr.ipv4)); + addr->addr.ipv4.sin_port = htons(port); + addr->len = sizeof(struct sockaddr_in); + break; + } + } + + if (ai == NULL) { + /* no matching address-info found */ + g_debug("%s: %s:%d", G_STRLOC, address, port); + ret = -1; + } + + freeaddrinfo(first_ai); + + if (ret != 0) return ret; +#else + struct hostent *he; + + he = gethostbyname(address); + if (NULL == he) { + return -1; + } + + g_assert(he->h_addrtype == AF_INET); + g_assert(he->h_length == sizeof(struct in_addr)); + + memcpy(&(addr->addr.ipv4.sin_addr.s_addr), he->h_addr_list[0], he->h_length); + addr->addr.ipv4.sin_family = AF_INET; + addr->addr.ipv4.sin_port = htons(port); + addr->len = sizeof(struct sockaddr_in); +#endif /* HAVE_GETADDRINFO */ + } + + (void) network_address_refresh_name(addr); + + return 0; +} + +static gint network_address_set_address_un(network_address *addr, const gchar *address) { + g_return_val_if_fail(addr, -1); + g_return_val_if_fail(address, -1); + + if (strlen(address) >= sizeof(addr->addr.un.sun_path) - 1) { + g_critical("unix-path is too long: %s", address); + return -1; + } + + addr->addr.un.sun_family = AF_UNIX; + strcpy(addr->addr.un.sun_path, address); + addr->len = sizeof(struct sockaddr_un); + + network_address_refresh_name(addr); + + return 0; +} + +/** + * translate a address-string into a network_address structure + * + * - if the address contains a colon we assume IPv4, + * - ":3306" -> (tcp) "0.0.0.0:3306" + * - if it starts with a / it is a unix-domain socket + * - "/tmp/socket" -> (unix) "/tmp/socket" + * + * @param addr the address-struct + * @param address the address string + * @return 0 on success, -1 otherwise + */ +gint network_address_set_address(network_address *addr, const gchar *address) { + const gchar *port_part = NULL; + gchar *ip_part = NULL; + gint ret; + + g_return_val_if_fail(addr, -1); + + /* split the address:port */ + if (address[0] == '/') { + return network_address_set_address_un(addr, address); + } else if (address[0] == '[') { + const gchar *s; + if (NULL == (s = strchr(address + 1, ']'))) { + return -1; + } + /* may be NULL for strdup(..., 0) */ + ip_part = g_strndup(address + 1, s - (address + 1)); + + if (*(s+1) == ':') { + port_part = s + 2; + } + } else if (NULL != (port_part = strchr(address, ':'))) { + /* may be NULL for strdup(..., 0) */ + ip_part = g_strndup(address, port_part - address); + port_part++; + } else { + ip_part = g_strdup(address); + } + + /* if there is a colon, there should be a port number */ + if (NULL != port_part) { + char *port_err = NULL; + guint port; + + port = strtoul(port_part, &port_err, 10); + + if (*port_part == '\0') { + g_critical("%s: IP-addr in the form [][:], is '%s'. No port", + G_STRLOC, address); + ret = -1; + } else if (*port_err != '\0') { + g_critical("%s: IP-addr in the form [][:], is '%s'. port at '%s'", + G_STRLOC, address, port_err); + ret = -1; + } else { + ret = network_address_set_address_ip(addr, ip_part, port); + } + } else { + /* perhaps it is a plain IP address, lets add the default-port */ + ret = network_address_set_address_ip(addr, ip_part, 3306); + } + + if (ip_part) g_free(ip_part); + + return ret; +} + +GQuark +network_address_error(void) { + return g_quark_from_static_string("network-address-error"); +} + +#ifdef HAVE_INET_NTOP +static const gchar * +network_address_tostring_inet_ntop(network_address *addr, + gchar *dst, gsize *dst_len, + GError **gerr) { + const char *addr_str; + gsize initial_dst_len = *dst_len; + + /* resolve the peer-addr if we haven't done so yet */ + switch (addr->addr.common.sa_family) { + case AF_INET: + addr_str = inet_ntop(AF_INET, &addr->addr.ipv4.sin_addr, + dst, *dst_len); + if (NULL == addr_str) { + if (ENOSPC == errno) { + g_set_error(gerr, + NETWORK_ADDRESS_ERROR, + NETWORK_ADDRESS_ERROR_DST_TOO_SMALL, + "inet_ntop() failed: %s (%d)", + g_strerror(errno), + errno); + } else { + g_set_error(gerr, + NETWORK_ADDRESS_ERROR, + NETWORK_ADDRESS_ERROR_UNKNOWN, + "inet_ntop() failed: %s (%d)", + g_strerror(errno), + errno); + } + return NULL; + } + *dst_len = strlen(addr_str) + 1; + return addr_str; + case AF_INET6: + addr_str = inet_ntop(AF_INET6, &addr->addr.ipv6.sin6_addr, + dst, *dst_len); + + if (NULL == addr_str) { + if (ENOSPC == errno) { + g_set_error(gerr, + NETWORK_ADDRESS_ERROR, + NETWORK_ADDRESS_ERROR_DST_TOO_SMALL, + "inet_ntop() failed: %s (%d)", + g_strerror(errno), + errno); + } else { + g_set_error(gerr, + NETWORK_ADDRESS_ERROR, + NETWORK_ADDRESS_ERROR_UNKNOWN, + "inet_ntop() failed: %s (%d)", + g_strerror(errno), + errno); + } + return NULL; + } + *dst_len = strlen(addr_str) + 1; + + return addr_str; + case AF_UNIX: + *dst_len = g_strlcpy(dst, addr->addr.un.sun_path, *dst_len); + + if (*dst_len >= initial_dst_len) { + /* g_strlcpy() got overrun */ + g_set_error(gerr, + NETWORK_ADDRESS_ERROR, + NETWORK_ADDRESS_ERROR_DST_TOO_SMALL, + "dst too small"); + return NULL; + } + /* g_strlcpy() returns the size without \0, we return the size with \0 */ + *dst_len += 1; + + return dst; + default: + g_set_error(gerr, + NETWORK_ADDRESS_ERROR, + NETWORK_ADDRESS_ERROR_INVALID_ADDRESS_FAMILY, + "can't convert a address of family '%d' into a string", + addr->addr.common.sa_family); + return NULL; + } +} + +#else + +/** + * resolve a struct sockaddr into a string + */ +static const gchar * +network_address_tostring_inet_ntoa(network_address *addr, gchar *dst, + gsize *dst_len, GError **gerr) +{ + const char *addr_str; + gsize initial_dst_len = *dst_len; + + /* resolve the peer-addr if we haven't done so yet */ + switch (addr->addr.common.sa_family) { + case AF_INET: + addr_str = inet_ntoa(addr->addr.ipv4.sin_addr); + + if (NULL == addr_str) { + g_set_error(gerr, + NETWORK_ADDRESS_ERROR, + NETWORK_ADDRESS_ERROR_UNKNOWN, + "inet_ntoa() failed: %d", + errno); + return NULL; + } + *dst_len = g_strlcpy(dst, addr_str, *dst_len); + + return dst; + case AF_UNIX: + *dst_len = g_strlcpy(dst, addr->addr.un.sun_path, *dst_len); + if (*dst_len >= initial_dst_len) { + /* g_strlcpy() got overrun */ + g_set_error(gerr, + NETWORK_ADDRESS_ERROR, + NETWORK_ADDRESS_ERROR_DST_TOO_SMALL, + "dst too small"); + return NULL; + } + /* g_strlcpy() returns the size without \0, we return the size with \0 */ + *dst_len += 1; + + return dst; + default: + g_set_error(gerr, + NETWORK_ADDRESS_ERROR, + NETWORK_ADDRESS_ERROR_INVALID_ADDRESS_FAMILY, + "can't convert a address of family '%d' into a string", + addr->addr.common.sa_family); + return NULL; + } +} + + + +#endif + +char * +network_address_tostring(network_address *addr, char *dst, gsize *dst_len, GError **gerr) { + const char *addr_str; + + if (NULL == dst) { + g_set_error(gerr, + NETWORK_ADDRESS_ERROR, + NETWORK_ADDRESS_ERROR_INVALID, + "dst is NULL"); + return NULL; + } + if (NULL == dst_len) { + g_set_error(gerr, + NETWORK_ADDRESS_ERROR, + NETWORK_ADDRESS_ERROR_INVALID, + "dst_len is NULL"); + return NULL; + } + +#if defined(HAVE_INET_NTOP) + addr_str = network_address_tostring_inet_ntop(addr, dst, dst_len, gerr); +#else + addr_str = network_address_tostring_inet_ntoa(addr, dst, dst_len, gerr); +#endif + + if (NULL == addr_str) { + /* gerr is already set, just return NULL */ + return NULL; + } + + return dst; +} + +gint network_address_refresh_name(network_address *addr) { + GError *gerr = NULL; + char buf[255]; + gsize buf_len = sizeof(buf); + + if (NULL == network_address_tostring(addr, buf, &buf_len, &gerr)) { + g_critical("%s: %s", + G_STRLOC, + gerr->message); + g_clear_error(&gerr); + return -1; + } + + if (addr->addr.common.sa_family == AF_INET) { + g_string_printf(addr->name, "%s:%d", + buf, + ntohs(addr->addr.ipv4.sin_port)); + } else if (addr->addr.common.sa_family == AF_INET6) { + g_string_printf(addr->name, "[%s]:%d", + buf, + ntohs(addr->addr.ipv6.sin6_port)); + } else { + g_string_assign(addr->name, buf); + } + + return 0; +} + +network_address *network_address_copy(network_address *dst, network_address *src) { + if (!dst) dst = network_address_new(); + + dst->len = src->len; + dst->addr = src->addr; + g_string_assign_len(dst->name, S(src->name)); + + return dst; +} + diff --git a/src/network-address.h b/src/network-address.h new file mode 100644 index 0000000..f9399ec --- /dev/null +++ b/src/network-address.h @@ -0,0 +1,66 @@ +#ifndef _NETWORK_ADDRESS_H_ +#define _NETWORK_ADDRESS_H_ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#ifdef HAVE_NETINET_IN_H +#include /** struct sockaddr_in */ +#endif +#include + +#ifdef HAVE_SYS_UN_H +#include /** struct sockaddr_un */ +#endif +#include /** struct sockaddr (freebsd and hp/ux need it) */ + +#include "network-exports.h" + +#ifdef __hpux +/* see http://curl.haxx.se/mail/lib-2009-04/0287.html */ +typedef int network_socklen_t; +#else +typedef socklen_t network_socklen_t; +#endif + + +#define NETWORK_ADDRESS_ERROR network_address_error() + +GQuark +network_address_error(void); + +enum { + NETWORK_ADDRESS_ERROR_UNKNOWN, + NETWORK_ADDRESS_ERROR_INVALID_ADDRESS_FAMILY, + NETWORK_ADDRESS_ERROR_DST_TOO_SMALL, + NETWORK_ADDRESS_ERROR_INVALID +}; + +typedef struct { + union { + struct sockaddr_in ipv4; + struct sockaddr_in6 ipv6; +#ifdef HAVE_SYS_UN_H + struct sockaddr_un un; +#endif + struct sockaddr common; + } addr; + + GString *name; + network_socklen_t len; + /* set TRUE *only *after successful bind */ + gboolean can_unlink_socket; +} network_address; + +NETWORK_API network_address *network_address_new(void); +NETWORK_API void network_address_free(network_address *); +NETWORK_API void network_address_reset(network_address *addr); +NETWORK_API network_address *network_address_copy(network_address *, network_address *); +NETWORK_API gint network_address_set_address(network_address *, const gchar *); +NETWORK_API gint network_address_refresh_name(network_address *); +NETWORK_API char *network_address_tostring(network_address *, char *, gsize *, GError **); + +#endif diff --git a/src/network-backend.c b/src/network-backend.c new file mode 100644 index 0000000..edb3bb1 --- /dev/null +++ b/src/network-backend.c @@ -0,0 +1,657 @@ +#include "network-backend.h" + +#include +#include + +#include "chassis-plugin.h" +#include "glib-ext.h" +#include "network-mysqld-proto.h" +#include "network-mysqld-packet.h" +#include "character-set.h" +#include "cetus-util.h" +#include "cetus-users.h" + +const char *backend_state_t_str[] = { + "unkown", + "online", + "down", + "maintaining", + "deleted" +}; + +const char *backend_type_t_str[] = { + "unkown", + "read/write", + "readonly" +}; + +network_backend_t *network_backend_new() { + network_backend_t *b; + + b = g_new0(network_backend_t, 1); + + b->pool = network_connection_pool_new(); + b->uuid = g_string_new(NULL); + b->addr = network_address_new(); + b->server_group = g_string_new(NULL); + b->address = g_string_new(NULL); + b->challenges = g_ptr_array_new(); + + return b; +} + + +void network_backend_free(network_backend_t *b) { + if (!b) return; + + network_connection_pool_free(b->pool); + + if (b->addr) network_address_free(b->addr); + if (b->uuid) g_string_free(b->uuid, TRUE); + if (b->challenges) g_ptr_array_free(b->challenges, TRUE); + if (b->server_group) g_string_free(b->server_group, TRUE); + + if (b->config) { + if (b->config->default_username) { + g_string_free(b->config->default_username, TRUE); + } + + if (b->config->default_db) { + g_string_free(b->config->default_db, TRUE); + } + + g_free(b->config); + } + + g_string_free(b->address, TRUE); + g_debug("%s: call network_backend_free end", + G_STRLOC); + g_free(b); +} + +int network_backend_init_extra(network_backend_t *b, chassis *chas) { + if (chas->max_idle_connections != 0) { + b->pool->max_idle_connections = chas->max_idle_connections; + } + + if (chas->mid_idle_connections != 0) { + b->pool->mid_idle_connections = chas->mid_idle_connections; + } + + return 0; +} + +int network_backend_conns_count(network_backend_t *b) +{ + int in_use = b->connected_clients; + int pooled = network_connection_pool_total_conns_count(b->pool); + return in_use + pooled; +} + +/* + * save challenges from backend, will be used to authenticate front user + */ +void network_backend_save_challenge(network_backend_t *b, + const network_mysqld_auth_challenge *chal) +{ + if (b->challenges->len < 256) { + static const guint32 not_supported = CLIENT_SSL| CLIENT_LOCAL_FILES | CLIENT_DEPRECATE_EOF; + + network_mysqld_auth_challenge *challenge = + network_mysqld_auth_challenge_copy(chal); + + challenge->capabilities &= ~not_supported; + char *old_str = challenge->server_version_str; + challenge->server_version_str = g_strdup_printf("%s (%s)", old_str, PACKAGE_STRING); + g_free(old_str); + + g_ptr_array_add(b->challenges, challenge); + } +} + +struct network_mysqld_auth_challenge * +network_backend_get_challenge(network_backend_t *b) +{ + if (b->challenges->len == 0) { + g_message("challenges len 0 for backend:%s", b->addr->name->str); + return NULL; + } + int ndx = b->chal_ndx % b->challenges->len; + b->chal_ndx++; + network_mysqld_auth_challenge *challenge = g_ptr_array_index(b->challenges, ndx); + return challenge; +} + +static network_group_t *network_group_new(); +static void network_group_free(network_group_t *); +static void network_group_add(network_group_t *, network_backend_t *); +static void network_group_update(network_group_t *gp); + +network_backends_t *network_backends_new() { + network_backends_t *bs; + + bs = g_new0(network_backends_t, 1); + + bs->backends = g_ptr_array_new(); + bs->groups = g_ptr_array_new_with_free_func((GDestroyNotify)network_group_free); + return bs; +} + +void network_backends_free(network_backends_t *bs) { + gsize i, j; + + if (!bs) return; + + g_message("%s: call network_backends_free", G_STRLOC); + + for (i = 0; i < bs->backends->len; i++) { + network_backend_t *backend = bs->backends->pdata[i]; + + for (j = 0; j < backend->challenges->len; j++) { + network_mysqld_auth_challenge *challenge = backend->challenges->pdata[j]; + network_mysqld_auth_challenge_free(challenge); + } + + network_backend_free(backend); + } + g_ptr_array_free(bs->backends, TRUE); + g_ptr_array_free(bs->groups, TRUE); + g_free(bs); +} + +static gboolean network_backends_into_group(network_backends_t *bs, + network_backend_t *backend) +{ + if (backend && backend->server_group) { + network_group_t *gp; + if ((gp = network_backends_get_group(bs, backend->server_group)) != NULL) { + network_group_add(gp, backend); + return TRUE; + } + } + return FALSE; +} + +static void network_backends_add_group(network_backends_t *bs, const char *name); + +/* + * FIXME: 1) remove _set_address, make this function callable with result of same + * 2) differentiate between reasons for "we didn't add" (now -1 in all cases) + */ +int network_backends_add(network_backends_t *bs, const gchar *address, + backend_type_t type, backend_state_t state, void *srv) +{ + network_backend_t *new_backend = network_backend_new(); + new_backend->type = type; + new_backend->state = state; + new_backend->pool->srv = srv; + + char *group_p = NULL; + if ((group_p = strrchr(address, '@')) != NULL) { + network_backends_add_group(bs, group_p + 1); + g_string_assign(new_backend->server_group, group_p + 1); + g_string_assign_len(new_backend->address, address, group_p - address); + } else { + g_string_assign(new_backend->address, address); + } + + if (0 != network_address_set_address(new_backend->addr, new_backend->address->str)) { + network_backend_free(new_backend); + return -1; + } + + guint i; + /* check if this backend is already known */ + for (i = 0; i < bs->backends->len; i++) { + network_backend_t *old_backend = g_ptr_array_index(bs->backends, i); + + if (strleq(S(old_backend->addr->name), S(new_backend->addr->name))) { + network_backend_free(new_backend); + + g_critical("backend %s is already known!", address); + return -1; + } + } + + g_ptr_array_add(bs->backends, new_backend); + if (type == BACKEND_TYPE_RO) { + bs->ro_server_num += 1; + } + network_backends_into_group(bs, new_backend); + g_message("added %s backend: %s, state: %s", backend_type_t_str[type], + address, backend_state_t_str[state]); + + return 0; +} + +/** + * we just change the state to deleted + */ +int network_backends_remove(network_backends_t *bs, guint index) { + network_backend_t *b = bs->backends->pdata[index]; + if (b != NULL) { + if (b->type == BACKEND_TYPE_RO && bs->ro_server_num > 0) { + bs->ro_server_num -= 1; + } + + return network_backends_modify(bs, index, BACKEND_TYPE_UNKNOWN, + BACKEND_STATE_DELETED); + } + return 0; +} + +/** + * updated the _DOWN state to _UNKNOWN if the backends were + * down for at least 4 seconds + * + * we only check once a second to reduce the overhead on connection setup + * + * @returns number of updated backends + */ +int network_backends_check(network_backends_t *bs) { + GTimeVal now; + guint i; + int backends_woken_up = 0; + gint64 t_diff; + + g_get_current_time(&now); + ge_gtimeval_diff(&bs->backend_last_check, &now, &t_diff); + + /* check max(once a second) */ + /* this also covers the "time went backards" case */ + if (t_diff < G_USEC_PER_SEC) { + if (t_diff < 0) { + g_message("%s: time went backwards (%"G_GINT64_FORMAT" usec)!", + G_STRLOC, t_diff); + bs->backend_last_check.tv_usec = 0; + bs->backend_last_check.tv_sec = 0; + } + return 0; + } + + bs->backend_last_check = now; + + for (i = 0; i < bs->backends->len; i++) { + network_backend_t *cur = bs->backends->pdata[i]; + + if (cur->state != BACKEND_STATE_DOWN) continue; + + /* check if a backend is marked as down for more than 4 sec */ + if (now.tv_sec - cur->state_since.tv_sec > 4) { + g_debug("%s: backend %s was down for more than 4 secs, waking it up", + G_STRLOC, + cur->addr->name->str); + + cur->state = BACKEND_STATE_UNKNOWN; + cur->state_since = now; + backends_woken_up++; + } + } + + return backends_woken_up; +} + +/** + * modify the backends to new type and new state. + * + * @returns 0 for success -1 for error. + */ + +int network_backends_modify(network_backends_t *bs, guint ndx, + backend_type_t type, backend_state_t state) +{ + GTimeVal now; + g_get_current_time(&now); + if (ndx >= network_backends_count(bs)) return -1; + network_backend_t *cur = bs->backends->pdata[ndx]; + + g_message("change backend: %s from type: %s, state: %s to type: %s, state: %s", + cur->addr->name->str, backend_type_t_str[cur->type], + backend_state_t_str[cur->state], + backend_type_t_str[type], backend_state_t_str[state]); + + if (cur->type != type) { + cur->type = type; + if (type == BACKEND_TYPE_RO) { + bs->ro_server_num += 1; + } else { + bs->ro_server_num -= 1; + } + network_group_t *gp = network_backends_get_group(bs, cur->server_group); + if (gp) + network_group_update(gp); + } + if (cur->state != state) { + cur->state = state; + cur->state_since = now; + } + + g_debug("%s: backend state:%d for backend:%p", G_STRLOC, cur->state, cur); + + return 0; +} + +network_backend_t *network_backends_get(network_backends_t *bs, guint ndx) { + if (ndx >= network_backends_count(bs)) return NULL; + + /* FIXME: shouldn't we copy the backend or add ref-counting ? */ + return bs->backends->pdata[ndx]; +} + +guint network_backends_count(network_backends_t *bs) { + guint len; + + len = bs->backends->len; + + return len; +} + +#define DEFAULT_CHARSET '\x21' + +gboolean network_backends_load_config(network_backends_t *bs, chassis *srv) +{ + if (!cetus_users_contains(srv->priv->users, srv->default_username)) { + g_critical("%s: no required password here for user:%s", + G_STRLOC, srv->default_username); + return -1; + } + int i; + for(i = 0; i < network_backends_count(bs); i++ ) + { + network_backend_t *backend = network_backends_get(bs, i); + if (backend) { + backend->config = g_new0(backend_config, 1); + + backend->config->default_username = g_string_new(NULL); + g_string_append(backend->config->default_username, srv->default_username); + + if (srv->default_db != NULL && strlen(srv->default_db) > 0) { + backend->config->default_db = g_string_new(NULL); + g_string_append(backend->config->default_db, srv->default_db); + } + + backend->config->charset = charset_get_number(srv->default_charset); + + backend->config->mid_conn_pool = srv->mid_idle_connections; + backend->config->max_conn_pool = srv->max_idle_connections; + } + } + return 0; +} + +network_group_t *network_backends_get_group(network_backends_t *bs, const GString *name) +{ + int i = 0; + for (i = 0; i < bs->groups->len; ++i) { + network_group_t *group = g_ptr_array_index(bs->groups, i); + if (g_string_equal(group->name, name)) { + return group; + } + } + return NULL; +} + +static void network_backends_add_group(network_backends_t *bs, const char *name) +{ + GString *gp_name = g_string_new(name); + if (!network_backends_get_group(bs, gp_name)) {/* dup check */ + network_group_t *gp = network_group_new(gp_name); + g_ptr_array_add(bs->groups, gp); + } else { + g_string_free(gp_name, TRUE); + } +} + +static network_group_t *network_group_new(GString *name) +{ + network_group_t *gp = g_new0(network_group_t, 1); + gp->name = name; + return gp; +} + +static void network_group_free(network_group_t *gp) +{ + g_string_free(gp->name, TRUE); + g_free(gp); +} + +static void network_group_add(network_group_t *gp, network_backend_t *backend) +{ + g_assert(backend); + if (backend->type == BACKEND_TYPE_RW) { + if (gp->master) { + g_critical("only one master each group"); + } + gp->master = backend; + } else if (backend->type == BACKEND_TYPE_RO) { + if (gp->nslaves >= MAX_GROUP_SLAVES) { + g_critical("too many slaves for group"); + return; + } + int i = 0; + for (i = 0; i < gp->nslaves; ++i) { + network_backend_t *slave = gp->slaves[i]; + if (strleq(S(slave->addr->name), S(backend->addr->name))) { + return; + } + } + gp->nslaves += 1; + gp->slaves[gp->nslaves-1] = backend; + } +} + +/* some backend in this group changed rw type, update */ +static void network_group_update(network_group_t *gp) +{ + /* take out backends of this group evenly */ + GList *backends = NULL; + if (gp->master) { + backends = g_list_append(backends, gp->master); + gp->master = NULL; + } + int i; + for (i = 0; i < gp->nslaves; ++i) { + backends = g_list_append(backends, gp->slaves[i]); + } + gp->nslaves = 0; + + /* rearrange them into this group */ + GList *l; + for (l = backends; l != NULL; l = l->next) { + network_group_add(gp, l->data); + } + g_list_free(backends); +} + +static int backends_get_ro_ndx_round_robin(network_backends_t *bs) +{ + int ro_index = 0, remainder = 0; + if (bs->ro_server_num > 0) { + remainder = (bs->read_count++) % bs->ro_server_num; + } + int count = network_backends_count(bs); + int i = 0; + for (i = 0; i < count; i++) { + network_backend_t *backend = network_backends_get(bs, i); + if ((backend->type == BACKEND_TYPE_RO) + && (backend->state == BACKEND_STATE_UP || + backend->state == BACKEND_STATE_UNKNOWN)) + { + if (ro_index == remainder) { + break; + } + ro_index++; + } + } + return i < count ? i : -1; +} + +static int backends_get_ro_ndx_first(network_backends_t *bs) +{ + int i = 0; + int count = network_backends_count(bs); + for(i = 0; i < count; i++) { + network_backend_t *backend = network_backends_get(bs, i); + if ((backend->type == BACKEND_TYPE_RO) + && (backend->state == BACKEND_STATE_UP || + backend->state == BACKEND_STATE_UNKNOWN)) + { + return i; + } + } + return -1; +} + +static int backends_get_ro_ndx_random(network_backends_t *bs) +{ + int count = network_backends_count(bs); + int ndx = g_random_int_range(0, count); + network_backend_t *backend = network_backends_get(bs, ndx); + if (backend->type == BACKEND_TYPE_RO) { /* luckily run into a RO */ + if (backend->state == BACKEND_STATE_UP || + backend->state == BACKEND_STATE_UNKNOWN) { + return ndx; + } + } else { /* if we run into a RW, try it's neighbours */ + if (ndx - 1 >= 0) { + backend = network_backends_get(bs, ndx - 1); + if ((backend->type == BACKEND_TYPE_RO) + && (backend->state == BACKEND_STATE_UP || + backend->state == BACKEND_STATE_UNKNOWN)) { + return ndx-1; + } + } else if (ndx + 1 < count) { + if ((backend->type == BACKEND_TYPE_RO) + && (backend->state == BACKEND_STATE_UP || + backend->state == BACKEND_STATE_UNKNOWN)) { + return ndx+1; + } + } + } + + return backends_get_ro_ndx_first(bs); +} + +int network_backends_get_ro_ndx(network_backends_t *bs, backend_algo_t algo) +{ + switch (algo) { + case BACKEND_ALGO_ROUND_ROBIN: + return backends_get_ro_ndx_round_robin(bs); + case BACKEND_ALGO_RANDOM: + return backends_get_ro_ndx_random(bs); + case BACKEND_ALGO_FIRST: + return backends_get_ro_ndx_first(bs); + default: + return -1; + } +} + +int network_backends_get_rw_ndx(network_backends_t *bs) +{ + int i = 0; + int count = network_backends_count(bs); + for (i = 0; i < count; i++) { + network_backend_t *backend = network_backends_get(bs, i); + if ((BACKEND_TYPE_RW == backend->type) && + (backend->state == BACKEND_STATE_UP || + backend->state == BACKEND_STATE_UNKNOWN)) + { + break; + } + } + return i < count ? i : -1; +} + +int network_backends_find_address(network_backends_t *bs, const char *ipport) +{ + int count = network_backends_count(bs); + int i = 0; + for (i = 0; i < count; i++) { + network_backend_t *backend = network_backends_get(bs, i); + if (strcmp(backend->addr->name->str, ipport) == 0) { + return i; + } + } + return -1; +} + +network_mysqld_auth_challenge * +network_backends_get_challenge(network_backends_t *bs, int back_ndx) +{ + network_backend_t *b = network_backends_get(bs, back_ndx); + if (b) + return network_backend_get_challenge(b); + else + return NULL; +} + +/* round robin pick */ +network_backend_t *network_group_pick_slave_backend(network_group_t *group) +{ + int i; + network_backend_t *backend = NULL; + for (i = 0; i < group->nslaves; i++) { + size_t index = (group->slave_visit_cnt++) % group->nslaves; + backend = group->slaves[index]; + if (backend->state != BACKEND_STATE_UP && + backend->state != BACKEND_STATE_UNKNOWN) + { + g_debug(G_STRLOC ": skip dead backend(slave): %d", i); + continue; + } + + int total = network_backend_conns_count(backend); + int connected_clts = backend->connected_clients; + int cur_idle = total - connected_clts; + int max_idle_conns = backend->config->max_conn_pool; + + g_debug("%s, slave:%d, total:%d, connected:%d, idle:%d, max:%d", + G_STRLOC, (int) i, total, connected_clts, + cur_idle, max_idle_conns); + + if (cur_idle || total <= max_idle_conns) { + break; + } + } + if (i == group->nslaves) { + backend = NULL; + } + return backend; +} + +void network_group_get_slave_names(network_group_t *group, GString *slaves) +{ + int i; + for (i = 0; i < group->nslaves; ++i) { + network_backend_t *b = group->slaves[i]; + g_string_append(slaves, b->addr->name->str); + g_string_append_c(slaves, ' '); + } +} + +int network_backends_idle_conns(network_backends_t *bs) +{ + int sum = 0; + int count = network_backends_count(bs); + int i; + for (i = 0; i < count; i++) { + network_backend_t *b = network_backends_get(bs, i); + int pooled = network_connection_pool_total_conns_count(b->pool); + sum += pooled; + } + return sum; +} + +int network_backends_used_conns(network_backends_t *bs) +{ + int sum = 0; + int count = network_backends_count(bs); + int i; + for (i = 0; i < count; i++) { + network_backend_t *b = network_backends_get(bs, i); + int in_use = b->connected_clients; + sum += in_use; + } + return sum; +} + diff --git a/src/network-backend.h b/src/network-backend.h new file mode 100644 index 0000000..3c6d08d --- /dev/null +++ b/src/network-backend.h @@ -0,0 +1,135 @@ +#ifndef _BACKEND_H_ +#define _BACKEND_H_ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "network-conn-pool.h" +#include "chassis-mainloop.h" + +#include "network-exports.h" +#ifdef HAVE_OPENSSL +#include +#endif +typedef enum { + BACKEND_STATE_UNKNOWN, + BACKEND_STATE_UP, + BACKEND_STATE_DOWN, + BACKEND_STATE_MAINTAINING, + BACKEND_STATE_DELETED, +} backend_state_t; + +typedef enum { + BACKEND_TYPE_UNKNOWN, + BACKEND_TYPE_RW, + BACKEND_TYPE_RO, +} backend_type_t; + +typedef enum { + BACKEND_ALGO_ROUND_ROBIN, + BACKEND_ALGO_RANDOM, + BACKEND_ALGO_FIRST, +} backend_algo_t; + +typedef struct backend_config { + GString *default_username; + GString *default_db; + + int max_conn_pool; + int mid_conn_pool; + + gchar charset; + +} backend_config; + +typedef struct { + network_address *addr; + GString *server_group; + GString *address; /* original address, might be domain name or ip */ + + backend_state_t state; /**< UP or DOWN */ + backend_type_t type; /**< ReadWrite or ReadOnly */ + + GTimeVal state_since; /**< timestamp of the last state-change */ + + network_connection_pool *pool; /**< the pool of open connections */ + + /**< number of open connections to this backend for SQF */ + int connected_clients; + + /**< the UUID of the backend */ + GString *uuid; + + backend_config *config; + GPtrArray *challenges; + int chal_ndx; + time_t last_check_time; + int slave_delay_msec; /* valid if this is a ReadOnly slave */ +} network_backend_t; + +NETWORK_API network_backend_t *network_backend_new(); +NETWORK_API void network_backend_free(network_backend_t *b); +NETWORK_API int network_backend_conns_count(network_backend_t *b); +NETWORK_API int network_backend_init_extra(network_backend_t *b, chassis *chas); +void network_backend_save_challenge(network_backend_t *b, + const network_mysqld_auth_challenge *); +network_mysqld_auth_challenge *network_backend_get_challenge(network_backend_t *b); + +typedef struct { + unsigned int ro_server_num; + unsigned int read_count; + GPtrArray *backends; +#ifdef HAVE_OPENSSL + RSA *rsa; +#endif + /* GHashTable *ip_table; */ + GTimeVal backend_last_check; + GPtrArray *groups; /* GPtrArray */ +} network_backends_t; + +NETWORK_API network_backends_t *network_backends_new(); +NETWORK_API void network_backends_free(network_backends_t *); +NETWORK_API int network_backends_add(network_backends_t *, const gchar *, + backend_type_t , backend_state_t, void *); +NETWORK_API int network_backends_remove(network_backends_t *bs, guint index); +NETWORK_API int network_backends_check(network_backends_t *bs); +NETWORK_API int network_backends_modify(network_backends_t *, guint , backend_type_t, backend_state_t); +NETWORK_API network_backend_t *network_backends_get(network_backends_t *bs, guint ndx); +NETWORK_API guint network_backends_count(network_backends_t *bs); +NETWORK_API gboolean network_backends_load_user_profile(network_backends_t *, chassis *); + +NETWORK_API gboolean network_backends_load_config(network_backends_t *, chassis *); + +/* get backend index by ip:port string */ +int network_backends_find_address(network_backends_t *bs, const char *); + +network_mysqld_auth_challenge * +network_backends_get_challenge(network_backends_t *b, int back_ndx); + +#define MAX_GROUP_SLAVES 4 + +typedef struct network_group_t { + GString *name; + network_backend_t *master; + network_backend_t *slaves[MAX_GROUP_SLAVES]; + int nslaves; + unsigned int slave_visit_cnt; +} network_group_t; + +network_group_t *network_backends_get_group(network_backends_t *, const GString *name); + +/* round-robin pick a slave from group */ +network_backend_t *network_group_pick_slave_backend(network_group_t *); + +void network_group_get_slave_names(network_group_t *, GString *); + +int network_backends_get_ro_ndx(network_backends_t *, backend_algo_t); + +int network_backends_get_rw_ndx(network_backends_t *); + +int network_backends_idle_conns(network_backends_t *); +int network_backends_used_conns(network_backends_t *); + +#endif /* _BACKEND_H_ */ + diff --git a/src/network-compress.c b/src/network-compress.c new file mode 100644 index 0000000..62440f7 --- /dev/null +++ b/src/network-compress.c @@ -0,0 +1,106 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include "network-compress.h" + +#define CHUNK 16384 + +int cetus_compress_init(z_stream *strm) +{ + /* allocate deflate state */ + strm->zalloc = Z_NULL; + strm->zfree = Z_NULL; + strm->opaque = Z_NULL; + + int level = Z_DEFAULT_COMPRESSION; + + return deflateInit(strm, level); +} + +void cetus_compress_end(z_stream *strm) +{ + (void) deflateEnd(strm); +} + +int cetus_compress(z_stream *strm, GString *dst, char *src, int src_len, int end) +{ + int flush; + unsigned char out[CHUNK]; + + strm->avail_in = src_len; + flush = end ? Z_FINISH : Z_NO_FLUSH; + strm->next_in = (Bytef *) src; + + /* run deflate() on input until output buffer not full, finish + compression if all of source has been read in */ + do { + strm->avail_out = CHUNK; + strm->next_out = out; + deflate(strm, flush); /* no bad return value */ + unsigned int have = CHUNK - strm->avail_out; + if (have > 0) { + g_string_append_len(dst, (const gchar *) out, have); + } + + } while (strm->avail_out == 0); + + return Z_OK; +} + +static int cetus_uncompress_init(z_stream *strm) +{ + /* allocate inflate state */ + strm->zalloc = Z_NULL; + strm->zfree = Z_NULL; + strm->opaque = Z_NULL; + strm->avail_in = 0; + strm->next_in = Z_NULL; + + return inflateInit(strm); +} + +static void cetus_uncompress_end(z_stream *strm) +{ + (void) inflateEnd(strm); +} + +int cetus_uncompress(GString *uncompressed_packet, unsigned char *src, int len) +{ + int ret; + z_stream strm; + unsigned char out[CHUNK]; + + cetus_uncompress_init(&strm); + + /* decompress until deflate stream ends or end of file */ + strm.avail_in = len; + strm.next_in = src; + + /* run inflate() on input until output buffer not full */ + do { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = inflate(&strm, Z_NO_FLUSH); + switch (ret) { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; /* and fall through */ + case Z_DATA_ERROR: + case Z_MEM_ERROR: + (void)inflateEnd(&strm); + return ret; + } + unsigned int have = CHUNK - strm.avail_out; + + g_string_append_len(uncompressed_packet, (const gchar *) out, have); + + } while (strm.avail_out == 0); + + cetus_uncompress_end(&strm); + + return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR; +} + diff --git a/src/network-compress.h b/src/network-compress.h new file mode 100644 index 0000000..7226fa4 --- /dev/null +++ b/src/network-compress.h @@ -0,0 +1,22 @@ +#ifndef _NETWORK_COMPRESS_H_ +#define _NETWORK_COMPRESS_H_ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include "network-exports.h" + +#define MIN_COMPRESS_LENGTH 50 + +NETWORK_API int cetus_uncompress(GString *uncompressed_packet, unsigned char *src, int len); +NETWORK_API int cetus_compress(z_stream *strm, GString *dst, char *src, int src_len, int end); +NETWORK_API void cetus_compress_end(z_stream *strm); +NETWORK_API int cetus_compress_init(z_stream *strm); + +#endif diff --git a/src/network-conn-pool-wrap.c b/src/network-conn-pool-wrap.c new file mode 100644 index 0000000..dbbc53a --- /dev/null +++ b/src/network-conn-pool-wrap.c @@ -0,0 +1,369 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_SYS_FILIO_H +#include +#endif + +#include + +#include +#include +#include +#define ioctlsocket ioctl + +#include + +#include "glib-ext.h" + +#include "network-mysqld.h" +#include "network-mysqld-packet.h" +#include "chassis-event.h" + +#include "network-conn-pool.h" +#include "network-conn-pool-wrap.h" +#include "cetus-util.h" + +/** + * handle the events of a idling server connection in the pool + * + * make sure we know about connection close from the server side + * - wait_timeout + */ +static void network_mysqld_con_idle_handle(int event_fd, short events, void *user_data) { + network_connection_pool_entry *pool_entry = user_data; + network_connection_pool *pool = pool_entry->pool; + + if (events == EV_READ) { + int b = -1; + + /** + * @todo we have to handle the case that the server really sent use something + * up to now we just ignore it + */ + if (ioctlsocket(event_fd, FIONREAD, &b)) { + g_critical("ioctl(%d, FIONREAD) failed: %s", event_fd, + g_strerror(errno)); + } else if (b != 0) { + g_critical("ioctl(%d, FIONREAD) said something to read, oops: %d", + event_fd, b); + } else { + /* the server decided the close the connection (wait_timeout, crash, ... ) + * + * remove us from the connection pool and close the connection */ + + network_connection_pool_remove(pool, pool_entry); + if (pool->srv) { + chassis *srv = pool->srv; + srv->complement_conn_cnt++; + } + + g_message("%s:the server decided the close the connection", G_STRLOC); + } + } +} + +int network_pool_add_idle_conn(network_connection_pool *pool, chassis *srv, network_socket *server) { + network_connection_pool_entry *pool_entry = NULL; + pool_entry = network_connection_pool_add(pool, server); + event_set(&(server->event), server->fd, EV_READ, + network_mysqld_con_idle_handle, pool_entry); + g_debug("%s: ev:%p add network_mysqld_con_idle_handle for server:%p, fd:%d", + G_STRLOC, &(server->event), server, server->fd); + chassis_event_add(srv, &(server->event)); + return 0; +} + +/** + * move the con->server into connection pool and disconnect the + * proxy from its backend *only RW-edition + */ +int network_pool_add_conn(network_mysqld_con *con, int is_swap) { + proxy_plugin_con_t *st = con->plugin_con_state; + chassis *srv = con->srv; + chassis_private *g = srv->priv; + + /* con-server is already disconnected, got out */ + if (!con->server) return -1; + + if (!con->server->response) { + g_warning("%s: server response is null:%p", G_STRLOC, con); + return -1; + } + + if (st->backend == NULL) { + g_warning("%s: st backend is null:%p", G_STRLOC, con); + return -1; + } + + if (con->prepare_stmt_count > 0) { + g_debug("%s: con valid_prepare_stmt_cnt:%d for con:%p", + G_STRLOC, con->prepare_stmt_count, con); + return -1; + } + + if (con->is_in_sess_context) { + if (st->backend->type == BACKEND_TYPE_RW) { + g_message("%s: transact feature is changed:%p", + G_STRLOC, con); + return -1; + } else { + g_message("%s: now transact feature is changed, orig is read server:%p", + G_STRLOC, con); + } + } + + if (con->server->is_in_sess_context) { + g_message("%s: server is in sess context true:%p", + G_STRLOC, con); + return -1; + } + + gboolean to_be_put_to_pool = TRUE; + + if (!is_swap && con->servers == NULL) { + if (con->srv->is_reduce_conns) { + if (network_conn_pool_do_reduce_conns_verdict(st->backend->pool, + st->backend->connected_clients)) + { + to_be_put_to_pool = FALSE; + } + } + } + + if (to_be_put_to_pool == FALSE) { + if (con->server->recv_queue->chunks->length > 0) { + g_critical("%s: recv queue length :%d, state:%s", + G_STRLOC, con->server->recv_queue->chunks->length, + network_mysqld_con_st_name(con->state)); + } + + GString *packet; + while ((packet = g_queue_pop_head(con->server->recv_queue->chunks))) { + g_string_free(packet, TRUE); + } + + st->backend->connected_clients--; + + network_socket_free(con->server); + + st->backend = NULL; + st->backend_ndx = -1; + con->server = NULL; + + return -1; + } + + con->server->is_authed = 1; + network_connection_pool_entry *pool_entry = NULL; + + if (con->servers != NULL) { + int i, checked = 0; + network_socket *server; + network_backend_t *backend; + + for (i = 0; i < con->servers->len; i++) { + + if (st->backend_ndx_array == NULL) { + g_message("%s: st backend ndx array is null:%p", G_STRLOC, con); + } else { + if (st->backend_ndx_array[i] == 0) { + continue; + } + + int index = st->backend_ndx_array[i] - 1; + server = g_ptr_array_index(con->servers, index); + backend = network_backends_get(g->backends, i); + CHECK_PENDING_EVENT(&(server->event)); + + g_debug("%s: add conn fd:%d to pool:%p ", G_STRLOC, server->fd, backend->pool); + server->is_multi_stmt_set = con->client->is_multi_stmt_set; + pool_entry = network_connection_pool_add(backend->pool, server); + event_set(&(server->event), server->fd, EV_READ, + network_mysqld_con_idle_handle, pool_entry); + g_debug("%s: ev:%p add network_mysqld_con_idle_handle for server:%p, fd:%d", + G_STRLOC, &(server->event), server, server->fd); + + chassis_event_add(con->srv, &(server->event)); + + backend->connected_clients--; + g_debug("%s, con:%p, backend ndx:%d:connected_clients sub, clients:%d", + G_STRLOC, con, st->backend_ndx_array[i], backend->connected_clients); + checked++; + if (checked >= con->servers->len) { + break; + } + } + } + + g_ptr_array_free(con->servers, TRUE); + con->servers = NULL; + + if (st->backend_ndx_array) { + g_free(st->backend_ndx_array); + st->backend_ndx_array = NULL; + } + + } else { + con->is_prepared = 0; + con->prepare_stmt_count = 0; + g_debug("%s: con:%p, set prepare_stmt_count 0", G_STRLOC, con); + CHECK_PENDING_EVENT(&(con->server->event)); + + g_debug("%s: add conn fd:%d to pool:%p", G_STRLOC, con->server->fd, + st->backend->pool); + con->server->is_multi_stmt_set = con->client->is_multi_stmt_set; + /* insert the server socket into the connection pool */ + pool_entry = network_connection_pool_add(st->backend->pool, con->server); + + event_set(&(con->server->event), con->server->fd, EV_READ, + network_mysqld_con_idle_handle, pool_entry); + + g_debug("%s: ev:%p add network_mysqld_con_idle_handle for server:%p, fd:%d", + G_STRLOC, &(con->server->event), con->server, con->server->fd); + + chassis_event_add(con->srv, &(con->server->event)); + + st->backend->connected_clients--; + g_debug("%s, con:%p, backend ndx:%d:connected_clients sub, clients:%d", + G_STRLOC, con, st->backend_ndx, st->backend->connected_clients); + } + + st->backend = NULL; + st->backend_ndx = -1; + + con->server = NULL; + + return 0; +} + + +void mysqld_con_reserved_connections_add(network_mysqld_con *con, + network_socket *sock, int backend_idx) +{ + proxy_plugin_con_t *st = con->plugin_con_state; + if (st->backend_ndx_array == NULL) { + st->backend_ndx_array = g_new0(short, MAX_SERVER_NUM); + st->backend_ndx_array[st->backend_ndx] = 1; /* current sock index = 0 */ + } + + if (con->servers == NULL) { + con->servers = g_ptr_array_new(); + g_ptr_array_add(con->servers, con->server); /* current sock index = 0 */ + con->multiple_server_mode = 1; /* TODO: redundant var */ + } + g_ptr_array_add(con->servers, sock); + st->backend_ndx_array[backend_idx] = con->servers->len; +} + + +network_socket *mysqld_con_reserved_connections_get(network_mysqld_con *con, int backend_idx) +{ + proxy_plugin_con_t *st = con->plugin_con_state; + if (con->servers) { + int conn_idx = st->backend_ndx_array[backend_idx]; + if (conn_idx > 0) { + conn_idx -= 1; + network_socket *sock = g_ptr_array_index(con->servers, conn_idx); + return sock; + } + } + return NULL; +} + +/** + * swap the server connection with a connection from + * the connection pool , only RW-edition + * + * we can only switch backends if we have an authed connection in the pool. + * + * @return NULL if swapping failed + * the new backend on success + */ +network_socket *network_connection_pool_swap(network_mysqld_con *con, int backend_ndx) { + network_backend_t *backend = NULL; + proxy_plugin_con_t *st = con->plugin_con_state; + chassis_private *g = con->srv->priv; + + /* + * we can only change to another backend if the backend is already + * in the connection pool and connected + */ + backend = network_backends_get(g->backends, backend_ndx); + if (!backend) return NULL; + + g_debug(G_STRLOC ": user: %s", + con->client->response ? "nil" : con->client->response->username->str); + + g_debug("%s: check server switch for conn:%p, prep_stmt_cnt:%d, orig ndx:%d, now:%d", + G_STRLOC, con, con->prepare_stmt_count, st->backend_ndx, backend_ndx); + /** + * TODO only valid for successful prepare statements, not valid for data partition + */ + gboolean server_switch_need_add = FALSE; + if (st->backend_ndx != -1 && con->prepare_stmt_count > 0 && + st->backend_ndx != backend_ndx) + { + if (backend->type == BACKEND_TYPE_RW || con->use_slave_forced) { + server_switch_need_add = TRUE; + g_debug("%s: server switch is true", G_STRLOC); + } else { + if (backend->type == BACKEND_TYPE_RO) { + g_message("%s: use orig server", G_STRLOC); + return NULL; + } + } + /* first try reserved connections, before query the pool */ + network_socket *sock = mysqld_con_reserved_connections_get(con, backend_ndx); + if (sock) { + return sock; + } + } + + /** + * get a connection from the pool which matches our basic requirements + * - username has to match + */ + int is_robbed = 0; + GString empty_name = { "", 0, 0 }; + GString *name = con->client->response ? con->client->response->username : &empty_name; + network_socket *sock = network_connection_pool_get(backend->pool, name, &is_robbed); + if (sock == NULL) { + if (con->server) { + if (network_pool_add_conn(con, 1) != 0) { + g_warning("%s: move the curr conn back into the pool failed", G_STRLOC); + return NULL; + } + } + st->backend_ndx = -1; + st->backend = NULL; + con->server = NULL; + return NULL; + } + con->rob_other_conn = is_robbed; + + if (server_switch_need_add) { + mysqld_con_reserved_connections_add(con, sock, backend_ndx); + } else { + if (con->server) { + if (network_pool_add_conn(con, 1) != 0) { + g_debug("%s: take and move the current backend into the pool failed", G_STRLOC); + return NULL; + } else { + g_debug("%s: take and move the current backend into the pool", G_STRLOC); + } + } + } + + /* connect to the new backend */ + st->backend = backend; + st->backend->connected_clients++; + st->backend_ndx = backend_ndx; + + g_debug("%s, con:%p, backend ndx:%d:connected_clients add, clients:%d, sock:%p", + G_STRLOC, con, backend_ndx, st->backend->connected_clients, sock); + + return sock; +} + diff --git a/src/network-conn-pool-wrap.h b/src/network-conn-pool-wrap.h new file mode 100644 index 0000000..507badd --- /dev/null +++ b/src/network-conn-pool-wrap.h @@ -0,0 +1,13 @@ +#ifndef __NETWORK_CONN_POOL_LUA_H__ +#define __NETWORK_CONN_POOL_LUA_H__ + +#include "network-socket.h" +#include "network-mysqld.h" + +#include "network-exports.h" + +NETWORK_API int network_pool_add_conn(network_mysqld_con *con, int is_swap); +NETWORK_API int network_pool_add_idle_conn(network_connection_pool *pool, chassis *srv, network_socket *server); +NETWORK_API network_socket *network_connection_pool_swap(network_mysqld_con *con, int backend_ndx); + +#endif diff --git a/src/network-conn-pool.c b/src/network-conn-pool.c new file mode 100644 index 0000000..2bdedfa --- /dev/null +++ b/src/network-conn-pool.c @@ -0,0 +1,308 @@ +#include + +#include "network-conn-pool.h" +#include "network-mysqld-packet.h" +#include "glib-ext.h" +#include "sys-pedantic.h" + +/** @file + * connection pools + * + * in the pool we manage idle connections + * - keep them up as long as possible + * - make sure we don't run out of seconds + * - if the client is authed, we have to pick connection with the same user + * - ... + */ + +/** + * create a empty connection pool entry + * + * @return a connection pool entry + */ +network_connection_pool_entry *network_connection_pool_entry_new(void) { + network_connection_pool_entry *e; + + e = g_new0(network_connection_pool_entry, 1); + + return e; +} + +/** + * free a conn pool entry + * + * @param e the pool entry to free + * @param free_sock if true, the attached server-socket will be freed too + */ +void network_connection_pool_entry_free(network_connection_pool_entry *e, + gboolean free_sock) +{ + if (!e) return; + + if (e->sock && free_sock) { + network_socket *sock = e->sock; + + g_debug("%s:event del, ev:%p",G_STRLOC, &(sock->event)); + event_del(&(sock->event)); + network_socket_free(sock); + } + + g_free(e); +} + +/** + * free all pool entries of the queue + * + * used as GDestroyFunc in the user-hash of the pool + * + * @param q a GQueue to free + * + * @see network_connection_pool_new + * @see GDestroyFunc + */ +static void g_queue_free_all(gpointer q) { + GQueue *queue = q; + network_connection_pool_entry *entry; + + while ((entry = g_queue_pop_head(queue))) { + network_connection_pool_entry_free(entry, TRUE); + } + + g_queue_free(queue); +} + +/** + * init a connection pool + */ +network_connection_pool *network_connection_pool_new(void) { + network_connection_pool *pool; + + pool = g_new0(network_connection_pool, 1); + + pool->max_idle_connections = 20; + pool->mid_idle_connections = 10; + pool->min_idle_connections = 2; + pool->cur_idle_connections = 0; + pool->users = g_hash_table_new_full(g_hash_table_string_hash, + g_hash_table_string_equal, g_hash_table_string_free, + g_queue_free_all); + + return pool; +} + +/** + * free all entries of the pool + * + */ +void network_connection_pool_free(network_connection_pool *pool) { + if (!pool) return; + + g_hash_table_foreach_remove(pool->users, g_hash_table_true, NULL); + + g_hash_table_destroy(pool->users); + + g_free(pool); +} + +/** + * find the entry which has more than max_idle connections idling + * + * @return TRUE for the first entry having more than _user_data idling connections + * @see network_connection_pool_get_conns + */ +static gboolean +find_idle_conns(gpointer UNUSED_PARAM(_key), gpointer _val, gpointer _user_data) +{ + guint idle_conns_threshold = *(gint *)_user_data; + GQueue *conns = _val; + + g_debug("%s: conns length:%d, idle_conns_threshold:%d", G_STRLOC, + conns->length, idle_conns_threshold); + return (conns->length > idle_conns_threshold); +} + +GQueue *network_connection_pool_get_conns(network_connection_pool *pool, GString *username, int *is_robbed) +{ + GQueue *conns = NULL; + + if (username && username->len > 0) { + conns = g_hash_table_lookup(pool->users, username); + /** + * if we know this use, return a authed connection + */ + g_debug("%s: get user-specific idling connection for '%s' -> %p", + G_STRLOC, username->str, conns); + if (conns && conns->length > 0) { + return conns; + } + } + + /** + * we don't have a entry yet, check the others if we have more than + * min_idle waiting + */ + + conns = g_hash_table_find(pool->users, find_idle_conns, &(pool->min_idle_connections)); + + g_debug("%s: (get_conns) try to find max-idling conns for user '%s' -> %p", + G_STRLOC, username ? username->str : "", conns); + + if (conns != NULL && is_robbed) { + *is_robbed = 1; + } + + return conns; +} + +/** + * get a connection from the pool + * + * make sure we have at least for each user + * if we have more, reuse a connect to reauth it to another user + * + * @param pool connection pool to get the connection from + * @param username (optional) name of the auth connection + * @param default_db (unused) unused name of the default-db + */ +network_socket *network_connection_pool_get(network_connection_pool *pool, + GString *username, int *is_robbed) +{ + network_connection_pool_entry *entry = NULL; + GQueue *conns = network_connection_pool_get_conns(pool, username, is_robbed); + + if (conns) { + if (conns->length > 0) { + entry = g_queue_pop_head(conns); + g_debug("%s: (get) entry for user '%s' -> %p, now:%ld", + G_STRLOC, username ? username->str : "", entry, time(0)); + } else { + g_debug("%s: conns length is zero for user '%s'", + G_STRLOC, username ? username->str : ""); + } + } else { + g_message("%s: conns is null for user '%s'", G_STRLOC, username ? username->str : "" ); + } + + if (!entry) { + g_debug("%s: (get) no entry for user '%s' -> %p", G_STRLOC, + username ? username->str : "", conns); + return NULL; + } + network_socket *sock = entry->sock; + + if (sock->recv_queue->chunks->length > 0) { + g_warning("%s: server recv queue not empty", G_STRLOC); + } + + g_debug("%s: recv queue length:%d, sock:%p", + G_STRLOC, sock->recv_queue->chunks->length, sock); + + network_connection_pool_entry_free(entry, FALSE); + + g_debug("%s:event del, ev:%p",G_STRLOC, &(sock->event)); + /* remove the idle handler from the socket */ + event_del(&(sock->event)); + + g_debug("%s: (get) got socket for user '%s' -> %p, charset:%s", G_STRLOC, + username ? username->str : "", sock, sock->charset->str); + + if (sock->is_in_sess_context) { + g_message("%s: conn is in sess context for user:'%s'", G_STRLOC, username ? username->str : "" ); + } + + pool->cur_idle_connections--; + + return sock; +} + +/** + * add a connection to the connection pool + */ +network_connection_pool_entry *network_connection_pool_add(network_connection_pool *pool, + network_socket *sock) +{ + if (!g_queue_is_empty(sock->recv_queue->chunks)) { + g_warning("%s: server recv queue not empty", G_STRLOC); + } + + network_connection_pool_entry *entry; + entry = network_connection_pool_entry_new(); + entry->sock = sock; + entry->pool = pool; + + sock->is_authed = 1; + + g_debug("%s: (add) adding socket to pool for user '%s' -> %p", + G_STRLOC, sock->response->username->str, sock); + + GQueue *conns = NULL; + if (NULL == (conns = g_hash_table_lookup(pool->users, + sock->response->username))) + { + conns = g_queue_new(); + g_hash_table_insert(pool->users, g_string_dup(sock->response->username), conns); + } + + g_queue_push_head(conns, entry); + + pool->cur_idle_connections++; + + return entry; +} + +/** + * remove the connection referenced by entry from the pool + */ +void network_connection_pool_remove(network_connection_pool *pool, + network_connection_pool_entry *entry) +{ + network_socket *sock = entry->sock; + GQueue *conns; + + if (NULL == (conns = g_hash_table_lookup(pool->users, sock->response->username))) { + return; + } + + network_connection_pool_entry_free(entry, TRUE); + + g_queue_remove(conns, entry); + + pool->cur_idle_connections--; +} + +gboolean +network_conn_pool_do_reduce_conns_verdict(network_connection_pool *pool, + int connected_clients) +{ + if (pool->cur_idle_connections > pool->mid_idle_connections) { + if (connected_clients < pool->cur_idle_connections) { + return TRUE; + } + } + + return FALSE; +} + +int network_connection_pool_total_conns_count(network_connection_pool *pool) +{ + GHashTable *users = pool->users; + int total = 0; + if (users != NULL) { + GHashTableIter iter; + GString *key; + GQueue *queue; + g_hash_table_iter_init(&iter, users); + /* count all users' pooled connections */ + while (g_hash_table_iter_next(&iter, (void **)&key, (void **)&queue)) { + total += queue->length; + } + } + + if (pool->cur_idle_connections != total) { + g_warning("%s: pool cur idle connections stat error:%d, total:%d", + G_STRLOC, pool->cur_idle_connections, total); + pool->cur_idle_connections = total; + } + + return total; +} diff --git a/src/network-conn-pool.h b/src/network-conn-pool.h new file mode 100644 index 0000000..72ae8f5 --- /dev/null +++ b/src/network-conn-pool.h @@ -0,0 +1,43 @@ +#ifndef _NETWORK_CONN_POOL_H_ +#define _NETWORK_CONN_POOL_H_ + +#include + +#include "network-socket.h" +#include "network-exports.h" + +typedef struct { + /** GHashTable> */ + GHashTable *users; + void *srv; + + int cur_idle_connections; + + guint max_idle_connections; + guint mid_idle_connections; + guint min_idle_connections; + +} network_connection_pool; + +typedef struct { + network_socket *sock; /** the idling socket */ + network_connection_pool *pool; /** a pointer back to the pool */ +} network_connection_pool_entry; + +NETWORK_API network_socket *network_connection_pool_get(network_connection_pool *pool, + GString *username, int *is_robbed); + +NETWORK_API network_connection_pool_entry * +network_connection_pool_add(network_connection_pool *, network_socket *); + +NETWORK_API void network_connection_pool_remove(network_connection_pool *pool, + network_connection_pool_entry *entry); +NETWORK_API GQueue *network_connection_pool_get_conns(network_connection_pool *, + GString *, int *); + +NETWORK_API network_connection_pool *network_connection_pool_new(void); +NETWORK_API void network_connection_pool_free(network_connection_pool *pool); +NETWORK_API int network_connection_pool_total_conns_count(network_connection_pool *pool); + +NETWORK_API gboolean network_conn_pool_do_reduce_conns_verdict(network_connection_pool *, int); +#endif diff --git a/src/network-exports.h b/src/network-exports.h new file mode 100644 index 0000000..e1138a4 --- /dev/null +++ b/src/network-exports.h @@ -0,0 +1,7 @@ +#ifndef _NETWORK_EXPORTS_H_ +#define _NETWORK_EXPORTS_H_ + +#define NETWORK_API extern + +#endif + diff --git a/src/network-injection.c b/src/network-injection.c new file mode 100644 index 0000000..3c46453 --- /dev/null +++ b/src/network-injection.c @@ -0,0 +1,93 @@ +#include + +#include "network-injection.h" + +#include "network-mysqld-proto.h" +#include "network-mysqld-packet.h" +#include "glib-ext.h" +#include "chassis-timings.h" +#include "cetus-util.h" + +#define TIME_DIFF_US(t2, t1) \ + ((t2.tv_sec - t1.tv_sec) * 1000000.0 + (t2.tv_usec - t1.tv_usec)) + + +/** + * Initialize an injection struct. + */ +injection *injection_new(int id, GString *query) { + injection *i; + + i = g_new0(injection, 1); + i->id = id; + i->query = query; + i->resultset_is_needed = FALSE; /* don't buffer the resultset */ + + return i; +} + +/** + * Free an injection struct + */ +void injection_free(injection *i) { + if (!i) return; + + if (i->query) { + g_string_free(i->query, TRUE); + i->query = NULL; + } + + g_free(i); +} + +network_injection_queue *network_injection_queue_new() { + return g_queue_new(); +} + +void network_injection_queue_free(network_injection_queue *q) { + if (!q) return; + + network_injection_queue_reset(q); + + g_queue_free(q); +} + +void network_injection_queue_reset(network_injection_queue *q) { + injection *inj; + if (!q) return; + + while ((inj = g_queue_pop_head(q))) { + injection_free(inj); + } +} + +void network_injection_queue_append(network_injection_queue *q, injection *inj) { + g_queue_push_tail(q, inj); +} + +void network_injection_queue_prepend(network_injection_queue *q, injection *inj) { + g_queue_push_head(q, inj); +} + +proxy_resultset_t *proxy_resultset_new() { + proxy_resultset_t *res; + + res = g_new0(proxy_resultset_t, 1); + + return res; +} + +/** + * Free a resultset struct + */ +void proxy_resultset_free(proxy_resultset_t *res) { + if (!res) return; + + if (res->fields) { + network_mysqld_proto_fielddefs_free(res->fields); + } + + g_free(res); +} + + diff --git a/src/network-injection.h b/src/network-injection.h new file mode 100644 index 0000000..8424040 --- /dev/null +++ b/src/network-injection.h @@ -0,0 +1,88 @@ +#ifndef _QUERY_HANDLING_H_ +#define _QUERY_HANDLING_H_ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "network-exports.h" + +typedef struct { + /** + * the content of the OK packet + */ + guint16 server_status; + guint16 warning_count; + guint64 affected_rows; + guint64 insert_id; + + gboolean was_resultset; /**< if set, affected_rows and insert_id are ignored */ + + /* + * < if set, the row data is binary encoded. we need the metadata to decode + */ + gboolean binary_encoded; + + /** + * MYSQLD_PACKET_OK or MYSQLD_PACKET_ERR + */ + guint8 query_status; +} query_status; + +typedef struct { + GString *query; + + /**< a unique id set by the scripts to map the query to a handler */ + int id; + + /* the userdata's need them */ + GQueue *result_queue; /**< the data to parse */ + + /**< summary information about the query status */ + query_status qstat; + + guint64 rows; + guint64 bytes; + + /**< flag to announce if we have to buffer the result for later processing */ + gboolean resultset_is_needed; +} injection; + +/** + * a injection_queue is GQueue for now + * + */ +typedef GQueue network_injection_queue; + +NETWORK_API network_injection_queue *network_injection_queue_new(void); +NETWORK_API void network_injection_queue_free(network_injection_queue *q); +NETWORK_API void network_injection_queue_reset(network_injection_queue *q); +NETWORK_API void network_injection_queue_prepend(network_injection_queue *q, injection *inj); +NETWORK_API void network_injection_queue_append(network_injection_queue *q, injection *inj); + +/** + * parsed result set + */ +typedef struct { + GQueue *result_queue; /**< where the packets are read from */ + + GPtrArray *fields; /**< the parsed fields */ + + GList *rows_chunk_head; /**< pointer to the EOF packet after the fields */ + GList *row; /**< the current row */ + + query_status qstat; /**< state of this query */ + + guint64 rows; + guint64 bytes; +} proxy_resultset_t; + +NETWORK_API injection *injection_new(int id, GString *query); +NETWORK_API void injection_free(injection *i); + +NETWORK_API proxy_resultset_t *proxy_resultset_new(); +NETWORK_API void proxy_resultset_free(proxy_resultset_t *res); + +#endif /* _QUERY_HANDLING_H_ */ diff --git a/src/network-mysqld-packet.c b/src/network-mysqld-packet.c new file mode 100644 index 0000000..2225d90 --- /dev/null +++ b/src/network-mysqld-packet.c @@ -0,0 +1,1774 @@ +/** + * codec's for the MySQL client protocol + */ + +#include +#include +#include + +#include "network-mysqld-packet.h" +#include "glib-ext.h" + +network_mysqld_com_query_result_t *network_mysqld_com_query_result_new() { + network_mysqld_com_query_result_t *com_query; + + com_query = g_new0(network_mysqld_com_query_result_t, 1); + com_query->state = PARSE_COM_QUERY_INIT; + /* + * can have 3 values: NULL for unknown, OK for a OK packet, + * ERR for a error-packet + */ + com_query->query_status = MYSQLD_PACKET_NULL; + + return com_query; +} + +void +network_mysqld_com_query_result_free(network_mysqld_com_query_result_t *udata) +{ + if (!udata) return; + + g_free(udata); +} + +/** + * @return -1 on error + * 0 on success and done + * 1 on success and need more + */ +int network_mysqld_proto_get_com_query_result(network_packet *packet, + network_mysqld_com_query_result_t *query, gboolean use_binary_row_data) +{ + int is_finished = 0; + guint8 status; + int err = 0; + network_mysqld_eof_packet_t *eof_packet; + network_mysqld_ok_packet_t *ok_packet; + + /** + * if we get a OK in the first packet there will be no result-set + */ + switch (query->state) { + case PARSE_COM_QUERY_INIT: + err = network_mysqld_proto_peek_int8(packet, &status); + if (err) break; + + switch (status) { + case MYSQLD_PACKET_ERR: + { + g_debug("%s: status error, query state:%d, status:%d, packet offset:%d", + G_STRLOC, query->state, status, (int) packet->offset); + query->query_status = MYSQLD_PACKET_ERR; + is_finished = 1; + break; + } + case MYSQLD_PACKET_OK: /* e.g. DELETE FROM tbl */ + query->query_status = MYSQLD_PACKET_OK; + + ok_packet = network_mysqld_ok_packet_new(); + + err = network_mysqld_proto_get_ok_packet(packet, ok_packet); + + if (!err) { + if (ok_packet->server_status & + SERVER_MORE_RESULTS_EXISTS) + { + + } else { + is_finished = 1; + } + + query->server_status = ok_packet->server_status; + g_debug("%s: server status in ok packet, got: %d", + G_STRLOC, + ok_packet->server_status); + query->warning_count = ok_packet->warnings; + query->affected_rows = ok_packet->affected_rows; + query->insert_id = ok_packet->insert_id; + query->was_resultset = 0; + query->binary_encoded = use_binary_row_data; + } + + network_mysqld_ok_packet_free(ok_packet); + break; + case MYSQLD_PACKET_NULL: + /* OH NO, LOAD DATA INFILE :) */ + query->state = PARSE_COM_QUERY_LOCAL_INFILE_DATA; + is_finished = 1; + break; + case MYSQLD_PACKET_EOF: + g_critical("%s: COM_QUERY should not be (EOF), got: 0x%02x", + G_STRLOC, + status); + err = 1; + break; + default: + query->query_status = MYSQLD_PACKET_OK; + /* looks like a result */ + query->state = PARSE_COM_QUERY_FIELD; + break; + } + break; + case PARSE_COM_QUERY_FIELD: + + err = err || network_mysqld_proto_peek_int8(packet, &status); + if (err) break; + + switch (status) { + case MYSQLD_PACKET_ERR: + case MYSQLD_PACKET_OK: + case MYSQLD_PACKET_NULL: + g_critical("%s: COM_QUERY !=(OK|NULL|ERR), got: 0x%02x", + G_STRLOC, + status); + + err = 1; + + break; + case MYSQLD_PACKET_EOF: + /** + * in 5.0 we have CURSORs + * + * COM_STMT_EXECUTE would have _CURSOR_EXISTS set in + * the EOF and no resultset + * COM_STMT_FETCH would be executed afterwards to + * fetch the rows from the cursor + * + * Other commands may have that flag set too, + * with no special meaning + * + */ + if (packet->data->len == 9) { + eof_packet = network_mysqld_eof_packet_new(); + + err = network_mysqld_proto_get_eof_packet(packet, + eof_packet); + + if (!err) { +#if MYSQL_VERSION_ID >= 50000 + /* 5.5 may send a SERVER_MORE_RESULTS_EXISTS as part of the first + * EOF together with SERVER_STATUS_CURSOR_EXISTS. In that case, + * we aren't finished. (#61998) + * + * Only if _CURSOR_EXISTS is set alone AND this is COM_STMT_EXECUTE, + * we have a field-definition-only resultset + * + * CURSOR_EXISTS indications that COM_STMT_FETCH should be used to + * fetch data for this cursor, but can only be don't if we have + * a prepared statement + */ + if (use_binary_row_data && + eof_packet->server_status & SERVER_STATUS_CURSOR_EXISTS && + !(eof_packet->server_status & SERVER_MORE_RESULTS_EXISTS)) { + is_finished = 1; + g_message("%s: is_finished here without PARSE_COM_QUERY_RESULT", + G_STRLOC); + } else { + query->state = PARSE_COM_QUERY_RESULT; + } +#else + query->state = PARSE_COM_QUERY_RESULT; +#endif + g_debug("%s: set query state:%d for parse.data:%p", + G_STRLOC, query->state, query); + + /* track the server_status of the 1st EOF packet */ + query->server_status = eof_packet->server_status; + g_debug("%s: server status in eof packet, got: %d", + G_STRLOC, + eof_packet->server_status); + } + + network_mysqld_eof_packet_free(eof_packet); + } else { + query->state = PARSE_COM_QUERY_RESULT; + g_debug("%s: set query state:%d for parse.data:%p", + G_STRLOC, query->state, query); + } + break; + default: + break; + } + break; + case PARSE_COM_QUERY_RESULT: + err = err || network_mysqld_proto_peek_int8(packet, &status); + if (err) break; + + switch (status) { + case MYSQLD_PACKET_EOF: + if (packet->data->len == 9) { + eof_packet = network_mysqld_eof_packet_new(); + + err = err || network_mysqld_proto_get_eof_packet(packet, eof_packet); + + if (!err) { + query->was_resultset = 1; + +#ifndef SERVER_PS_OUT_PARAMS +#define SERVER_PS_OUT_PARAMS 4096 +#endif + /** + * a PS_OUT_PARAMS is set if a COM_STMT_EXECUTE executes + * a CALL sp(?) where sp is a PROCEDURE with OUT params + * + * ... + * 05 00 00 12 fe 00 00 0a 10 -- end column-def + * (auto-commit, more-results, ps-out-params) + * ... + * 05 00 00 14 fe 00 00 02 00 -- end of rows (auto-commit), + * see the missing (more-results, ps-out-params) + * 07 00 00 15 00 00 00 02 00 00 00 -- OK for the CALL + * + * for all other resultsets we trust the status-flags of + * the 2nd EOF packet + */ + if (!(query->server_status & SERVER_PS_OUT_PARAMS)) { + query->server_status = eof_packet->server_status; + } + query->warning_count = eof_packet->warnings; + + if (query->server_status & SERVER_MORE_RESULTS_EXISTS) { + query->state = PARSE_COM_QUERY_INIT; + g_debug("%s: here query state:%d", G_STRLOC, query->state); + } else { + is_finished = 1; + g_debug("%s: here query state:%d", G_STRLOC, query->state); + } + } + + network_mysqld_eof_packet_free(eof_packet); + } + + break; + case MYSQLD_PACKET_ERR: + /* like + * + * EXPLAIN SELECT *FROM dual; returns an error + * + * EXPLAIN SELECT 1 FROM dual; returns a result-set + * */ + is_finished = 1; + break; + case MYSQLD_PACKET_OK: + case MYSQLD_PACKET_NULL: + default: + query->rows++; + query->bytes += packet->data->len; + break; + } + break; + case PARSE_COM_QUERY_LOCAL_INFILE_DATA: + /* we will receive a empty packet if we are done */ + if (packet->data->len == packet->offset) { + query->state = PARSE_COM_QUERY_LOCAL_INFILE_RESULT; + is_finished = 1; + } + break; + case PARSE_COM_QUERY_LOCAL_INFILE_RESULT: + err = err || network_mysqld_proto_get_int8(packet, &status); + if (err) break; + + switch (status) { + case MYSQLD_PACKET_OK: + is_finished = 1; + break; + case MYSQLD_PACKET_NULL: + case MYSQLD_PACKET_ERR: + case MYSQLD_PACKET_EOF: + default: + g_critical("%s: COM_QUERY,should be (OK), got: 0x%02x", + G_STRLOC, + status); + + err = 1; + + break; + } + + break; + } + + if (err) return -1; + + return is_finished; +} + + +network_mysqld_com_stmt_prep_result_t * +network_mysqld_com_stmt_prepare_result_new() +{ + network_mysqld_com_stmt_prep_result_t *udata; + + udata = g_new0(network_mysqld_com_stmt_prep_result_t, 1); + udata->first_packet = TRUE; + + return udata; +} + +void +network_mysqld_com_stmt_prepare_result_free(network_mysqld_com_stmt_prep_result_t *udata) { + if (!udata) return; + + g_free(udata); +} + +int network_mysqld_proto_get_com_stmt_prep_result( + network_packet *packet, + network_mysqld_com_stmt_prep_result_t *udata) +{ + guint8 status; + int is_finished = 0; + int err = 0; + + err = err || network_mysqld_proto_get_int8(packet, &status); + + if (udata->first_packet == 1) { + udata->first_packet = 0; + + switch (status) { + case MYSQLD_PACKET_OK: + g_assert(packet->data->len == 12 + NET_HEADER_SIZE); + + /* the header contains the number of EOFs we expect to see + * - no params -> 0 + * - params | fields -> 1 + * - params + fields -> 2 + */ + udata->want_eofs = 0; + + if (packet->data->str[NET_HEADER_SIZE + 5] != 0 || + packet->data->str[NET_HEADER_SIZE + 6] != 0) + { + udata->want_eofs++; + } + if (packet->data->str[NET_HEADER_SIZE + 7] != 0 || + packet->data->str[NET_HEADER_SIZE + 8] != 0) + { + udata->want_eofs++; + } + + if (udata->want_eofs == 0) { + is_finished = 1; + } + + g_debug("%s: want_eofs value:%d", + G_STRLOC, + udata->want_eofs); + udata->status = status; + break; + case MYSQLD_PACKET_ERR: + is_finished = 1; + g_message("%s: network_mysqld_proto_get_com_stmt_prep_rs get packet err:%d", + G_STRLOC, + status); + udata->status = status; + break; + default: + g_error("%s: COM_STMT_PREPARE should either get a (OK|ERR), got %02x", + G_STRLOC, + status); + break; + } + } else { + switch (status) { + case MYSQLD_PACKET_OK: + case MYSQLD_PACKET_NULL: + case MYSQLD_PACKET_ERR: + g_error("%s: COM_STMT_PREPARE should not be (OK|ERR|NULL), got: %02x", + G_STRLOC, + status); + break; + case MYSQLD_PACKET_EOF: + if (--udata->want_eofs == 0) { + is_finished = 1; + } + g_debug("%s: other want_eofs value:%d", + G_STRLOC, + udata->want_eofs); + break; + default: + break; + } + } + + if (err) return -1; + + return is_finished; +} + +network_mysqld_com_init_db_result_t *network_mysqld_com_init_db_result_new() { + network_mysqld_com_init_db_result_t *udata; + + udata = g_new0(network_mysqld_com_init_db_result_t, 1); + udata->db_name = NULL; + + return udata; +} + + +void +network_mysqld_com_init_db_result_free(network_mysqld_com_init_db_result_t *udata) +{ + if (udata->db_name) g_string_free(udata->db_name, TRUE); + + g_free(udata); +} + +int network_mysqld_com_init_db_result_track_state(network_packet *packet, + network_mysqld_com_init_db_result_t *udata) +{ + network_mysqld_proto_skip_network_header(packet); + if (network_mysqld_proto_skip(packet, 1) == -1) { + return -1; + } + + if (packet->offset != packet->data->len) { + udata->db_name = g_string_new(NULL); + + network_mysqld_proto_get_gstr_len(packet, + packet->data->len - packet->offset, udata->db_name); + } else { + if (udata->db_name) g_string_free(udata->db_name, TRUE); + udata->db_name = NULL; + } + + return 0; +} + +int network_mysqld_proto_get_com_init_db( + network_packet *packet, + network_mysqld_com_init_db_result_t *udata, + network_mysqld_con *con) +{ + guint8 status; + int is_finished; + int err = 0; + + /** + * in case we have a init-db statement we track the db-change on the server-side + * connection + */ + err = err || network_mysqld_proto_get_int8(packet, &status); + + g_debug("%s: COM_INIT_DB got %02x", + G_STRLOC, + status); + + switch (status) { + case MYSQLD_PACKET_ERR: + is_finished = 1; + if (udata->db_name && udata->db_name->len) { + g_message("%s: COM_INIT_DB failed, want db:%s, client default db still:%s", + G_STRLOC, udata->db_name->str, con->client->default_db->str); + } else { + g_message("%s: COM_INIT_DB failed, client default db still:%s", + G_STRLOC, con->client->default_db->str); + } + break; + case MYSQLD_PACKET_OK: + /** + * track the change of the init_db */ + if (con->server) g_string_truncate(con->server->default_db, 0); + + if (udata->db_name && udata->db_name->len) { + + g_string_truncate(con->client->default_db, 0); + if (con->server) { + g_string_append_len(con->server->default_db, + S(udata->db_name)); + g_debug("%s:set server default db:%s for con:%p", + G_STRLOC, con->server->default_db->str, con); + } + + g_string_append_len(con->client->default_db, + S(udata->db_name)); + g_debug("%s: COM_INIT_DB set default db success:%s", + G_STRLOC, con->client->default_db->str); + } else { + if (con->server) { + g_string_append_len(con->server->default_db, + S(con->client->default_db)); + } + } + + is_finished = 1; + break; + default: + g_critical("%s: COM_INIT_DB should be (ERR|OK), got %02x", + G_STRLOC, + status); + + return -1; + } + + if (err) return -1; + + return is_finished; +} + +/** + * init the tracking of the sub-states of the protocol + */ +int network_mysqld_con_command_states_init(network_mysqld_con *con, network_packet *packet) +{ + guint8 cmd; + int err = 0; + + err = err || network_mysqld_proto_skip_network_header(packet); + err = err || network_mysqld_proto_get_int8(packet, &cmd); + + if (err) return -1; + + con->parse.command = cmd; + + g_debug("%s: reset command:%d ", G_STRLOC, cmd); + packet->offset = 0; /* reset the offset again for the next functions */ + + /* init the parser for the commands */ + switch (con->parse.command) { + case COM_QUERY: + case COM_PROCESS_INFO: + case COM_STMT_EXECUTE: + con->parse.data = network_mysqld_com_query_result_new(); + con->parse.data_free = (GDestroyNotify)network_mysqld_com_query_result_free; + break; + case COM_STMT_PREPARE: + con->parse.data = network_mysqld_com_stmt_prepare_result_new(); + con->parse.data_free = (GDestroyNotify)network_mysqld_com_stmt_prepare_result_free; + break; + case COM_INIT_DB: + con->parse.data = network_mysqld_com_init_db_result_new(); + con->parse.data_free = (GDestroyNotify)network_mysqld_com_init_db_result_free; + + if (network_mysqld_com_init_db_result_track_state(packet, con->parse.data) != 0) { + return -1; + } + + break; + case COM_QUIT: + /* track COM_QUIT going to the server, to be able to tell if the server + * a) simply went away or + * b) closed the connection because the client asked it to + * If b) we should not print a message at the next EV_READ event from the server fd + */ + con->com_quit_seen = TRUE; + default: + break; + } + + return 0; +} + +/** + * @param packet the current packet that is passing by + * + * + * @return -1 on invalid packet, + * 0 need more packets, + * 1 for the last packet + */ +int network_mysqld_proto_get_query_result(network_packet *packet, network_mysqld_con *con) { + guint8 status; + int is_finished = 0; + int err = 0; + + err = err || network_mysqld_proto_skip_network_header(packet); + if (err) { + g_message("%s: skip header error, command:%d.", G_STRLOC, con->parse.command); + return -1; + } + + /* forward the response to the client */ + switch (con->parse.command) { + case COM_RESET_CONNECTION: + err = err || network_mysqld_proto_get_int8(packet, &status); + if (err) return -1; + + switch (status) { + case MYSQLD_PACKET_ERR: + con->is_changed_user_failed = 1; + is_finished = 1; + break; + case MYSQLD_PACKET_OK: + is_finished = 1; + break; + default: + g_debug_hexdump(G_STRLOC, S(packet->data)); + g_message("%s: got a 0x%02x packet for COM_[0%02x], expected only (ERR|EOF)", + G_STRLOC, + con->parse.command, + (guint8)status); + return -1; + } + + break; + + case COM_CHANGE_USER: + /** + * - OK + * - ERR + * - EOF for auth switch TODO + */ + err = err || network_mysqld_proto_get_int8(packet, &status); + if (err) return -1; + + switch (status) { + case MYSQLD_PACKET_ERR: + con->is_changed_user_failed = 1; + int offset = packet->offset; + packet->offset = NET_HEADER_SIZE; + network_mysqld_err_packet_t *err_packet; + err_packet = network_mysqld_err_packet_new(); + if (!network_mysqld_proto_get_err_packet(packet, err_packet)) { + g_warning("%s:error code:%d,errmsg:%s,sqlstate:%s", + G_STRLOC, (int) err_packet->errcode, + err_packet->errmsg->str, err_packet->sqlstate->str); + } else { + g_warning("%s: change user failed", G_STRLOC); + } + network_mysqld_err_packet_free(err_packet); + packet->offset = offset; + is_finished = 1; + break; + + case MYSQLD_PACKET_OK: + g_string_assign_len(con->server->response->username, + S(con->client->response->username)); + g_debug("%s: save username for server, con:%p", G_STRLOC, con); + + if (con->client->default_db->len > 0) { + g_string_truncate(con->server->default_db, 0); + g_string_append(con->server->default_db, con->client->default_db->str); + g_debug("%s:set server default db:%s for con:%p", + G_STRLOC, con->server->default_db->str, con); + } + is_finished = 1; + break; + case MYSQLD_PACKET_EOF: + /* TODO: + * - added extra states to the state-engine in network-mysqld.c to + * track the packets that are sent back and forth + * to switch the auth-method in COM_CHANGE_USER + */ + g_message("%s: COM_CHANGE_USER's auth-method-switch not supported.", + G_STRLOC); + return -1; + default: + g_debug_hexdump(G_STRLOC, S(packet->data)); + g_message("%s: got a 0x%02x packet for COM_[0%02x], expected only (ERR|OK)", + G_STRLOC, + con->parse.command, + (guint8)status); + return -1; + } + break; + case COM_INIT_DB: + g_debug("%s: COM_INIT_DB finish check.", G_STRLOC); + is_finished = network_mysqld_proto_get_com_init_db(packet, con->parse.data, con); + + break; + case COM_REFRESH: + case COM_STMT_RESET: + case COM_PING: + case COM_TIME: + case COM_REGISTER_SLAVE: + case COM_PROCESS_KILL: + err = err || network_mysqld_proto_get_int8(packet, &status); + if (err) return -1; + + switch (status) { + case MYSQLD_PACKET_ERR: + case MYSQLD_PACKET_OK: + is_finished = 1; + break; + default: + g_debug_hexdump(G_STRLOC, S(packet->data)); + g_message("%s: got a 0x%02x packet for COM_[0%02x],expected only (ERR|OK)", + G_STRLOC, + con->parse.command, + (guint8)status); + return -1; + } + break; + case COM_DEBUG: + case COM_SET_OPTION: + case COM_SHUTDOWN: + err = err || network_mysqld_proto_get_int8(packet, &status); + if (err) return -1; + + switch (status) { + case MYSQLD_PACKET_ERR: /* COM_DEBUG may not have the right permissions */ + case MYSQLD_PACKET_EOF: + is_finished = 1; + break; + default: + g_debug_hexdump(G_STRLOC, S(packet->data)); + g_message("%s: got a 0x%02x packet for COM_[0%02x], expected only (ERR|EOF)", + G_STRLOC, + con->parse.command, + (guint8)status); + return -1; + } + break; + + case COM_FIELD_LIST: + err = err || network_mysqld_proto_get_int8(packet, &status); + if (err) return -1; + + /* we transfer some data and wait for the EOF */ + switch (status) { + case MYSQLD_PACKET_ERR: + case MYSQLD_PACKET_EOF: + is_finished = 1; + break; + + case MYSQLD_PACKET_NULL: + case MYSQLD_PACKET_OK: + g_debug_hexdump(G_STRLOC, S(packet->data)); + g_message("%s: got a 0x%02x for COM_[0%02x], expected ERR, EOF or field data", + G_STRLOC, + con->parse.command, + (guint8)status); + return -1; + default: + break; + } + break; +#if MYSQL_VERSION_ID >= 50000 + case COM_STMT_FETCH: + /* */ + err = err || network_mysqld_proto_peek_int8(packet, &status); + if (err) return -1; + + switch (status) { + case MYSQLD_PACKET_EOF: { + network_mysqld_eof_packet_t *eof_packet = network_mysqld_eof_packet_new(); + + err = err || network_mysqld_proto_get_eof_packet(packet, eof_packet); + if (!err) { + if ((eof_packet->server_status & SERVER_STATUS_LAST_ROW_SENT) || + (eof_packet->server_status & SERVER_STATUS_CURSOR_EXISTS)) { + is_finished = 1; + } + } + + network_mysqld_eof_packet_free(eof_packet); + + break; + } + case MYSQLD_PACKET_ERR: + is_finished = 1; + break; + default: + break; + } + break; +#endif + case COM_QUIT: /* sometimes we get a packet before the connection closes */ + case COM_STATISTICS: + /* just one packet, no EOF */ + is_finished = 1; + + break; + case COM_STMT_PREPARE: + is_finished = network_mysqld_proto_get_com_stmt_prep_result(packet, + con->parse.data); + break; + case COM_STMT_EXECUTE: + /* COM_STMT_EXECUTE result packets are basically the same as COM_QUERY ones, + * the only difference is the encoding of the actual data - fields are in there, too. + */ + is_finished = network_mysqld_proto_get_com_query_result(packet, + con->parse.data, TRUE); + break; + case COM_PROCESS_INFO: + case COM_QUERY: + is_finished = network_mysqld_proto_get_com_query_result(packet, + con->parse.data, FALSE); + break; + case COM_BINLOG_DUMP: + /** + * the binlog-dump event stops, forward all packets as we see them + * and keep the command active + */ + is_finished = 1; + break; + default: + g_debug_hexdump(G_STRLOC, S(packet->data)); + g_message("%s: COM_(0x%02x) is not handled", + G_STRLOC, + con->parse.command); + err = 1; + break; + } + + if (err) return -1; + + return is_finished; +} + +int network_mysqld_proto_get_fielddef(network_packet *packet, + network_mysqld_proto_fielddef_t *field, guint32 capabilities) +{ + int err = 0; + + if (capabilities & CLIENT_PROTOCOL_41) { + guint16 field_charsetnr; + guint32 field_length; + guint8 field_type; + guint16 field_flags; + guint8 field_decimals; + + err = err || network_mysqld_proto_get_lenenc_str(packet, &field->catalog, NULL); + err = err || network_mysqld_proto_get_lenenc_str(packet, &field->db, NULL); + err = err || network_mysqld_proto_get_lenenc_str(packet, &field->table, NULL); + err = err || network_mysqld_proto_get_lenenc_str(packet, &field->org_table, NULL); + err = err || network_mysqld_proto_get_lenenc_str(packet, &field->name, NULL); + err = err || network_mysqld_proto_get_lenenc_str(packet, &field->org_name, NULL); + + err = err || network_mysqld_proto_skip(packet, 1); /* filler */ + + err = err || network_mysqld_proto_get_int16(packet, &field_charsetnr); + err = err || network_mysqld_proto_get_int32(packet, &field_length); + err = err || network_mysqld_proto_get_int8(packet, &field_type); + err = err || network_mysqld_proto_get_int16(packet, &field_flags); + err = err || network_mysqld_proto_get_int8(packet, &field_decimals); + + err = err || network_mysqld_proto_skip(packet, 2); /* filler */ + if (!err) { + field->charsetnr = field_charsetnr; + field->length = field_length; + field->type = field_type; + field->flags = field_flags; + field->decimals = field_decimals; + } + } else { + guint8 len = 0; + guint32 field_length; + guint8 field_type; + guint8 field_decimals; + + /* see protocol.cc Protocol::send_fields */ + + err = err || network_mysqld_proto_get_lenenc_str(packet, &field->table, NULL); + err = err || network_mysqld_proto_get_lenenc_str(packet, &field->name, NULL); + err = err || network_mysqld_proto_get_int8(packet, &len); + err = err || (len != 3); + err = err || network_mysqld_proto_get_int24(packet, &field_length); + err = err || network_mysqld_proto_get_int8(packet, &len); + err = err || (len != 1); + err = err || network_mysqld_proto_get_int8(packet, &field_type); + err = err || network_mysqld_proto_get_int8(packet, &len); + if (len == 3) { /* the CLIENT_LONG_FLAG is set */ + guint16 field_flags; + + err = err || network_mysqld_proto_get_int16(packet, &field_flags); + + if (!err) field->flags = field_flags; + } else if (len == 2) { + guint8 field_flags; + + err = err || network_mysqld_proto_get_int8(packet, &field_flags); + + if (!err) field->flags = field_flags; + } else { + err = -1; + } + err = err || network_mysqld_proto_get_int8(packet, &field_decimals); + + if (!err) { + field->charsetnr = 0x08 /* latin1_swedish_ci */; + field->length = field_length; + field->type = field_type; + field->decimals = field_decimals; + } + } + + return err ? -1 : 0; +} + +network_mysqld_ok_packet_t *network_mysqld_ok_packet_new() { + network_mysqld_ok_packet_t *ok_packet; + + ok_packet = g_new0(network_mysqld_ok_packet_t, 1); + + return ok_packet; +} + +void network_mysqld_ok_packet_free(network_mysqld_ok_packet_t *ok_packet) { + if (!ok_packet) return; + + g_free(ok_packet); +} + + +/** + * decode a OK packet from the network packet + */ +int network_mysqld_proto_get_ok_packet(network_packet *packet, + network_mysqld_ok_packet_t *ok_packet) +{ + guint8 field_count; + guint64 affected, insert_id; + guint16 server_status, warning_count = 0; + guint32 capabilities = CLIENT_PROTOCOL_41; + + int err = 0; + + err = err || network_mysqld_proto_get_int8(packet, &field_count); + if (err) return -1; + + if (field_count != 0) { + g_critical("%s: expected the first byte to be 0, got %d", + G_STRLOC, + field_count); + return -1; + } + + err = err || network_mysqld_proto_get_lenenc_int(packet, &affected); + err = err || network_mysqld_proto_get_lenenc_int(packet, &insert_id); + err = err || network_mysqld_proto_get_int16(packet, &server_status); + if (capabilities & CLIENT_PROTOCOL_41) { + err = err || network_mysqld_proto_get_int16(packet, &warning_count); + } + + if (!err) { + ok_packet->affected_rows = affected; + ok_packet->insert_id = insert_id; + ok_packet->server_status = server_status; + ok_packet->warnings = warning_count; + g_debug("%s: server status, got: %d", G_STRLOC, + ok_packet->server_status); + } + + return err ? -1 : 0; +} + +int network_mysqld_proto_append_ok_packet(GString *packet, + network_mysqld_ok_packet_t *ok_packet) +{ + guint32 capabilities = CLIENT_PROTOCOL_41; + + network_mysqld_proto_append_int8(packet, 0); /* no fields */ + network_mysqld_proto_append_lenenc_int(packet, ok_packet->affected_rows); + network_mysqld_proto_append_lenenc_int(packet, ok_packet->insert_id); + network_mysqld_proto_append_int16(packet, ok_packet->server_status); /* autocommit */ + if (capabilities & CLIENT_PROTOCOL_41) { + network_mysqld_proto_append_int16(packet, ok_packet->warnings); /* no warnings */ + } + + return 0; +} + +static network_mysqld_err_packet_t * +network_mysqld_err_packet_new_full(network_mysqld_protocol_t version) +{ + network_mysqld_err_packet_t *err_packet; + + err_packet = g_new0(network_mysqld_err_packet_t, 1); + err_packet->sqlstate = g_string_new(NULL); + err_packet->errmsg = g_string_new(NULL); + err_packet->version = version; + + return err_packet; +} + +network_mysqld_err_packet_t *network_mysqld_err_packet_new() { + return network_mysqld_err_packet_new_full(NETWORK_MYSQLD_PROTOCOL_VERSION_41); +} + +network_mysqld_err_packet_t *network_mysqld_err_packet_new_pre41() { + return network_mysqld_err_packet_new_full(NETWORK_MYSQLD_PROTOCOL_VERSION_PRE41); +} + +void network_mysqld_err_packet_free(network_mysqld_err_packet_t *err_packet) { + if (!err_packet) return; + + g_string_free(err_packet->sqlstate, TRUE); + g_string_free(err_packet->errmsg, TRUE); + + g_free(err_packet); +} + +/** + * decode an ERR packet from the network packet + */ +int network_mysqld_proto_get_err_packet(network_packet *packet, + network_mysqld_err_packet_t *err_packet) +{ + guint8 field_count, marker; + guint16 errcode; + gchar *sqlstate = NULL, *errmsg = NULL; + guint32 capabilities = CLIENT_PROTOCOL_41; + + int err = 0; + + err = err || network_mysqld_proto_get_int8(packet, &field_count); + if (err) return -1; + + if (field_count != MYSQLD_PACKET_ERR) { + g_critical("%s: expected the first byte to be 0xff, got %d", + G_STRLOC, + field_count); + return -1; + } + + err = err || network_mysqld_proto_get_int16(packet, &errcode); + if (capabilities & CLIENT_PROTOCOL_41) { + err = err || network_mysqld_proto_get_int8(packet, &marker); + err = err || (marker != '#'); + err = err || network_mysqld_proto_get_str_len(packet, &sqlstate, 5); + } + if (packet->offset < packet->data->len) { + err = err || network_mysqld_proto_get_str_len(packet, + &errmsg, packet->data->len - packet->offset); + } + + if (!err) { + err_packet->errcode = errcode; + if (errmsg) g_string_assign(err_packet->errmsg, errmsg); + g_string_assign(err_packet->sqlstate, sqlstate); + } + + if (sqlstate) g_free(sqlstate); + if (errmsg) g_free(errmsg); + + return err ? -1 : 0; +} + + + +/** + * create a ERR packet + * + * @note the sqlstate has to match the SQL standard. + * If no matching SQL state is known, leave it at NULL + * + * @param packet network packet + * @param err_packet the error structure + * + * @return 0 on success + */ +int network_mysqld_proto_append_err_packet(GString *packet, + network_mysqld_err_packet_t *err_packet) +{ + int errmsg_len; + + network_mysqld_proto_append_int8(packet, 0xff); /* ERR */ + network_mysqld_proto_append_int16(packet, err_packet->errcode); /* errorcode */ + if (err_packet->version == NETWORK_MYSQLD_PROTOCOL_VERSION_41) { + g_string_append_c(packet, '#'); + if (err_packet->sqlstate && (err_packet->sqlstate->len > 0)) { + g_string_append_len(packet, err_packet->sqlstate->str, 5); + } else { + g_string_append_len(packet, C("07000")); + } + } + + errmsg_len = err_packet->errmsg->len; + if (errmsg_len >= 512) errmsg_len = 512; + g_string_append_len(packet, err_packet->errmsg->str, errmsg_len); + + return 0; +} + +network_mysqld_eof_packet_t *network_mysqld_eof_packet_new() { + network_mysqld_eof_packet_t *eof_packet; + + eof_packet = g_new0(network_mysqld_eof_packet_t, 1); + + return eof_packet; +} + +void network_mysqld_eof_packet_free(network_mysqld_eof_packet_t *eof_packet) { + if (!eof_packet) return; + + g_free(eof_packet); +} + + +/** + * decode a OK packet from the network packet + */ +int network_mysqld_proto_get_eof_packet(network_packet *packet, + network_mysqld_eof_packet_t *eof_packet) +{ + guint8 field_count; + guint16 server_status, warning_count; + + int err = 0; + + err = err || network_mysqld_proto_get_int8(packet, &field_count); + if (err) return -1; + + if (field_count != MYSQLD_PACKET_EOF) { + g_critical("%s: expected the first byte to be 0xfe, got %d", + G_STRLOC, + field_count); + return -1; + } + + err = err || network_mysqld_proto_get_int16(packet, &warning_count); + err = err || network_mysqld_proto_get_int16(packet, &server_status); + if (!err) { + eof_packet->server_status = server_status; + eof_packet->warnings = warning_count; + g_debug("%s: server status, got: %d", + G_STRLOC, + eof_packet->server_status); + } + + return err ? -1 : 0; +} + + +network_mysqld_auth_challenge *network_mysqld_auth_challenge_new() { + network_mysqld_auth_challenge *shake; + + shake = g_new0(network_mysqld_auth_challenge, 1); + + shake->auth_plugin_data = g_string_new(""); + shake->scrambled_password = g_string_new(""); + shake->capabilities = CETUS_DEFAULT_FLAGS; + shake->auth_plugin_name = g_string_new(NULL); + + return shake; +} + +void network_mysqld_auth_challenge_free(network_mysqld_auth_challenge *shake) { + if (!shake) return; + + if (shake->server_version_str) g_free(shake->server_version_str); + if (shake->auth_plugin_data) g_string_free(shake->auth_plugin_data, TRUE); + if (shake->scrambled_password) g_string_free(shake->scrambled_password, TRUE); + if (shake->auth_plugin_name) g_string_free(shake->auth_plugin_name, TRUE); + + g_free(shake); +} + +network_mysqld_auth_challenge * +network_mysqld_auth_challenge_copy(const network_mysqld_auth_challenge *src) { + network_mysqld_auth_challenge *dst; + + if (!src) return NULL; + + dst = network_mysqld_auth_challenge_new(); + dst->protocol_version = src->protocol_version; + dst->server_version = src->server_version; + dst->thread_id = src->thread_id; + dst->capabilities = src->capabilities; + dst->charset = src->charset; + dst->server_status = src->server_status; + dst->server_version_str = g_strdup(src->server_version_str); + g_string_assign_len(dst->auth_plugin_data, S(src->auth_plugin_data)); + g_string_assign_len(dst->auth_plugin_name, S(src->auth_plugin_name)); + + return dst; +} + + +void network_mysqld_auth_challenge_set_challenge(network_mysqld_auth_challenge *shake) { + guint i; + + /* 20 chars */ + + g_string_set_size(shake->auth_plugin_data, 21); + + for (i = 0; i < 20; i++) { + /* 33 - 127 are printable characters */ + shake->auth_plugin_data->str[i] = (94.0 * (rand() / (RAND_MAX + 1.0))) + 33; + } + + shake->auth_plugin_data->len = 20; + shake->auth_plugin_data->str[shake->auth_plugin_data->len] = '\0'; +} + +int network_mysqld_proto_get_auth_challenge(network_packet *packet, + network_mysqld_auth_challenge *shake) +{ + int maj, min, patch; + gchar *auth_plugin_data_1 = NULL, *auth_plugin_data_2 = NULL; + guint16 capabilities1, capabilities2; + guint8 status; + int err = 0; + guint8 auth_plugin_data_len; + + err = err || network_mysqld_proto_get_int8(packet, &status); + + if (err) return -1; + + switch (status) { + case 0xff: + return -1; + case 0x0a: + break; + default: + g_debug("%s: unknown protocol %d", + G_STRLOC, + status + ); + return -1; + } + + err = err || network_mysqld_proto_get_string(packet, &shake->server_version_str); + err = err || (NULL == shake->server_version_str); /* the server-version has to be set */ + + err = err || network_mysqld_proto_get_int32(packet, &shake->thread_id); + + /** + * get the scramble buf + * + * 8 byte here and some the other 12 sometime later + */ + err = err || network_mysqld_proto_get_str_len(packet, &auth_plugin_data_1, 8); + + err = err || network_mysqld_proto_skip(packet, 1); + + err = err || network_mysqld_proto_get_int16(packet, &capabilities1); + err = err || network_mysqld_proto_get_int8(packet, &shake->charset); + err = err || network_mysqld_proto_get_int16(packet, &shake->server_status); + + /* capabilities is extended in 5.5.x to carry 32bits to announce CLIENT_PLUGIN_AUTH */ + err = err || network_mysqld_proto_get_int16(packet, &capabilities2); + err = err || network_mysqld_proto_get_int8(packet, &auth_plugin_data_len); + + err = err || network_mysqld_proto_skip(packet, 10); + + if (!err) { + shake->capabilities = capabilities1 | (capabilities2 << 16); + + if (shake->capabilities & CLIENT_PLUGIN_AUTH) { + guint8 auth_plugin_data2_len = 0; + + /* CLIENT_PLUGIN_AUTH enforces auth_plugin_data_len + * + * we have at least 12 bytes */ + + if (auth_plugin_data_len > 8) { + auth_plugin_data2_len = auth_plugin_data_len - 8; + } + + err = network_mysqld_proto_get_str_len(packet, &auth_plugin_data_2, + auth_plugin_data2_len); + err = err || network_mysqld_proto_skip(packet, 12 - MIN(12, auth_plugin_data2_len)); + if (!err) { + /* Bug#59453 ... MySQL 5.5.7-9 and 5.6.0-1 don't send a trailing \0 + * + * if there is no trailing \0, get the rest of the packet + */ + if (0 != network_mysqld_proto_get_gstr(packet, shake->auth_plugin_name)) { + err = err || network_mysqld_proto_get_gstr_len(packet, + packet->data->len - packet->offset, + shake->auth_plugin_name); + } + } + } else { + err = err || network_mysqld_proto_get_str_len(packet, &auth_plugin_data_2, 12); + err = err || network_mysqld_proto_skip(packet, 1); + } + } + + if (!err) { + /* process the data */ + + if (3 != sscanf(shake->server_version_str, "%d.%d.%d%*s", &maj, &min, &patch)) { + /* can't parse the protocol */ + + g_critical("%s: protocol 10, but version number not parsable", G_STRLOC); + + return -1; + } + + /** + * out of range + */ + if (min < 0 || min > 100 || + patch < 0 || patch > 100 || + maj < 0 || maj > 10) { + g_critical("%s: protocol 10, but version number out of range", G_STRLOC); + + return -1; + } + + shake->server_version = + maj * 10000 + + min * 100 + + patch; + + + /** + * build auth_plugin_data + * + * auth_plugin_data_1 + auth_plugin_data_2 == auth_plugin_data + */ + g_string_truncate(shake->auth_plugin_data, 0); + + if (shake->capabilities & CLIENT_PLUGIN_AUTH) { + g_string_assign_len(shake->auth_plugin_data, auth_plugin_data_1, + MIN(8, auth_plugin_data_len)); + if (auth_plugin_data_len > 8) { + g_string_append_len(shake->auth_plugin_data, auth_plugin_data_2, + auth_plugin_data_len - 8); + } + } else { + g_string_assign_len(shake->auth_plugin_data, auth_plugin_data_1, 8); + g_string_append_len(shake->auth_plugin_data, auth_plugin_data_2, 12); + } + + /* some final assertions */ + if (shake->capabilities & CLIENT_PLUGIN_AUTH) { + if (shake->auth_plugin_data->len != auth_plugin_data_len) { + err = 1; + } + } else { + if (shake->auth_plugin_data->len != 20) { + err = 1; + } + } + } + + if (auth_plugin_data_1) g_free(auth_plugin_data_1); + if (auth_plugin_data_2) g_free(auth_plugin_data_2); + + return err ? -1 : 0; +} + +int network_mysqld_proto_append_auth_challenge(GString *packet, + network_mysqld_auth_challenge *shake) +{ + guint i; + + network_mysqld_proto_append_int8(packet, 0x0a); + if (shake->server_version_str) { + g_string_append(packet, shake->server_version_str); + } else if (shake->server_version > 30000 && shake->server_version < 100000) { + g_string_append_printf(packet, "%d.%02d.%02d", + shake->server_version / 10000, + (shake->server_version % 10000) / 100, + shake->server_version % 100 + ); + } else { + g_string_append_len(packet, C("5.0.99")); + } + network_mysqld_proto_append_int8(packet, 0x00); + network_mysqld_proto_append_int32(packet, shake->thread_id); + if (shake->auth_plugin_data->len) { + g_assert_cmpint(shake->auth_plugin_data->len, >=, 8); + g_string_append_len(packet, shake->auth_plugin_data->str, 8); + } else { + g_string_append_len(packet, C("01234567")); + } + network_mysqld_proto_append_int8(packet, 0x00); /* filler */ + network_mysqld_proto_append_int16(packet, shake->capabilities & 0xffff); + network_mysqld_proto_append_int8(packet, shake->charset); + network_mysqld_proto_append_int16(packet, shake->server_status); + network_mysqld_proto_append_int16(packet, (shake->capabilities >> 16) & 0xffff); + + if (shake->capabilities & CLIENT_PLUGIN_AUTH) { + g_assert_cmpint(shake->auth_plugin_data->len, <, 255); + network_mysqld_proto_append_int8(packet, shake->auth_plugin_data->len); + } else { + network_mysqld_proto_append_int8(packet, 0); + } + + /* add the fillers */ + for (i = 0; i < 10; i++) { + network_mysqld_proto_append_int8(packet, 0x00); + } + + if (shake->capabilities & CLIENT_PLUGIN_AUTH) { + g_assert_cmpint(shake->auth_plugin_data->len, >=, 8); + g_string_append_len(packet, shake->auth_plugin_data->str + 8, + shake->auth_plugin_data->len - 8); + + g_string_append_len(packet, S(shake->auth_plugin_name)); + g_string_append_c(packet, 0x00); + } else { + /* if we only have SECURE_CONNECTION it is 0-terminated */ + if (shake->auth_plugin_data->len) { + g_assert_cmpint(shake->auth_plugin_data->len, >=, 8); + g_string_append_len(packet, shake->auth_plugin_data->str + 8, + shake->auth_plugin_data->len - 8); + } else { + g_string_append_len(packet, C("890123456789")); + } + network_mysqld_proto_append_int8(packet, 0x00); + } + + return 0; +} + +network_mysqld_auth_response * +network_mysqld_auth_response_new(guint32 server_capabilities) { + network_mysqld_auth_response *auth; + + auth = g_new0(network_mysqld_auth_response, 1); + + /* we have to make sure scramble->buf is not-NULL to get + * the "empty string" and not a "NULL-string" + */ + auth->auth_plugin_data = g_string_new(""); + auth->auth_plugin_name = g_string_new(NULL); + auth->username = g_string_new(""); + auth->database = g_string_new(""); + auth->client_capabilities = CETUS_DEFAULT_FLAGS; + auth->server_capabilities = server_capabilities; + + return auth; +} + +void network_mysqld_auth_response_free(network_mysqld_auth_response *auth) { + if (!auth) return; + + if (auth->auth_plugin_data) g_string_free(auth->auth_plugin_data, TRUE); + if (auth->auth_plugin_name) g_string_free(auth->auth_plugin_name, TRUE); + if (auth->username) g_string_free(auth->username, TRUE); + if (auth->database) g_string_free(auth->database, TRUE); + + g_free(auth); +} + + +int network_mysqld_proto_get_auth_response(network_packet *packet, + network_mysqld_auth_response *auth) +{ + int err = 0; + guint16 l_cap; + /* extract the default db from it */ + + /* + * @\0\0\1 + * \215\246\3\0 - client-flags + * \0\0\0\1 - max-packet-len + * \10 - charset-num + * \0\0\0\0 + * \0\0\0\0 + * \0\0\0\0 + * \0\0\0\0 + * \0\0\0\0 + * \0\0\0 - fillers + * root\0 - username + * \24 - len of the scrambled buf + * ~ \272 \361 \346 + * \211 \353 D \351 + * \24 \243 \223 \257 + * \0 ^ \n \254 + * t \347 \365 \244 + * + * world\0 + */ + + /* 4.0 uses 2 byte, 4.1+ uses 4 bytes, but the proto-flag is in the lower 2 bytes */ + err = err || network_mysqld_proto_peek_int16(packet, &l_cap); + if (err) return -1; + + if (l_cap & CLIENT_PROTOCOL_41) { + err = err || network_mysqld_proto_get_int32(packet, &auth->client_capabilities); + err = err || network_mysqld_proto_get_int32(packet, &auth->max_packet_size); + err = err || network_mysqld_proto_get_int8(packet, &auth->charset); + + err = err || network_mysqld_proto_skip(packet, 23); + + err = err || network_mysqld_proto_get_gstr(packet, auth->username); + + guint8 len; + /* new auth is 1-byte-len + data */ + err = err || network_mysqld_proto_get_int8(packet, &len); + err = err || network_mysqld_proto_get_gstr_len(packet, len, auth->auth_plugin_data); + + if ((auth->server_capabilities & CLIENT_CONNECT_WITH_DB) && + (auth->client_capabilities & CLIENT_CONNECT_WITH_DB)) { + err = err || network_mysqld_proto_get_gstr(packet, auth->database); + } + + if ((auth->server_capabilities & CLIENT_PLUGIN_AUTH) && + (auth->client_capabilities & CLIENT_PLUGIN_AUTH)) { + err = err || network_mysqld_proto_get_gstr(packet, auth->auth_plugin_name); + } + } else { + err = err || network_mysqld_proto_get_int16(packet, &l_cap); + err = err || network_mysqld_proto_get_int24(packet, &auth->max_packet_size); + err = err || network_mysqld_proto_get_gstr(packet, auth->username); + if (packet->data->len != packet->offset) { + /* if there is more, it is the password without a terminating \0 */ + err = err || network_mysqld_proto_get_gstr_len(packet, + packet->data->len - packet->offset, auth->auth_plugin_data); + } + + if (!err) { + auth->client_capabilities = l_cap; + } + } + + return err ? -1 : 0; +} + + +/* Change the auth process to the simple auth process */ +int network_mysqld_proto_get_and_change_auth_response(network_packet *packet, + network_mysqld_auth_response *auth) +{ + int err = 0; + guint32 mysql_packet_len = 0; + guint16 l_cap; + /* extract the default db from it */ + + /* + * @\0\0\1 + * \215\246\3\0 - client-flags + * \0\0\0\1 - max-packet-len + * \10 - charset-num + * \0\0\0\0 + * \0\0\0\0 + * \0\0\0\0 + * \0\0\0\0 + * \0\0\0\0 + * \0\0\0 - fillers + * root\0 - username + * \24 - len of the scrambled buf + * ~ \272 \361 \346 + * \211 \353 D \351 + * \24 \243 \223 \257 + * \0 ^ \n \254 + * t \347 \365 \244 + * + * world\0 + */ + + err = err || network_mysqld_proto_get_int24(packet, &mysql_packet_len); + if (err) return -1; + + packet->offset = NET_HEADER_SIZE; + mysql_packet_len += NET_HEADER_SIZE; + + /* 4.0 uses 2 byte, 4.1+ uses 4 bytes, but the proto-flag is in the lower 2 bytes */ + err = err || network_mysqld_proto_peek_int16(packet, &l_cap); + if (err) return -1; + + if (l_cap & CLIENT_PROTOCOL_41) { + unsigned char *cap = (unsigned char *) packet->data->str + packet->offset; + /* unset connect attrs and plugin auth flags in Extended Client Capabilities */ + cap[2] &= ~(8|16); + err = err || network_mysqld_proto_get_int32(packet, &auth->client_capabilities); + + err = err || network_mysqld_proto_get_int32(packet, &auth->max_packet_size); + err = err || network_mysqld_proto_get_int8(packet, &auth->charset); + + err = err || network_mysqld_proto_skip(packet, 23); + + err = err || network_mysqld_proto_get_gstr(packet, auth->username); + + guint8 len; + /* new auth is 1-byte-len + data */ + err = err || network_mysqld_proto_get_int8(packet, &len); + + err = err || network_mysqld_proto_get_gstr_len(packet, len, auth->auth_plugin_data); + + if ((auth->server_capabilities & CLIENT_CONNECT_WITH_DB) && + (auth->client_capabilities & CLIENT_CONNECT_WITH_DB)) { + err = err || network_mysqld_proto_get_gstr(packet, auth->database); + } + + if (mysql_packet_len != packet->offset) { + network_mysqld_proto_set_packet_len(packet->data, packet->offset - NET_HEADER_SIZE); + packet->data->len = packet->offset; + } + /* if ((auth->server_capabilities & CLIENT_PLUGIN_AUTH) && + (auth->client_capabilities & CLIENT_PLUGIN_AUTH)) { + err = err || network_mysqld_proto_get_gstr(packet, auth->auth_plugin_name); + }*/ + } else { + err = err || network_mysqld_proto_get_int16(packet, &l_cap); + err = err || network_mysqld_proto_get_int24(packet, &auth->max_packet_size); + err = err || network_mysqld_proto_get_gstr(packet, auth->username); + if (packet->data->len != packet->offset) { + /* if there is more, it is the password without a terminating \0 */ + err = err || network_mysqld_proto_get_gstr_len(packet, + packet->data->len - packet->offset, auth->auth_plugin_data); + } + + if (!err) { + auth->client_capabilities = l_cap; + } + } + + return err ? -1 : 0; +} + +/** + * append the auth struct to the mysqld packet + */ +int network_mysqld_proto_append_auth_response(GString *packet, + network_mysqld_auth_response *auth) +{ + if (!(auth->client_capabilities & CLIENT_PROTOCOL_41)) { + network_mysqld_proto_append_int16(packet, auth->client_capabilities); + network_mysqld_proto_append_int24(packet, auth->max_packet_size); /* max-allowed-packet */ + + if (auth->username->len) g_string_append_len(packet, S(auth->username)); + network_mysqld_proto_append_int8(packet, 0x00); /* trailing \0 */ + + if (auth->auth_plugin_data->len) { + g_string_append_len(packet, S(auth->auth_plugin_data)); /* no trailing \0 */ + } + } else { + network_mysqld_proto_append_int32(packet, auth->client_capabilities); + network_mysqld_proto_append_int32(packet, auth->max_packet_size); /* max-allowed-packet */ + + network_mysqld_proto_append_int8(packet, auth->charset); /* charset */ + + int i; + for (i = 0; i < 23; i++) { /* filler */ + network_mysqld_proto_append_int8(packet, 0x00); + } + + if (auth->username->len) g_string_append_len(packet, S(auth->username)); + network_mysqld_proto_append_int8(packet, 0x00); /* trailing \0 */ + + /* scrambled password */ + + /* server supports the secure-auth (4.1+) which is 255 bytes max + * + * if ->len is longer than 255, wrap around ... should be reported back + * to the upper layers + */ + network_mysqld_proto_append_int8(packet, auth->auth_plugin_data->len); + g_string_append_len(packet, auth->auth_plugin_data->str, + auth->auth_plugin_data->len & 0xff); + + if ((auth->server_capabilities & CLIENT_CONNECT_WITH_DB) && + (auth->database->len > 0)) { + g_string_append_len(packet, S(auth->database)); + network_mysqld_proto_append_int8(packet, 0x00); /* trailing \0 */ + } + + if ((auth->client_capabilities & CLIENT_PLUGIN_AUTH) && + (auth->server_capabilities & CLIENT_PLUGIN_AUTH)) { + g_string_append_len(packet, S(auth->auth_plugin_name)); + network_mysqld_proto_append_int8(packet, 0x00); /* trailing \0 */ + } + } + + return 0; +} + +int network_mysqld_proto_get_stmt_id(network_packet *packet, guint32 *stmt_id) +{ + guint8 packet_type; + int err = 0; + + err = err || network_mysqld_proto_get_int8(packet, &packet_type); + if (err) return -1; + + if (COM_STMT_EXECUTE != packet_type && COM_STMT_CLOSE != packet_type) { + g_critical("%s: expected the first byte to be %02x or %02x, got %02x", + G_STRLOC, + COM_STMT_EXECUTE, + COM_STMT_CLOSE, + packet_type); + return -1; + } + + err = err || network_mysqld_proto_get_int32(packet, stmt_id); + + return err ? -1 : 0; +} + +int network_mysqld_proto_change_stmt_id_from_ok_packet(network_packet *packet, + int server_index) +{ + guint8 packet_type; + int err = 0; + int *p = NULL; + + err = err || network_mysqld_proto_get_int8(packet, &packet_type); + if (err) return -1; + + if (0x00 != packet_type) { + g_debug("%s: expected the first byte to be %02x, got %02x", + G_STRLOC, + 0x00, + packet_type); + return -1; + } + + p = (int *)(((unsigned char *)packet->data->str) + packet->offset); + + /* if (*p > MAX_STMT_ID) { + }*/ + + g_debug("%s: stmt id:%d, server index:%d", + G_STRLOC, + *p, + server_index); + + *p = *p & 0x00007fff; + + *p = *p | (server_index << 16); + + g_debug("%s: new stmt id:%d, server index:%d", + G_STRLOC, + *p, + server_index); + + return 0; +} + +int network_mysqld_proto_change_stmt_id_from_clt_stmt(network_packet *packet) { + guint8 packet_type; + int err = 0; + int *p = NULL; + + err = err || network_mysqld_proto_get_int8(packet, &packet_type); + if (err) return -1; + + if (COM_STMT_EXECUTE != packet_type && COM_STMT_CLOSE!= packet_type) { + g_critical("%s: expected the first byte to be %02x or %02x, got %02x", + G_STRLOC, + COM_STMT_EXECUTE, + COM_STMT_CLOSE, + packet_type); + return -1; + } + + p = (int *)(((unsigned char *)packet->data->str) + packet->offset); + + *p = *p & 0x00007fff; + + return 0; +} + +int network_mysqld_proto_append_query_packet(GString *packet, const char *query) +{ + network_mysqld_proto_append_int8(packet, COM_QUERY); + g_string_append_len(packet, query, strlen(query)); + return 0; +} + +int mysqld_proto_append_change_user_packet(GString *packet, + mysqld_change_user_packet_t *chuser) +{ + network_mysqld_proto_append_int8(packet, COM_CHANGE_USER); + + /* user name as string.NUL */ + g_string_append_len(packet, chuser->username->str, chuser->username->len + 1); + + if (!chuser->auth_plugin_data) { + g_string_append_c(packet, 0); + } else { + GString *auth_response = g_string_new(NULL); + network_mysqld_proto_password_scramble(auth_response, + S(chuser->auth_plugin_data), S(chuser->hashed_pwd)); + + /* auth response length as int<1> */ + network_mysqld_proto_append_int8(packet, auth_response->len); + /* auth response as string.var_len */ + g_string_append_len(packet, S(auth_response)); + g_string_free(auth_response, TRUE); + } + /* schema name as string.NUL */ + g_string_append_len(packet, chuser->database->str, chuser->database->len + 1); + /* charset as int<2> */ + network_mysqld_proto_append_int16(packet, chuser->charset); + return 0; +} diff --git a/src/network-mysqld-packet.h b/src/network-mysqld-packet.h new file mode 100644 index 0000000..0fc53eb --- /dev/null +++ b/src/network-mysqld-packet.h @@ -0,0 +1,253 @@ +#ifndef __NETWORK_MYSQLD_PACKET__ +#define __NETWORK_MYSQLD_PACKET__ + +#include + +#include "network-exports.h" + +#include "network-mysqld-proto.h" +#include "network-mysqld.h" + +#ifndef COM_RESET_CONNECTION +#define COM_RESET_CONNECTION 0x1F +#endif + +/** + * mid-level protocol + * + * the MySQL protocal is split up in three layers: + * + * - low-level (encoding of fields in a packet) + * - mid-level (encoding of packets) + * - high-level (grouping packets into a sequence) + */ + +typedef enum { + NETWORK_MYSQLD_PROTOCOL_VERSION_PRE41, + NETWORK_MYSQLD_PROTOCOL_VERSION_41 +} network_mysqld_protocol_t; + +/** + * tracking the state of the response of a COM_QUERY packet + */ +typedef struct { + enum { + PARSE_COM_QUERY_INIT, + PARSE_COM_QUERY_FIELD, + PARSE_COM_QUERY_RESULT, + PARSE_COM_QUERY_LOCAL_INFILE_DATA, + PARSE_COM_QUERY_LOCAL_INFILE_RESULT + } state; + + guint16 server_status; + guint16 warning_count; + guint64 affected_rows; + guint64 insert_id; + + gboolean was_resultset; + gboolean binary_encoded; + + guint64 rows; + guint64 bytes; + + guint8 query_status; +} network_mysqld_com_query_result_t; + +/** + * these capability flags are introduced in later versions of mysql, + * and default used in CLIENT_BASIC_FLAGS, clear them in CETUS_DEFAULT_FLAGS + */ +#ifndef CLIENT_CONNECT_ATTRS +#define CLIENT_CONNECT_ATTRS (1UL << 20) +#endif +#ifndef CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA +#define CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA (1UL << 21) +#endif +#ifndef CLIENT_CAN_HANDLE_EXPIRED_PASSWORDS +#define CLIENT_CAN_HANDLE_EXPIRED_PASSWORDS (1UL << 22) +#endif +#ifndef CLIENT_SESSION_TRACK +#define CLIENT_SESSION_TRACK (1UL << 23) +#endif +#ifndef CLIENT_DEPRECATE_EOF +#define CLIENT_DEPRECATE_EOF (1UL << 24) +#endif +#ifndef CLIENT_PLUGIN_AUTH +#define CLIENT_PLUGIN_AUTH (1UL << 19) +#endif + +#define CETUS_DEFAULT_FLAGS CLIENT_BASIC_FLAGS \ + & ~CLIENT_PLUGIN_AUTH /* not support plugin auth */ \ + & ~CLIENT_NO_SCHEMA /* permit database.table.column */ \ + & ~CLIENT_IGNORE_SPACE \ + & ~CLIENT_CONNECT_ATTRS \ + & ~CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA \ + & ~CLIENT_CAN_HANDLE_EXPIRED_PASSWORDS \ + & ~CLIENT_SESSION_TRACK \ + & ~CLIENT_DEPRECATE_EOF + + +NETWORK_API network_mysqld_com_query_result_t *network_mysqld_com_query_result_new(void); +NETWORK_API void network_mysqld_com_query_result_free(network_mysqld_com_query_result_t *); +NETWORK_API gboolean network_mysqld_com_query_result_is_local_infile(network_mysqld_com_query_result_t *); +NETWORK_API int network_mysqld_proto_get_com_query_result(network_packet *packet, + network_mysqld_com_query_result_t *udata, gboolean use_binary_row_data); + +/** + * tracking the response of a COM_STMT_PREPARE command + * + * depending on the kind of statement that was prepare we will receive 0-2 EOF packets + */ +typedef struct { + gboolean first_packet; + gint want_eofs; + int status; /* MYSQLD_PACKET_[OK/ERR] */ +} network_mysqld_com_stmt_prep_result_t; + +NETWORK_API network_mysqld_com_stmt_prep_result_t *network_mysqld_com_stmt_prepare_result_new(void); +NETWORK_API void network_mysqld_com_stmt_prepare_result_free(network_mysqld_com_stmt_prep_result_t *udata); +NETWORK_API int network_mysqld_proto_get_com_stmt_prep_result(network_packet *packet, + network_mysqld_com_stmt_prep_result_t *udata); + +/** + * tracking the response of a COM_INIT_DB command + * + * we have to track the default internally can only accept it + * if the server side OK'ed it + */ +typedef struct { + GString *db_name; +} network_mysqld_com_init_db_result_t; + +NETWORK_API network_mysqld_com_init_db_result_t *network_mysqld_com_init_db_result_new(void); +NETWORK_API void network_mysqld_com_init_db_result_free(network_mysqld_com_init_db_result_t *com_init_db); +NETWORK_API int network_mysqld_com_init_db_result_track_state(network_packet *, + network_mysqld_com_init_db_result_t *); +NETWORK_API int network_mysqld_proto_get_com_init_db_result(network_packet *, + network_mysqld_com_init_db_result_t *, network_mysqld_con *); + +NETWORK_API int network_mysqld_proto_get_query_result(network_packet *, network_mysqld_con *); +NETWORK_API int network_mysqld_con_command_states_init(network_mysqld_con *, network_packet *); + +typedef struct { + guint64 affected_rows; + guint64 insert_id; + guint16 server_status; + guint16 warnings; + + gchar *msg; +} network_mysqld_ok_packet_t; + +NETWORK_API network_mysqld_ok_packet_t *network_mysqld_ok_packet_new(void); +NETWORK_API void network_mysqld_ok_packet_free(network_mysqld_ok_packet_t *udata); + +NETWORK_API int network_mysqld_proto_get_ok_packet(network_packet *, network_mysqld_ok_packet_t *); +NETWORK_API int network_mysqld_proto_append_ok_packet(GString *, network_mysqld_ok_packet_t *); + +typedef struct { + GString *errmsg; + GString *sqlstate; + + guint16 errcode; + network_mysqld_protocol_t version; +} network_mysqld_err_packet_t; + +NETWORK_API network_mysqld_err_packet_t *network_mysqld_err_packet_new(void); +NETWORK_API network_mysqld_err_packet_t *network_mysqld_err_packet_new_pre41(void); +NETWORK_API void network_mysqld_err_packet_free(network_mysqld_err_packet_t *udata); + +NETWORK_API int network_mysqld_proto_get_err_packet(network_packet *, network_mysqld_err_packet_t *); +NETWORK_API int network_mysqld_proto_append_err_packet(GString *, network_mysqld_err_packet_t *); + +typedef struct { + guint16 server_status; + guint16 warnings; +} network_mysqld_eof_packet_t; + +NETWORK_API network_mysqld_eof_packet_t *network_mysqld_eof_packet_new(void); +NETWORK_API void network_mysqld_eof_packet_free(network_mysqld_eof_packet_t *udata); + +NETWORK_API int network_mysqld_proto_get_eof_packet(network_packet *, network_mysqld_eof_packet_t *); + +struct network_mysqld_auth_challenge { + guint8 protocol_version; + gchar *server_version_str; + guint32 server_version; + guint32 thread_id; + GString *auth_plugin_data; + GString *scrambled_password; + guint32 capabilities; + guint8 charset; + guint16 server_status; + GString *auth_plugin_name; +}; + +NETWORK_API network_mysqld_auth_challenge *network_mysqld_auth_challenge_new(void); +NETWORK_API void network_mysqld_auth_challenge_free(network_mysqld_auth_challenge *shake); +NETWORK_API int network_mysqld_proto_get_auth_challenge(network_packet *, network_mysqld_auth_challenge *); +NETWORK_API int network_mysqld_proto_append_auth_challenge(GString *, network_mysqld_auth_challenge *); +NETWORK_API void network_mysqld_auth_challenge_set_challenge(network_mysqld_auth_challenge *); +NETWORK_API network_mysqld_auth_challenge * +network_mysqld_auth_challenge_copy(const network_mysqld_auth_challenge *); + +struct network_mysqld_auth_response { + guint32 client_capabilities; + guint32 server_capabilities; + guint32 max_packet_size; + guint8 charset; + GString *username; + GString *auth_plugin_data; + GString *database; + GString *auth_plugin_name; +}; + +NETWORK_API network_mysqld_auth_response *network_mysqld_auth_response_new(guint server_capabilities); +NETWORK_API void network_mysqld_auth_response_free(network_mysqld_auth_response *); +NETWORK_API int network_mysqld_proto_append_auth_response(GString *, network_mysqld_auth_response *); +NETWORK_API int network_mysqld_proto_get_auth_response(network_packet *, network_mysqld_auth_response *); +NETWORK_API int network_mysqld_proto_get_and_change_auth_response(network_packet *, + network_mysqld_auth_response *); + +/* COM_STMT_* */ + +typedef struct { + GString *stmt_text; +} network_mysqld_stmt_prep_pack_t; + +typedef struct { + guint32 stmt_id; + guint16 num_columns; + guint16 num_params; + guint16 warnings; +} network_mysqld_stmt_prep_ok_pack_t; + + +typedef struct { + guint32 stmt_id; + guint8 flags; + guint32 iteration_count; + guint8 new_params_bound; + GPtrArray *params; /**< array */ +} network_mysqld_stmt_exec_pack_t; + +NETWORK_API int network_mysqld_proto_get_stmt_id(network_packet *packet, guint32 *stmt_id); +NETWORK_API int network_mysqld_proto_change_stmt_id_from_ok_packet(network_packet *packet, + int server_index); +NETWORK_API int network_mysqld_proto_change_stmt_id_from_ok(network_packet *packet, int server_index); +NETWORK_API int network_mysqld_proto_change_stmt_id_from_clt_stmt(network_packet *packet); + + +NETWORK_API int network_mysqld_proto_append_query_packet(GString *, const char *); + + +typedef struct { + guint8 charset; + const GString *database; + const GString *auth_plugin_data; + const GString *username; + const GString *hashed_pwd; +} mysqld_change_user_packet_t; + +int mysqld_proto_append_change_user_packet(GString *, mysqld_change_user_packet_t *); + +#endif diff --git a/src/network-mysqld-proto.c b/src/network-mysqld-proto.c new file mode 100644 index 0000000..3bfab24 --- /dev/null +++ b/src/network-mysqld-proto.c @@ -0,0 +1,904 @@ +#include +#include +#include + +#include "network-mysqld-proto.h" + +#include "sys-pedantic.h" +#include "glib-ext.h" +#include "cetus-util.h" + +/** + * decode a length-encoded integer from a network packet + * + * _off is incremented on success + * + * @param packet the MySQL-packet to decode + * @param v destination of the integer + * @return 0 on success, non-0 on error + * + */ +int network_mysqld_proto_get_lenenc_int(network_packet *packet, guint64 *v) { + guint off = packet->offset; + guint64 ret = 0; + unsigned char *bytestream = (unsigned char *)packet->data->str; + + if (off >= packet->data->len) { + g_critical("%s: offset is too large:%d, packet len:%d", + G_STRLOC, + (int) off, (int) packet->data->len); + + return -1; + } + + if (bytestream[off] < 251) { /* */ + ret = bytestream[off]; + } else if (bytestream[off] == 252) { /* 2 byte length */ + if (off + 2 >= packet->data->len) return -1; + ret = (bytestream[off + 1] << 0) | + (bytestream[off + 2] << 8) ; + off += 2; + } else if (bytestream[off] == 253) { /* 3 byte */ + if (off + 3 >= packet->data->len) return -1; + ret = (bytestream[off + 1] << 0) | + (bytestream[off + 2] << 8) | + (bytestream[off + 3] << 16); + + off += 3; + } else if (bytestream[off] == 254) { /* 8 byte */ + if (off + 8 >= packet->data->len) return -1; + ret =(unsigned int) ((bytestream[off + 5] << 0) | + (bytestream[off + 6] << 8) | + (bytestream[off + 7] << 16) | + (bytestream[off + 8] << 24)); + ret <<= 32; + + ret += (unsigned int) ((bytestream[off + 1] << 0) | + (bytestream[off + 2] << 8) | + (bytestream[off + 3] << 16) | + (bytestream[off + 4] << 24)); + + off += 8; + } else { + /* if we hit this place we complete have no idea about the protocol */ + if (bytestream[off] != 251) { + g_critical("%s: bytestream[%d] is %d", + G_STRLOC, + off, bytestream[off]); + } + + /* either ERR (255) or NULL (251) */ + packet->offset = off + 1; + + return -1; + + } + off += 1; + + packet->offset = off; + + *v = ret; + + return 0; +} + +/** + * skip bytes in the network packet + * + * a assertion makes sure that we can't skip over the end of the packet + * + * @param packet the MySQL network packet + * @param size bytes to skip + * + */ +int network_mysqld_proto_skip(network_packet *packet, gsize size) { + if (packet->offset + size > packet->data->len) return -1; + + packet->offset += size; + + return 0; +} + +int network_mysqld_proto_skip_lenenc_str(network_packet *packet) +{ + guint64 len; + + if (packet->offset >= packet->data->len) { + g_debug_hexdump(G_STRLOC, S(packet->data)); + return -1; + } + if (packet->offset >= packet->data->len) { + return -1; + } + + if (network_mysqld_proto_get_lenenc_int(packet, &len)) return -1; + + if (packet->offset + len > packet->data->len) return -1; + + packet->offset += len; + return 0; +} + +/** + * get a fixed-length integer from the network packet + * + * @param packet the MySQL network packet + * @param v destination of the integer + * @param size byte-len of the integer to decode + * @return a the decoded integer + */ +int network_mysqld_proto_peek_int_len(network_packet *packet, guint64 *v, gsize size) { + gsize i; + int shift; + guint32 r_l = 0, r_h = 0; + guchar *bytes = (guchar *)packet->data->str + packet->offset; + + if (packet->offset > packet->data->len) { + return -1; + } + if (packet->offset + size > packet->data->len) { + return -1; + } + + /** + * for some reason left-shift > 32 leads to negative numbers + */ + for (i = 0, shift = 0; + i < size && i < 4; + i++, shift += 8, bytes++) { + r_l |= ((*bytes) << shift); + } + + for (shift = 0; + i < size; + i++, shift += 8, bytes++) { + r_h |= ((*bytes) << shift); + } + + *v = (((guint64)r_h << 32) | r_l); + + return 0; +} + +int network_mysqld_proto_get_int_len(network_packet *packet, guint64 *v, gsize size) { + int err = 0; + + err = err || network_mysqld_proto_peek_int_len(packet, v, size); + + if (err) return -1; + + packet->offset += size; + + return 0; +} +/** + * get a 8-bit integer from the network packet + * + * @param packet the MySQL network packet + * @param v dest for the number + * @return 0 on success, non-0 on error + * @see network_mysqld_proto_get_int_len() + */ +int network_mysqld_proto_get_int8(network_packet *packet, guint8 *v) { + guint64 v64; + + if (network_mysqld_proto_get_int_len(packet, &v64, 1)) return -1; + + g_assert_cmpint(v64 & 0xff, ==, v64); /* check that we really only got one byte back */ + + *v = v64 & 0xff; + + return 0; +} + +/** + * get a 8-bit integer from the network packet + * + * @param packet the MySQL network packet + * @param v dest for the number + * @return 0 on success, non-0 on error + * @see network_mysqld_proto_get_int_len() + */ +int network_mysqld_proto_peek_int8(network_packet *packet, guint8 *v) { + guint64 v64; + + if (network_mysqld_proto_peek_int_len(packet, &v64, 1)) return -1; + + /* check that we really only got one byte back */ + g_assert_cmpint(v64 & 0xff, ==, v64); + + *v = v64 & 0xff; + + return 0; +} + + +/** + * get a 16-bit integer from the network packet + * + * @param packet the MySQL network packet + * @param v dest for the number + * @return 0 on success, non-0 on error + * @see network_mysqld_proto_get_int_len() + */ +int network_mysqld_proto_get_int16(network_packet *packet, guint16 *v) { + guint64 v64; + + if (network_mysqld_proto_get_int_len(packet, &v64, 2)) return -1; + + /* check that we really only got two byte back */ + g_assert_cmpint(v64 & 0xffff, ==, v64); + + *v = v64 & 0xffff; + + return 0; +} + +/** + * get a 16-bit integer from the network packet + * + * @param packet the MySQL network packet + * @param v dest for the number + * @return 0 on success, non-0 on error + * @see network_mysqld_proto_get_int_len() + */ +int network_mysqld_proto_peek_int16(network_packet *packet, guint16 *v) { + guint64 v64; + + if (network_mysqld_proto_peek_int_len(packet, &v64, 2)) return -1; + + /* check that we really only got two byte back */ + g_assert_cmpint(v64 & 0xffff, ==, v64); + + *v = v64 & 0xffff; + + return 0; +} + + +/** + * get a 24-bit integer from the network packet + * + * @param packet the MySQL network packet + * @param v dest for the number + * @return 0 on success, non-0 on error + * @see network_mysqld_proto_get_int_len() + */ +int network_mysqld_proto_get_int24(network_packet *packet, guint32 *v) { + guint64 v64; + + if (network_mysqld_proto_get_int_len(packet, &v64, 3)) return -1; + + /* check that we really only got two byte back */ + g_assert_cmpint(v64 & 0x00ffffff, ==, v64); + + *v = v64 & 0x00ffffff; + + return 0; +} + + +/** + * get a 32-bit integer from the network packet + * + * @param packet the MySQL network packet + * @param v dest for the number + * @return 0 on success, non-0 on error + * @see network_mysqld_proto_get_int_len() + */ +int network_mysqld_proto_get_int32(network_packet *packet, guint32 *v) { + guint64 v64; + + if (network_mysqld_proto_get_int_len(packet, &v64, 4)) return -1; + + *v = v64 & 0xffffffff; + + return 0; +} + +/** + * get a string from the network packet + * + * @param packet the MySQL network packet + * @param s dest of the string + * @param len length of the string + * @return 0 on success, non-0 otherwise + * @return the string (allocated) or NULL of len is 0 + */ +int network_mysqld_proto_get_str_len(network_packet *packet, gchar **s, gsize len) { + gchar *str; + + if (len == 0) { + *s = NULL; + return 0; + } + + if (packet->offset > packet->data->len) { + return -1; + } + if (packet->offset + len > packet->data->len) { + g_critical("%s: packet-offset out of range: %u + "F_SIZE_T" > "F_SIZE_T, + G_STRLOC, + packet->offset, len, packet->data->len); + + return -1; + } + + str = g_malloc(len + 1); + memcpy(str, packet->data->str + packet->offset, len); + str[len] = '\0'; + + packet->offset += len; + + *s = str; + + return 0; +} + +static int network_mysqld_proto_get_str_len2(network_packet *packet, + gchar *s, gsize len) { + if (len == 0) { + return 0; + } + + if (packet->offset > packet->data->len) { + return -1; + } + if (packet->offset + len > packet->data->len) { + g_critical("%s: packet-offset out of range: %u + "F_SIZE_T" > "F_SIZE_T, + G_STRLOC, + packet->offset, len, packet->data->len); + + return -1; + } + + if (len) { + memcpy(s, packet->data->str + packet->offset, len); + s[len] = '\0'; + } + + packet->offset += len; + + return 0; +} + +/** + * get a variable-length string from the network packet + * + * variable length strings are prefixed with variable-length integer + * defining the length of the string + * + * @param packet the MySQL network packet + * @param s destination of the decoded string + * @param _len destination of the length of the decoded string, if len is non-NULL + * @return 0 on success, non-0 on error + * @see network_mysqld_proto_get_str_len(), network_mysqld_proto_get_lenenc_int() + */ +int +network_mysqld_proto_get_lenenc_str(network_packet *packet, + gchar **s, guint64 *_len) +{ + guint64 len; + + if (packet->offset >= packet->data->len) { + g_debug_hexdump(G_STRLOC, S(packet->data)); + return -1; + } + if (packet->offset >= packet->data->len) { + return -1; + } + + if (network_mysqld_proto_get_lenenc_int(packet, &len)) return -1; + + if (len > PACKET_LEN_MAX) return -1; + + if (packet->offset + len > packet->data->len) return -1; + + if (_len) *_len = len; + + return network_mysqld_proto_get_str_len(packet, s, len); +} + +int network_mysqld_proto_get_column(network_packet *packet, gchar *s, gsize s_size) { + guint64 len; + int err = network_mysqld_proto_get_lenenc_int(packet, &len); + if (err) { + return -1; + } + + if (s_size <= len) { + g_critical("%s: column too long:%ld, buffer size:%ld", G_STRLOC, len, s_size); + return -1; + } + + if (packet->offset + len > packet->data->len) + return -1; + + return network_mysqld_proto_get_str_len2(packet, s, len); +} + + +/** + * get a NUL-terminated string from the network packet + * + * @param packet the MySQL network packet + * @param s dest of the string + * @return 0 on success, non-0 otherwise + * @see network_mysqld_proto_get_str_len() + */ +int network_mysqld_proto_get_string(network_packet *packet, gchar **s) { + guint64 len; + int err = 0; + + for (len = 0; packet->offset + len < packet->data->len && + *(packet->data->str + packet->offset + len); len++); + + if (*(packet->data->str + packet->offset + len) != '\0') { + /* this has to be a \0 */ + return -1; + } + + if (len > 0) { + if (packet->offset >= packet->data->len) { + return -1; + } + if (packet->offset + len > packet->data->len) { + return -1; + } + + /** + * copy the string w/o the NUL byte + */ + err = err || network_mysqld_proto_get_str_len(packet, s, len); + } + + err = err || network_mysqld_proto_skip(packet, 1); + + return err ? -1 : 0; +} + + +/** + * get a GString from the network packet + * + * @param packet the MySQL network packet + * @param len bytes to copy + * @param out a GString which carries the string + * @return 0 on success, -1 on error + */ +int network_mysqld_proto_get_gstr_len(network_packet *packet, gsize len, GString *out) { + int err = 0; + + if (!out) return -1; + + g_string_truncate(out, 0); + + if (!len) return 0; /* nothing to copy */ + + err = err || (packet->offset >= packet->data->len); /* the offset is already too large */ + err = err || (packet->offset + len > packet->data->len); /* offset would get too large */ + + if (!err) { + g_string_append_len(out, packet->data->str + packet->offset, len); + packet->offset += len; + } + + return err ? -1 : 0; +} + +/** + * get a NUL-terminated GString from the network packet + * + * @param packet the MySQL network packet + * @param out a GString which carries the string + * @return a pointer to the string in out + * + * @see network_mysqld_proto_get_gstr_len() + */ +int network_mysqld_proto_get_gstr(network_packet *packet, GString *out) { + guint64 len; + int err = 0; + + for (len = 0; packet->offset + len < packet->data->len && + *(packet->data->str + packet->offset + len) != '\0'; len++); + + if (packet->offset + len == packet->data->len) { /* havn't found a trailing \0 */ + return -1; + } + + if (len > 0) { + g_assert(packet->offset < packet->data->len); + g_assert(packet->offset + len <= packet->data->len); + + err = err || network_mysqld_proto_get_gstr_len(packet, len, out); + } + + /* skip the \0 */ + err = err || network_mysqld_proto_skip(packet, 1); + + return err ? -1 : 0; +} + +/** + * create a empty field for a result-set definition + * + * @return a empty MYSQL_FIELD + */ +MYSQL_FIELD *network_mysqld_proto_fielddef_new() { + MYSQL_FIELD *field; + + field = g_new0(MYSQL_FIELD, 1); + + return field; +} + +void network_mysqld_mysql_field_row_free(GPtrArray *row) { + if (row) { + g_ptr_array_free(row, TRUE); + } +} + +/** + * free a MYSQL_FIELD and its components + * + * @param field the MYSQL_FIELD to free + */ +void network_mysqld_proto_fielddef_free(MYSQL_FIELD *field) { + if (field->catalog) g_free(field->catalog); + if (field->db) g_free(field->db); + if (field->name) g_free(field->name); + if (field->org_name) g_free(field->org_name); + if (field->table) g_free(field->table); + if (field->org_table) g_free(field->org_table); + + g_free(field); +} + +/** + * create a array of MYSQL_FIELD + * + * @return a empty array of MYSQL_FIELD + */ +GPtrArray *network_mysqld_proto_fielddefs_new(void) { + GPtrArray *fields; + + fields = g_ptr_array_new(); + + return fields; +} + +/** + * free a array of MYSQL_FIELD + * + * @param fields array of MYSQL_FIELD to free + * @see network_mysqld_proto_field_free() + */ +void network_mysqld_proto_fielddefs_free(GPtrArray *fields) { + guint i; + + for (i = 0; i < fields->len; i++) { + MYSQL_FIELD *field = fields->pdata[i]; + + if (field) network_mysqld_proto_fielddef_free(field); + } + + g_ptr_array_free(fields, TRUE); +} + +/** + * set length of the packet in the packet header + * + * each MySQL packet is + * - is prefixed by a 4 byte packet header + * - length is max 16Mbyte (3 Byte) + * - sequence-id (1 Byte) + * + * To encode a packet of more then 16M clients have to send multiple 16M frames + * + * the sequence-id is incremented for each related packet and wrapping from 255 to 0 + * + * @param header string of at least 4 byte to write the packet header to + * @param length length of the packet + * @param id sequence-id of the packet + * @return 0 + */ +int network_mysqld_proto_set_packet_len(GString *_header, guint32 length) { + unsigned char *header = (unsigned char *)_header->str; + + g_assert_cmpint(length, <=, PACKET_LEN_MAX); + + header[0] = (length >> 0) & 0xFF; + header[1] = (length >> 8) & 0xFF; + header[2] = (length >> 16) & 0xFF; + + return 0; +} + +int network_mysqld_proto_set_packet_id(GString *_header, guint8 id) { + unsigned char *header = (unsigned char *)_header->str; + + header[3] = id; + + return 0; +} + +int network_mysqld_proto_append_packet_len(GString *_header, guint32 length) { + return network_mysqld_proto_append_int24(_header, length); +} + +int network_mysqld_proto_append_packet_id(GString *_header, guint8 id) { + return network_mysqld_proto_append_int8(_header, id); +} + +/** + * decode the packet length from a packet header + * + * @param header the first 3 bytes of the network packet + * @return the packet length + * @see network_mysqld_proto_set_header() + */ +guint32 network_mysqld_proto_get_packet_len(GString *_header) { + unsigned char *header = (unsigned char *)_header->str; + + return header[0] | header[1] << 8 | header[2] << 16; +} + +/** + * decode the packet id from a packet header + * + * @param header the first 4 bytes of the network packet + * @return the packet id + * @see network_mysqld_proto_set_packet_id() + */ +guint8 network_mysqld_proto_get_packet_id(GString *_header) { + unsigned char *header = (unsigned char *)_header->str; + + return header[3]; +} + + +/** + * append the variable-length integer to the packet + * + * @param packet the MySQL network packet + * @param length integer to encode + * @return 0 + */ +int network_mysqld_proto_append_lenenc_int(GString *packet, guint64 length) { + if (length < 251) { + g_string_append_c(packet, length); + } else if (length < 65536) { + g_string_append_c(packet, (gchar)252); + g_string_append_c(packet, (length >> 0) & 0xff); + g_string_append_c(packet, (length >> 8) & 0xff); + } else if (length < 16777216) { + g_string_append_c(packet, (gchar)253); + g_string_append_c(packet, (length >> 0) & 0xff); + g_string_append_c(packet, (length >> 8) & 0xff); + g_string_append_c(packet, (length >> 16) & 0xff); + } else { + g_string_append_c(packet, (gchar)254); + + g_string_append_c(packet, (length >> 0) & 0xff); + g_string_append_c(packet, (length >> 8) & 0xff); + g_string_append_c(packet, (length >> 16) & 0xff); + g_string_append_c(packet, (length >> 24) & 0xff); + + g_string_append_c(packet, (length >> 32) & 0xff); + g_string_append_c(packet, (length >> 40) & 0xff); + g_string_append_c(packet, (length >> 48) & 0xff); + g_string_append_c(packet, (length >> 56) & 0xff); + } + + return 0; +} + +/** + * encode a GString in to a MySQL len-encoded string + * + * @param packet the MySQL network packet + * @param s string to encode + * @param length length of the string to encode + * @return 0 + */ +int +network_mysqld_proto_append_lenenc_str_len(GString *packet, const char *s, + guint64 length) +{ + if (!s) { + g_string_append_c(packet, (gchar)251); /** this is NULL */ + } else { + network_mysqld_proto_append_lenenc_int(packet, length); + g_string_append_len(packet, s, length); + } + + return 0; +} + +/** + * encode a GString in to a MySQL len-encoded string + * + * @param packet the MySQL network packet + * @param s string to encode + * + * @see network_mysqld_proto_append_lenenc_str_len() + */ +int network_mysqld_proto_append_lenenc_str(GString *packet, const char *s) { + return network_mysqld_proto_append_lenenc_str_len(packet, s, s ? strlen(s) : 0); +} + +/** + * encode fixed length integer in to a network packet + * + * @param packet the MySQL network packet + * @param num integer to encode + * @param size byte size of the integer + * @return 0 + */ +static int network_mysqld_proto_append_int_len(GString *packet, guint64 num, gsize size) { + gsize i; + + for (i = 0; i < size; i++) { + g_string_append_c(packet, num & 0xff); + num >>= 8; + } + + return 0; +} + +/** + * encode 8-bit integer in to a network packet + * + * @param packet the MySQL network packet + * @param num integer to encode + * + * @see network_mysqld_proto_append_int_len() + */ +int network_mysqld_proto_append_int8(GString *packet, guint8 num) { + return network_mysqld_proto_append_int_len(packet, num, 1); +} + +/** + * encode 16-bit integer in to a network packet + * + * @param packet the MySQL network packet + * @param num integer to encode + * + * @see network_mysqld_proto_append_int_len() + */ +int network_mysqld_proto_append_int16(GString *packet, guint16 num) { + return network_mysqld_proto_append_int_len(packet, num, 2); +} + +/** + * encode 24-bit integer in to a network packet + * + * @param packet the MySQL network packet + * @param num integer to encode + * + * @see network_mysqld_proto_append_int_len() + */ +int network_mysqld_proto_append_int24(GString *packet, guint32 num) { + return network_mysqld_proto_append_int_len(packet, num, 3); +} + + +/** + * encode 32-bit integer in to a network packet + * + * @param packet the MySQL network packet + * @param num integer to encode + * + * @see network_mysqld_proto_append_int_len() + */ +int network_mysqld_proto_append_int32(GString *packet, guint32 num) { + return network_mysqld_proto_append_int_len(packet, num, 4); +} + +/** + * hash the password as MySQL 4.1 and later assume + * + * SHA1(password) + * + * @see network_mysqld_proto_scramble + */ +int network_mysqld_proto_password_hash(GString *response, const char *password, + gsize password_len) +{ + GChecksum *cs; + + /* first round: SHA1(password) */ + cs = g_checksum_new(G_CHECKSUM_SHA1); + + g_checksum_update(cs, (guchar *)password, password_len); + + g_string_set_size(response, g_checksum_type_get_length(G_CHECKSUM_SHA1)); + /* will be overwritten with the right value in the next step */ + response->len = response->allocated_len; + g_checksum_get_digest(cs, (guchar *)response->str, &(response->len)); + + g_checksum_free(cs); + + return 0; +} + +/** + * scramble the hashed password with the challenge + * + * @param response dest + * @param challenge the challenge string as sent by the mysql-server + * @param challenge_len length of the challenge + * @param hashed_pwd hashed password + * @param hashed_pwd_len length of the hashed password + * + * @see network_mysqld_proto_password_hash + */ +int network_mysqld_proto_password_scramble(GString *response, + const char *challenge, gsize challenge_len, + const char *hashed_pwd, gsize hashed_pwd_len) { + int i; + GChecksum *cs; + GString *step2; + + g_return_val_if_fail(NULL != challenge, -1); + g_return_val_if_fail(20 == challenge_len || 21 == challenge_len, -1); + g_return_val_if_fail(NULL != hashed_pwd, -1); + g_return_val_if_fail(20 == hashed_pwd_len || 21 == hashed_pwd_len, -1); + + /** + * we have to run + * + * XOR(SHA1(password), SHA1(challenge + SHA1(SHA1(password))) + * + * where SHA1(password) is the hashed_pwd and + * challenge is ... challenge + * + * XOR(hashed_pwd, SHA1(challenge + SHA1(hashed_pwd))) + * + */ + + if (hashed_pwd_len == 21) hashed_pwd_len--; + + /* 1. SHA1(hashed_pwd) */ + step2 = g_string_new(NULL); + network_mysqld_proto_password_hash(step2, hashed_pwd, hashed_pwd_len); + + /* 2. SHA1(challenge + SHA1(hashed_pwd) */ + cs = g_checksum_new(G_CHECKSUM_SHA1); + + /* if the challenge is 21 bytes long it means we're behind a 5.5.7 or up server + * that supports authentication plugins. After auth-plugin-data-2 the protocol adds + * a spacing character to split it from the next part of the packet: auth-plugin-name. + * That spacing char '\0' is the 21th byte. + * + * We assume that auth-plugin-data is always 20 bytes, + * on this scnenario it is 21 so we need + * to ignore the last byte: the trailing '\0'. + */ + if (challenge_len == 21) challenge_len--; + + g_checksum_update(cs, (guchar *)challenge, challenge_len); + g_checksum_update(cs, (guchar *)step2->str, step2->len); + + g_string_set_size(response, g_checksum_type_get_length(G_CHECKSUM_SHA1)); + response->len = response->allocated_len; + g_checksum_get_digest(cs, (guchar *)response->str, &(response->len)); + + g_checksum_free(cs); + + /* XOR the hashed_pwd with SHA1(challenge + SHA1(hashed_pwd)) */ + for (i = 0; i < 20; i++) { + response->str[i] = (guchar)response->str[i] ^ (guchar)hashed_pwd[i]; + } + + g_string_free(step2, TRUE); + + return 0; +} + + +int network_mysqld_proto_skip_network_header(network_packet *packet) { + return network_mysqld_proto_skip(packet, NET_HEADER_SIZE); +} + +/*@}*/ diff --git a/src/network-mysqld-proto.h b/src/network-mysqld-proto.h new file mode 100644 index 0000000..d461ba2 --- /dev/null +++ b/src/network-mysqld-proto.h @@ -0,0 +1,109 @@ +#ifndef _NETWORK_MYSQLD_PROTO_H_ +#define _NETWORK_MYSQLD_PROTO_H_ + +#include +#include + +#include "network-exports.h" +/** + * 4.0 is missing too many things for us to support it, so we have to error out. + */ +#if MYSQL_VERSION_ID < 41000 +#error You need at least MySQL 4.1 to compile this software. +#endif +/** + * 4.1 uses other defines + * + * this should be one step to get closer to backward-compatibility + */ +#if MYSQL_VERSION_ID < 50000 +#define COM_STMT_EXECUTE COM_EXECUTE +#define COM_STMT_PREPARE COM_PREPARE +#define COM_STMT_CLOSE COM_CLOSE_STMT +#define COM_STMT_SEND_LONG_DATA COM_LONG_DATA +#define COM_STMT_RESET COM_RESET_STMT +#endif + +#define MYSQLD_PACKET_OK (0) +#define MYSQLD_PACKET_RAW (0xfa) /* used for proxy.response.type only */ +#define MYSQLD_PACKET_NULL (0xfb) /* 0xfb */ +/* 0xfc */ +/* 0xfd */ +#define MYSQLD_PACKET_EOF (0xfe) /* 0xfe */ +#define MYSQLD_PACKET_ERR (0xff) /* 0xff */ + +#define PACKET_LEN_MAX (0x00ffffff) +#define PACKET_LEN_UNSET (0xffffffff) + +typedef struct { + GString *data; + guint offset; +} network_packet; + +typedef enum { + NETWORK_MYSQLD_LENENC_TYPE_INT, + NETWORK_MYSQLD_LENENC_TYPE_NULL, + NETWORK_MYSQLD_LENENC_TYPE_EOF, + NETWORK_MYSQLD_LENENC_TYPE_ERR +} network_mysqld_lenenc_type; + +NETWORK_API int network_mysqld_proto_skip(network_packet *packet, gsize size); +NETWORK_API int network_mysqld_proto_skip_network_header(network_packet *packet); + +NETWORK_API int network_mysqld_proto_get_int_len(network_packet *packet, guint64 *v, gsize size); + +NETWORK_API int network_mysqld_proto_get_int8(network_packet *packet, guint8 *v); +NETWORK_API int network_mysqld_proto_get_int16(network_packet *packet, guint16 *v); +NETWORK_API int network_mysqld_proto_get_int24(network_packet *packet, guint32 *v); +NETWORK_API int network_mysqld_proto_get_int32(network_packet *packet, guint32 *v); + +NETWORK_API int network_mysqld_proto_peek_int_len(network_packet *packet, guint64 *v, gsize size); +NETWORK_API int network_mysqld_proto_peek_int8(network_packet *packet, guint8 *v); +NETWORK_API int network_mysqld_proto_peek_int16(network_packet *packet, guint16 *v); + +NETWORK_API int network_mysqld_proto_append_int8(GString *packet, guint8 num); +NETWORK_API int network_mysqld_proto_append_int16(GString *packet, guint16 num); +NETWORK_API int network_mysqld_proto_append_int24(GString *packet, guint32 num); +NETWORK_API int network_mysqld_proto_append_int32(GString *packet, guint32 num); + +int network_mysqld_proto_skip_lenenc_str(network_packet *packet); +NETWORK_API int network_mysqld_proto_get_lenenc_str(network_packet *, gchar **, guint64 *); +NETWORK_API int network_mysqld_proto_get_str_len(network_packet *packet, gchar **s, gsize len); +NETWORK_API int network_mysqld_proto_get_string(network_packet *packet, gchar **s); +NETWORK_API int network_mysqld_proto_get_column(network_packet *packet, gchar *s, gsize s_size); + +NETWORK_API int network_mysqld_proto_get_gstr_len(network_packet *, gsize , GString *); +NETWORK_API int network_mysqld_proto_get_gstr(network_packet *packet, GString *out); + +NETWORK_API int network_mysqld_proto_get_lenenc_int(network_packet *packet, guint64 *v); + +typedef MYSQL_FIELD network_mysqld_proto_fielddef_t; +NETWORK_API network_mysqld_proto_fielddef_t *network_mysqld_proto_fielddef_new(void); +NETWORK_API void network_mysqld_proto_fielddef_free(network_mysqld_proto_fielddef_t *fielddef); +NETWORK_API void network_mysqld_mysql_field_row_free(GPtrArray *row); +NETWORK_API int network_mysqld_proto_get_fielddef(network_packet *packet, + network_mysqld_proto_fielddef_t *field, guint32 capabilities); + +typedef GPtrArray network_mysqld_proto_fielddefs_t; +NETWORK_API network_mysqld_proto_fielddefs_t *network_mysqld_proto_fielddefs_new(void); +NETWORK_API void network_mysqld_proto_fielddefs_free(network_mysqld_proto_fielddefs_t *fielddefs); + +NETWORK_API guint32 network_mysqld_proto_get_packet_len(GString *_header); +NETWORK_API guint8 network_mysqld_proto_get_packet_id(GString *_header); +NETWORK_API int network_mysqld_proto_append_packet_len(GString *header, guint32 len); +NETWORK_API int network_mysqld_proto_append_packet_id(GString *header, guint8 id); +NETWORK_API int network_mysqld_proto_set_packet_len(GString *header, guint32 len); +NETWORK_API int network_mysqld_proto_set_packet_id(GString *header, guint8 id); + +NETWORK_API int network_mysqld_proto_append_lenenc_int(GString *packet, guint64 len); +NETWORK_API int network_mysqld_proto_append_lenenc_str_len(GString *packet, + const char *s, guint64 len); +NETWORK_API int network_mysqld_proto_append_lenenc_str(GString *packet, const char *s); + +NETWORK_API int network_mysqld_proto_password_hash(GString *response, + const char *password, gsize password_len); +NETWORK_API int network_mysqld_proto_password_scramble(GString *response, + const char *challenge, gsize challenge_len, + const char *hashed_pwd, gsize hashed_pwd_len); + +#endif diff --git a/src/network-mysqld.c b/src/network-mysqld.c new file mode 100644 index 0000000..7ffba63 --- /dev/null +++ b/src/network-mysqld.c @@ -0,0 +1,4876 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#ifdef HAVE_SYS_FILIO_H +#include +#endif + +#include +#include + +#include /** inet_ntoa */ +#include +#include + +#include +#include + +#include +#include +#include +#include +#ifdef HAVE_SIGNAL_H +#include +#endif + +#ifdef HAVE_SYS_UIO_H +#include +#endif + +#include + +#include +#include + +#include "glib-ext.h" +#include "network-mysqld.h" +#include "network-mysqld-proto.h" +#include "network-mysqld-packet.h" +#include "network-conn-pool.h" +#include "chassis-mainloop.h" +#include "chassis-event.h" +#include "cetus-log.h" +#include "resultset_merge.h" +#include "network-conn-pool-wrap.h" +#include "sharding-query-plan.h" +#include "cetus-util.h" +#include "server-session.h" +#include "cetus-users.h" +#include "cetus-monitor.h" +#include "cetus-variable.h" +#include "plugin-common.h" +#ifdef NETWORK_DEBUG_TRACE_STATE_CHANGES +#include "cetus-query-queue.h" +#endif + +#include "network-compress.h" + +#ifdef HAVE_WRITEV +#define USE_BUFFERED_NETIO +#else +#undef USE_BUFFERED_NETIO +#endif + +#define XA_BUF_LEN 2048 +#define XA_CMD_BUF_LEN 64 +#define E_NET_CONNRESET ECONNRESET +#define E_NET_CONNABORTED ECONNABORTED +#define E_NET_INPROGRESS EINPROGRESS +#if EWOULDBLOCK == EAGAIN +/** + * some system make EAGAIN == EWOULDBLOCK which would lead to a + * error in the case handling + * + * set it to -1 as this error should never happen + */ +#define E_NET_WOULDBLOCK -1 +#else +#define E_NET_WOULDBLOCK EWOULDBLOCK +#endif + +static void network_mysqld_self_con_handle(int event_fd, short events, + void *user_data); + +/** + * call the cleanup callback for the current connection + * + * @param srv global context + * @param con connection context + * + * @return NETWORK_SOCKET_SUCCESS on success + */ +network_socket_retval_t +plugin_call_cleanup(chassis *srv, network_mysqld_con *con) { + NETWORK_MYSQLD_PLUGIN_FUNC(func) = NULL; + network_socket_retval_t retval = NETWORK_SOCKET_SUCCESS; + + if (!con->plugin_con_state && con->proxy_state == ST_PROXY_QUIT) + return retval; + + func = con->plugins.con_cleanup; + + if (!func) return retval; + + retval = (*func)(srv, con); + + return retval; +} + +/** + * call the timeout callback for the current connection + * + * @param srv global context + * @param con connection context + * + * @return NETWORK_SOCKET_SUCCESS on success + */ +static network_socket_retval_t +plugin_call_timeout(chassis *srv, network_mysqld_con *con) +{ + NETWORK_MYSQLD_PLUGIN_FUNC(func) = NULL; + network_socket_retval_t retval = NETWORK_SOCKET_ERROR; + + func = con->plugins.con_timeout; + + if (!func) { + /* default implementation */ + g_debug("%s: connection between %s and %s timed out. closing it", + G_STRLOC, + con->client->src->name->str, + con->server ? con->server->dst->name->str : "(server)"); + con->prev_state = con->state; + con->state = ST_ERROR; + return NETWORK_SOCKET_SUCCESS; + } + + if (!con->plugin_con_state && con->proxy_state == ST_PROXY_QUIT) { + g_critical("%s: %p quit because of proxy state", G_STRLOC, con); + return NETWORK_SOCKET_SUCCESS; + } + + retval = (*func)(srv, con); + + return retval; +} + + +chassis_private *network_mysqld_priv_init(void) { + chassis_private *priv; + + priv = g_new0(chassis_private, 1); + + priv->cons = g_ptr_array_new(); + priv->backends = network_backends_new(); + priv->users = cetus_users_new(); + priv->monitor = cetus_monitor_new(); + return priv; +} + +void network_mysqld_priv_shutdown(chassis *chas, chassis_private *priv) { + int i, len; + + if (!priv) return; + + len = priv->cons->len; + for (i = 0; i < len; i++) { + network_mysqld_con *con = g_ptr_array_index(priv->cons, i); + con->server_to_be_closed = 1; + plugin_call_cleanup(chas, con); + con->proxy_state = ST_PROXY_QUIT; + g_debug("%s: %p set proxy state ST_PROXY_QUIT", + G_STRLOC, con); + } +} + +void network_mysqld_priv_finally_free_shared(chassis *chas, + chassis_private *priv) +{ + int i, len; + + if (!priv) return; + + len = priv->cons->len; + + for (i = 0; i < len; i++) { + network_mysqld_con *con = g_ptr_array_index(priv->cons, i); + g_debug("%s: %p finally release, total:%d", G_STRLOC, con, len); + network_mysqld_con_free(con); + } +} + +void network_mysqld_priv_free(chassis G_GNUC_UNUSED *chas, + chassis_private *priv) +{ + if (!priv) return; + + g_ptr_array_free(priv->cons, TRUE); + + network_backends_free(priv->backends); + cetus_users_free(priv->users); + g_free(priv->stats_variables); + cetus_monitor_free(priv->monitor); + g_free(priv); +} + +int network_mysqld_init(chassis *srv) { + srv->priv_free = network_mysqld_priv_free; + srv->priv_shutdown = network_mysqld_priv_shutdown; + srv->priv_finally_free_shared = network_mysqld_priv_finally_free_shared; + srv->priv = network_mysqld_priv_init(); + + cetus_users_read_json(srv->priv->users, srv->config_manager); + cetus_monitor_register_object(srv->priv->monitor, "users", + cetus_users_reload_callback, srv->priv->users); + cetus_variables_init_stats(&srv->priv->stats_variables, srv); + + return 0; +} + +/** + * create a connection + * + * @return a connection context + */ +network_mysqld_con *network_mysqld_con_new() { + network_mysqld_con *con; + + con = g_new0(network_mysqld_con, 1); + con->parse.command = -1; + + con->max_retry_serv_cnt = 72; + con->auth_switch_to_method = g_string_new(NULL); + con->auth_switch_to_round = 0; + con->auth_switch_to_data = g_string_new(NULL); + con->is_auto_commit = 1; + + con->orig_sql = g_string_new(NULL); + + con->connect_timeout.tv_sec = 2 * SECONDS; + con->connect_timeout.tv_usec = 0; + + con->read_timeout.tv_sec = 10 * MINUTES; + con->read_timeout.tv_usec = 0; + + con->write_timeout.tv_sec = 10 * MINUTES; + con->write_timeout.tv_usec = 0; + + con->wait_clt_next_sql.tv_sec = 0; + con->wait_clt_next_sql.tv_usec = 256 * 1000; +#ifdef NETWORK_DEBUG_TRACE_STATE_CHANGES + con->recent_queries = query_queue_new(20); +#endif + return con; +} + +void network_mysqld_add_connection(chassis *srv, network_mysqld_con *con, gboolean listen) +{ + con->srv = srv; + + g_ptr_array_add(srv->priv->cons, con); + if (listen) { + srv->priv->listen_conns = g_list_append(srv->priv->listen_conns, con); + } +} + +static void cetus_clean_conn_data(network_mysqld_con *con) +{ + merge_parameters_t *data = con->data; + + if (data->heap) { + g_free(data->heap); + } + + if (data->elements) { + g_free(data->elements); + } + + if (data->candidates) { + g_free(data->candidates); + } + + if (data->recv_queues) { + g_ptr_array_free(data->recv_queues, TRUE); + } + + g_free(con->data); + con->data = NULL; +} + +/** + * free a connection + * + * closes the client and server sockets + * + * @param con connection context + */ +void network_mysqld_con_free(network_mysqld_con *con) { + if (!con) return; + + g_debug("%s: connections total: %d, free con:%p", + G_STRLOC, con->srv->priv->cons->len, con); + + if (con->parse.data && con->parse.data_free) { + con->parse.data_free(con->parse.data); + } + + if (con->servers != NULL) { + g_warning("%s: servers are not null for con:%p", G_STRLOC, con); + } + + if (con->server) network_socket_free(con->server); + if (con->client) network_socket_free(con->client); + + if (con->hav_condi.condition_value) { + g_free(con->hav_condi.condition_value); + con->hav_condi.condition_value = NULL; + } + + if (con->modified_sql) { + g_string_free(con->modified_sql, TRUE); + } + + g_string_free(con->orig_sql, TRUE); + + if (con->data) { + cetus_clean_conn_data(con); + } + + if (con->sharding_plan) { + sharding_plan_free(con->sharding_plan); + } + g_string_free(con->auth_switch_to_method, TRUE); + g_string_free(con->auth_switch_to_data, TRUE); + + /* we are still in the conns-array */ + + g_ptr_array_remove_fast(con->srv->priv->cons, con); + con->srv->priv->listen_conns = + g_list_remove(con->srv->priv->listen_conns, con); + con->srv->allow_new_conns = TRUE; + +#ifdef NETWORK_DEBUG_TRACE_STATE_CHANGES + query_queue_free(con->recent_queries); +#endif + g_free(con); +} + +static struct timeval +network_mysqld_con_retry_timeout(network_mysqld_con *con) +{ + /* + *retry count 1 2 3 4... 8 9 10 11 12 ... + *timeout (ms) 20 30 40 50... 90 10 10 10 10 ... + */ + static struct timeval min_interval = {0, 10 * 1000}; /* 10ms */ + + struct timeval timeout = min_interval; + int cnt = con->retry_serv_cnt; + if (cnt <= 8) { + timeout.tv_usec += cnt * 10000; + } + return timeout; +} + +int network_mysqld_queue_reset(network_socket *sock) { + sock->packet_id_is_reset = TRUE; + + return 0; +} + + +static int network_mysqld_con_send_command(network_socket *con, int cmd, + const char *arg) +{ + GString *packet = g_string_new(NULL); + + network_mysqld_proto_append_int8(packet, cmd); + network_mysqld_proto_append_lenenc_str(packet, arg); + + network_queue_append(con->send_queue, packet); + g_string_free(packet, TRUE); + + return 0; +} + +/** + * appends a raw MySQL packet to the queue + * + * the packet is append the queue directly and shouldn't be used by the caller + * afterwards anymore and has to by in the MySQL Packet format + * + */ +int network_mysqld_queue_append_raw(network_socket *sock, + network_queue *queue, GString *data) +{ + guint32 packet_len; + guint8 packet_id; + + /* check that the length header is valid */ + if (queue != sock->send_queue && + queue != sock->recv_queue) { + g_critical("%s: queue = %p doesn't belong to sock %p", + G_STRLOC, + (void *)queue, + (void *)sock); + return -1; + } + + g_assert_cmpint(data->len, >=, 4); + + packet_len = network_mysqld_proto_get_packet_len(data); + packet_id = network_mysqld_proto_get_packet_id(data); + + g_assert_cmpint(packet_len, ==, data->len - 4); + + if (sock->packet_id_is_reset) { + /* the ->last_packet_id is undefined, accept what we get */ + sock->last_packet_id = packet_id; + g_debug("%s: set server pack id: %d", G_STRLOC, sock->last_packet_id); + sock->packet_id_is_reset = FALSE; + } else if (packet_id != (guint8) (sock->last_packet_id + 1)) { + sock->last_packet_id++; + g_debug("%s: server pack id ++: %d", G_STRLOC, sock->last_packet_id); + network_mysqld_proto_set_packet_id(data, sock->last_packet_id); + } else { + sock->last_packet_id++; + g_debug("%s: server pack id ++: %d", G_STRLOC, sock->last_packet_id); + } + + network_queue_append(queue, data); + + return 0; +} + +/** + * appends a payload to the queue + * + * the packet is copied and prepended with the mysql packet header + * before it is appended to the queue if neccesary the payload is + * spread over multiple mysql packets + */ +int network_mysqld_queue_append(network_socket *sock, network_queue *queue, + const char *data, size_t packet_len) +{ + gsize packet_offset = 0; + + do { + GString *s; + gsize cur_packet_len = MIN(packet_len, PACKET_LEN_MAX); + + s = g_string_sized_new(packet_len + 4); + + if (sock->packet_id_is_reset) { + sock->packet_id_is_reset = FALSE; + /** the ++last_packet_id will make sure we send a 0 */ + sock->last_packet_id = 0xff; + } + + network_mysqld_proto_append_packet_len(s, cur_packet_len); + network_mysqld_proto_append_packet_id(s, ++sock->last_packet_id); + + g_string_append_len(s, data + packet_offset, cur_packet_len); + + network_queue_append(queue, s); + + if (packet_len == PACKET_LEN_MAX) { + s = g_string_sized_new(4); + + network_mysqld_proto_append_packet_len(s, 0); + network_mysqld_proto_append_packet_id(s, ++sock->last_packet_id); + g_debug("%s: server pack id ++: %d", G_STRLOC, + sock->last_packet_id); + + network_queue_append(queue, s); + } + + packet_len -= cur_packet_len; + packet_offset += cur_packet_len; + } while (packet_len > 0); + + return 0; +} + + +/** + * create a OK packet and append it to the send-queue + * + * @param con a client socket + * @param affected_rows affected rows + * @param insert_id insert_id + * @param server_status server_status (bitfield of SERVER_STATUS_*) + * @param warnings number of warnings to fetch with SHOW WARNINGS + * @return 0 + * + * @todo move to network_mysqld_proto + */ +int network_mysqld_con_send_ok_full(network_socket *con, guint64 affected_rows, + guint64 insert_id, guint16 server_status, guint16 warnings) +{ + GString *packet = g_string_new(NULL); + network_mysqld_ok_packet_t *ok_packet; + + ok_packet = network_mysqld_ok_packet_new(); + ok_packet->affected_rows = affected_rows; + ok_packet->insert_id = insert_id; + ok_packet->server_status = server_status; + g_debug("%s: server status: %d", G_STRLOC, server_status); + ok_packet->warnings = warnings; + + network_mysqld_proto_append_ok_packet(packet, ok_packet); + + network_mysqld_queue_append(con, con->send_queue, S(packet)); + network_mysqld_queue_reset(con); + + g_string_free(packet, TRUE); + network_mysqld_ok_packet_free(ok_packet); + + return 0; +} + +/** + * send a simple OK packet + * + * - no affected rows + * - no insert-id + * - AUTOCOMMIT + * - no warnings + * + * @param con a client socket + */ +int network_mysqld_con_send_ok(network_socket *con) { + return network_mysqld_con_send_ok_full(con, 0, 0, + SERVER_STATUS_AUTOCOMMIT, 0); +} + +static int network_mysqld_con_send_error_full_all(network_socket *con, + const char *errmsg, gsize errmsg_len, guint errorcode, + const gchar *sqlstate, gboolean is_41_protocol) +{ + GString *packet; + network_mysqld_err_packet_t *err_packet; + + packet = g_string_sized_new(10 + errmsg_len); + + err_packet = is_41_protocol ? network_mysqld_err_packet_new() : + network_mysqld_err_packet_new_pre41(); + err_packet->errcode = errorcode; + if (errmsg) g_string_assign_len(err_packet->errmsg, errmsg, errmsg_len); + if (sqlstate) { + g_string_assign_len(err_packet->sqlstate, sqlstate, strlen(sqlstate)); + } + + network_mysqld_proto_append_err_packet(packet, err_packet); + + network_mysqld_queue_append(con, con->send_queue, S(packet)); + network_mysqld_queue_reset(con); + + network_mysqld_err_packet_free(err_packet); + g_string_free(packet, TRUE); + + return 0; +} + +/** + * send a error packet to the client connection + * + * @note the sqlstate has to match the SQL standard. + * If no matching SQL state is known, leave it at NULL + * + * @param con the client connection + * @param errmsg the error message + * @param errmsg_len byte-len of the error-message + * @param errorcode mysql error-code we want to send + * @param sqlstate if none-NULL, 5-char SQL state to send, + * if NULL, default SQL state is used + * + * @return 0 on success + */ +int network_mysqld_con_send_error_full(network_socket *con, const char *errmsg, + gsize errmsg_len, guint errorcode, const gchar *sqlstate) +{ + return network_mysqld_con_send_error_full_all(con, errmsg, errmsg_len, + errorcode, sqlstate, TRUE); +} + + +/** + * send a error-packet to the client connection + * + * errorcode is 1000, sqlstate is NULL + * + * @param con the client connection + * @param errmsg the error message + * @param errmsg_len byte-len of the error-message + * + * @see network_mysqld_con_send_error_full + */ +int network_mysqld_con_send_error(network_socket *con, const char *errmsg, + gsize errmsg_len) +{ + return network_mysqld_con_send_error_full(con, errmsg, + errmsg_len, ER_UNKNOWN_ERROR, NULL); +} + +/** + * send a error packet to the client connection (pre-4.1 protocol) + * + * @param con the client connection + * @param errmsg the error message + * @param errmsg_len byte-len of the error-message + * @param errorcode mysql error-code we want to send + * + * @return 0 on success + */ +int network_mysqld_con_send_error_pre41_full(network_socket *con, + const char *errmsg, gsize errmsg_len, guint errorcode) +{ + return network_mysqld_con_send_error_full_all(con, errmsg, errmsg_len, + errorcode, NULL, FALSE); +} + +/** + * send a error-packet to the client connection (pre-4.1 protocol) + * + * @param con the client connection + * @param errmsg the error message + * @param errmsg_len byte-len of the error-message + * + * @see network_mysqld_con_send_error_pre41_full + */ +int network_mysqld_con_send_error_pre41(network_socket *con, + const char *errmsg, gsize errmsg_len) +{ + return network_mysqld_con_send_error_pre41_full(con, errmsg, + errmsg_len, ER_UNKNOWN_ERROR); +} + + +/** + * get a full packet from the raw queue and move it to the packet queue + */ +network_socket_retval_t +network_mysqld_con_get_packet(chassis *chas, network_socket *con) +{ + GString *packet; + GString header; + char header_str[NET_HEADER_SIZE + 1] = {0}; + guint32 packet_len; + guint8 packet_id; + + network_queue *recv_queue_raw; + if (con->do_compress) { + g_debug("%s:queue from recv_queue_uncompress_raw:%p", G_STRLOC, con); + recv_queue_raw = con->recv_queue_uncompress_raw; + } else { + recv_queue_raw = con->recv_queue_raw; + } + + /** + * read the packet header (4 bytes) + */ + header.str = header_str; + header.allocated_len = sizeof(header_str); + header.len = 0; + + /* read the packet len if the leading packet */ + if (!network_queue_peek_str(recv_queue_raw, NET_HEADER_SIZE, + &header)) + { + g_debug("%s:wait for event", G_STRLOC); + /* too small */ + return NETWORK_SOCKET_WAIT_FOR_EVENT; + } + + packet_len = network_mysqld_proto_get_packet_len(&header); + if (packet_len > chas->cetus_max_allowed_packet) { + g_message("packet len: %d excess max_allowed_packet: %d", + packet_len, chas->cetus_max_allowed_packet); + return NETWORK_SOCKET_ERROR; + } + packet_id = network_mysqld_proto_get_packet_id(&header); + + /* move the packet from the raw queue to the recv-queue */ + if ((packet = network_queue_pop_str(recv_queue_raw, + packet_len + NET_HEADER_SIZE, NULL))) + { +#if NETWORK_DEBUG_TRACE_IO + g_debug("%s:output for sock:%p", G_STRLOC, con); + /* to trace the data we received from the socket, enable this */ + g_debug_hexdump(G_STRLOC, S(packet)); +#endif + + if (con->packet_id_is_reset) { + con->last_packet_id = packet_id; + con->packet_id_is_reset = FALSE; + } else if (packet_id != (guint8)(con->last_packet_id + 1)) { + g_message("%s: recv pack-id %d, but expected %d ... out of sync", + G_STRLOC, packet_id, + con->last_packet_id + 1); + g_string_free(packet, TRUE); + + return NETWORK_SOCKET_ERROR; + } else { + con->last_packet_id = packet_id; + } + + network_queue_append(con->recv_queue, packet); + } else { + g_debug("%s:wait for event", G_STRLOC); + return NETWORK_SOCKET_WAIT_FOR_EVENT; + } + + return NETWORK_SOCKET_SUCCESS; +} + +network_socket_retval_t +network_mysqld_con_get_uncompressed_packet(chassis *chas, network_socket *con) +{ + GString *packet; + GString header; + char header_str[NET_HEADER_SIZE + COMP_HEADER_SIZE + 1] = {0}; + guint32 packet_len; + + int header_length = NET_HEADER_SIZE + COMP_HEADER_SIZE; + + /** + * read the packet header (4 bytes) + */ + header.str = header_str; + header.allocated_len = sizeof(header_str); + header.len = 0; + + /* read the packet len if the leading packet */ + if (!network_queue_peek_str(con->recv_queue_raw, + header_length, &header)) + { + /* too small */ + return NETWORK_SOCKET_WAIT_FOR_EVENT; + } + + packet_len = network_mysqld_proto_get_packet_len(&header); + if (packet_len > chas->cetus_max_allowed_packet) { + g_message("packet len: %d excess max_allowed_packet: %d", + packet_len, chas->cetus_max_allowed_packet); + return NETWORK_SOCKET_ERROR; + } + + /* move the packet from the raw queue to the recv-queue */ + if ((packet = network_queue_pop_str(con->recv_queue_raw, + packet_len + NET_HEADER_SIZE + COMP_HEADER_SIZE, NULL))) + { +#if NETWORK_DEBUG_TRACE_IO + g_debug("%s:output for sock:%p", G_STRLOC, con); + /* to trace the data we received from the socket, enable this */ + g_debug_hexdump(G_STRLOC, S(packet)); +#endif + unsigned char *info = (unsigned char *) packet->str + NET_HEADER_SIZE; + int uncompressed_len = info[0] | info[1] << 8 | info[2] << 16; + g_debug("%s: do uncompress here, com len:%d, uncompress len:%d", + G_STRLOC, packet_len, uncompressed_len); + + GString *uncompressed_packet; + if (uncompressed_len == 0) { + uncompressed_len = packet_len; + uncompressed_packet = g_string_sized_new(uncompressed_len); + g_string_append_len(uncompressed_packet, (char *) (info + COMP_HEADER_SIZE), + uncompressed_len); + } else { + uncompressed_packet = g_string_sized_new(uncompressed_len); + cetus_uncompress(uncompressed_packet, + (unsigned char *) packet->str + header_length, packet_len); + g_debug("%s:call cetus_uncompress for con:%p", G_STRLOC, con); + } + + network_queue_append(con->recv_queue_uncompress_raw, uncompressed_packet); + g_string_free(packet, TRUE); + } else { + return NETWORK_SOCKET_WAIT_FOR_EVENT; + } + + return NETWORK_SOCKET_SUCCESS; +} + + +static void disp_err_packet(network_mysqld_con *con, network_packet *packet) +{ + int offset = packet->offset; + network_mysqld_err_packet_t *err_packet; + + packet->offset = NET_HEADER_SIZE; + err_packet = network_mysqld_err_packet_new(); + + if (!network_mysqld_proto_get_err_packet(packet, err_packet)) { + g_message("%s:clt:%s,src:%s,dst:%s,db:%s,%s,\ + error code:%d,errmsg:%s,sqlstate:%s, con:%p", + G_STRLOC, con->client->src->name->str, + con->server->src->name->str, + con->server->dst->name->str, + con->server->default_db->str, + con->orig_sql->str, + (int) err_packet->errcode, + err_packet->errmsg->str, + err_packet->sqlstate->str, + con); + if (con->dist_tran) { + int checked = 0; + switch (err_packet->errcode) { + case ER_XA_RBROLLBACK: + case ER_XA_RBDEADLOCK: + case ER_XA_RBTIMEOUT: + case ER_LOCK_WAIT_TIMEOUT: + case ER_LOCK_DEADLOCK: + g_message("%s: errcode for con:%p, xid:%s, errcode:%d", + G_STRLOC, con, con->xid_str, err_packet->errcode); + con->dist_tran_failed = 1; + con->xa_query_status_error_and_abort = 1; + con->is_commit_or_rollback = 1; + checked = 1; + break; + case ER_DUP_ENTRY: + checked = 1; + break; + case ER_XAER_DUPID: + g_critical("%s: dup xid for con:%p, xid:%s, xa state:%d", + G_STRLOC, con, con->xid_str, con->dist_tran_state); + break; + default: + g_warning("%s: errcode for con:%p, xid:%s, errcode:%d", + G_STRLOC, con, con->xid_str, err_packet->errcode); + break; + } + + if (!checked) { + if (strncasecmp(err_packet->sqlstate->str, "XA", 2) == 0) { + g_message("%s: query status error, xid:%s for con:%p", + G_STRLOC, con->xid_str, con); + con->dist_tran_failed = 1; + con->xa_query_status_error_and_abort = 1; + } + } + } + } else { + g_message("%s:clt:%s,src:%s,dst:%s,db:%s,%s", + G_STRLOC, con->client->src->name->str, + con->server->src->name->str, + con->server->dst->name->str, + con->server->default_db->str, con->orig_sql->str); + } + + network_mysqld_err_packet_free(err_packet); + packet->offset = offset; + + return; +} + + +network_socket_retval_t +network_mysqld_read_mul_packets(chassis G_GNUC_UNUSED *chas, + network_mysqld_con *con, network_socket *server, int *is_finished) +{ + int count = 0; + network_socket_retval_t ret; + + if (*is_finished) { + g_message("%s: allready finished for server:%p", G_STRLOC, server); + } + *is_finished = 0; + + g_debug("%s:server to read:%d, packet id:%d for con:%p", + G_STRLOC, (int) server->to_read, server->last_packet_id, con); + + off_t to_read = server->to_read; + + server->max_header_size_reached = 0; + switch (network_socket_read(server)) { + case NETWORK_SOCKET_SUCCESS: + break; + case NETWORK_SOCKET_WAIT_FOR_EVENT: + return NETWORK_SOCKET_WAIT_FOR_EVENT; + case NETWORK_SOCKET_ERROR: + return NETWORK_SOCKET_ERROR; + case NETWORK_SOCKET_ERROR_RETRY: + g_error("NETWORK_SOCKET_ERROR_RETRY wasn't expected"); + break; + } + + server->is_waiting = 0; + server->resp_len += to_read; + enum enum_server_command orig_command = con->parse.command; + if (con->attr_adj_state == ATTR_DIF_CHANGE_USER) { + con->parse.command = COM_CHANGE_USER; + g_debug("%s: set command COM_CHANGE_USER, attr adj:%d for con:%p", + G_STRLOC, con->attr_adj_state, con); + } else if (con->attr_adj_state == ATTR_DIF_DEFAULT_DB) { + con->parse.command = COM_INIT_DB; + g_debug("%s: set command COM_INIT_DB", G_STRLOC); + } else if (con->attr_adj_state == ATTR_DIF_SET_OPTION) { + con->parse.command = COM_SET_OPTION; + } + + network_socket *orig_server = con->server; + + if (con->parse.command == COM_QUERY) { + network_mysqld_com_query_result_t *com_query = con->parse.data; + int qs_state = server->parse.qs_state; + com_query->state = qs_state; + } + + if (server->do_compress) { + ret = network_mysqld_con_get_uncompressed_packet(chas, server); + if (ret == NETWORK_SOCKET_SUCCESS) { + ret = network_mysqld_con_get_packet(chas, server); + } + } else { + ret = network_mysqld_con_get_packet(chas, server); + } + + while (ret == NETWORK_SOCKET_SUCCESS) { + count++; + network_packet packet; + GList *chunk; + + chunk = server->recv_queue->chunks->tail; + packet.data = chunk->data; + packet.offset = 0; + + con->server = server; + *is_finished = network_mysqld_proto_get_query_result(&packet, con); + if (*is_finished == 1) { + g_debug("%s:packets read finished:%d, default db:%s, server db:%s", + G_STRLOC, count, + con->client->default_db->str, server->default_db->str); + if (con->parse.command == COM_QUERY) { + network_mysqld_com_query_result_t *query = con->parse.data; + if (query && query->query_status == MYSQLD_PACKET_ERR) { + disp_err_packet(con, &packet); + } + + if (query->warning_count > 0) { + g_critical("%s warning flag from server:%s is met:%s", + G_STRLOC, server->dst->name->str, con->orig_sql->str); + con->last_warning_met = 1; + } + } + break; + } + + ret = network_mysqld_con_get_packet(chas, server); + + if (ret == NETWORK_SOCKET_WAIT_FOR_EVENT) { + if (server->do_compress) { + ret = network_mysqld_con_get_uncompressed_packet(chas, server); + if (ret != NETWORK_SOCKET_SUCCESS) { + break; + } + } + } + } + + if (con->parse.command == COM_QUERY) { + network_mysqld_com_query_result_t *com_query = con->parse.data; + server->parse.qs_state = com_query->state; + } + + if (server->resp_len > con->srv->max_header_size) { + server->max_header_size_reached = 1; + g_debug("%s: reach max header size", G_STRLOC); + } + + g_debug("%s:after mul read, state:%d, ret:%d for con:%p", + G_STRLOC, server->parse.qs_state, ret, con); + + con->parse.command = orig_command; + + if ((*is_finished == 0) && ret == NETWORK_SOCKET_SUCCESS) { + g_message("%s: not finished for server:%p, orig server:%p", + G_STRLOC, con->server, orig_server); + } + + con->server = orig_server; + + return ret; +} + +/** + * read a MySQL packet from the socket + * + * the packet is added to the con->recv_queue and contains a full mysql packet + * with packet-header and everything + */ +network_socket_retval_t network_mysqld_read(chassis G_GNUC_UNUSED *chas, + network_socket *sock) +{ + switch (network_socket_read(sock)) { + case NETWORK_SOCKET_WAIT_FOR_EVENT: + return NETWORK_SOCKET_WAIT_FOR_EVENT; + case NETWORK_SOCKET_ERROR: + return NETWORK_SOCKET_ERROR; + case NETWORK_SOCKET_SUCCESS: + break; + case NETWORK_SOCKET_ERROR_RETRY: + g_error("NETWORK_SOCKET_ERROR_RETRY wasn't expected"); + break; + } + + if (sock->do_compress) { + int ret = network_mysqld_con_get_uncompressed_packet(chas, sock); + if (ret != NETWORK_SOCKET_SUCCESS) { + if (sock->recv_queue_uncompress_raw->len == 0) { + return ret; + } + } + } + + return network_mysqld_con_get_packet(chas, sock); +} + +network_socket_retval_t network_mysqld_write(chassis G_GNUC_UNUSED *chas, + network_socket *sock) +{ + network_socket_retval_t ret; + + ret = network_socket_write(sock, -1); + + return ret; +} + +/** + * call the hooks of the plugins for each state + * + * if the plugin doesn't implement a hook, we provide a default operation + * + * @param srv the global context + * @param con the connection context + * @param state state to handle + * @return NETWORK_SOCKET_SUCCESS on success + */ +network_socket_retval_t +plugin_call(chassis *srv, network_mysqld_con *con, int state) +{ + network_socket_retval_t ret; + NETWORK_MYSQLD_PLUGIN_FUNC(func) = NULL; + + if (!con->plugin_con_state && con->proxy_state == ST_PROXY_QUIT) { + g_critical("%s: %p quit because of proxy state not zero", + G_STRLOC, con); + return NETWORK_SOCKET_SUCCESS; + } + + switch (state) { + case ST_INIT: + func = con->plugins.con_init; + + if (!func) { + con->state = ST_CONNECT_SERVER; + } + break; + case ST_CONNECT_SERVER: + func = con->plugins.con_connect_server; + break; + case ST_SEND_HANDSHAKE: + func = con->plugins.con_send_handshake; + + if (!func) { + con->state = ST_READ_AUTH; + } + + break; + case ST_READ_AUTH: + func = con->plugins.con_read_auth; + + break; + case ST_SEND_AUTH_RESULT: + /* called after the auth data is sent to the client */ + func = con->plugins.con_send_auth_result; + + if (!func) { + /* + * figure out what to do next: + * - switch to 'read command from client' + * - close connection + * - read auth-data from client + * - read another auth-result packet from server + */ + switch (con->auth_result_state) { + case MYSQLD_PACKET_OK: + if (con->login_failed) { + g_message("%s: clt login failed:%p", G_STRLOC, con); + } else { + /* + * OK, delivered to client, + * switch to command phase + */ + con->state = ST_READ_QUERY; + if (con->is_client_compressed) { + con->client->do_compress = 1; + network_socket_set_send_buffer_size(con->client, + COMPRESS_BUF_SIZE); + } + } + break; + case MYSQLD_PACKET_ERR: + /* ERR delivered to client, close the conn now */ + con->prev_state = con->state; + con->state = ST_ERROR; + break; + default: + g_debug("%s: unexpected st for SEND_AUTH_RESULT: %02x", + G_STRLOC, con->auth_result_state); + con->prev_state = con->state; + con->state = ST_ERROR; + break; + } + } + break; + case ST_READ_QUERY: + func = con->plugins.con_read_query; + break; + case ST_GET_SERVER_CONNECTION_LIST: + func = con->plugins.con_get_server_conn_list; + break; + case ST_READ_QUERY_RESULT: + func = con->plugins.con_read_query_result; + break; + case ST_SEND_QUERY_RESULT: + func = con->plugins.con_send_query_result; + + if (!func) { + if (!con->server_to_be_closed) { + if (con->resp_too_long) { + g_message("%s: strange here for con:%p", + G_STRLOC, con); + } + con->state = ST_READ_QUERY; + g_debug("%s: set ST_READ_QUERY for con:%p", + G_STRLOC, con); + } else { + con->state = ST_CLOSE_SERVER; + g_debug("%s: set ST_CLOSE_SERVER for con:%p", + G_STRLOC, con); + } + } + break; + + case ST_ERROR: + g_debug("%s: not executing plugin func in state ST_ERROR", + G_STRLOC); + return NETWORK_SOCKET_SUCCESS; + default: + g_error("%s: unhandled state: %d", G_STRLOC, state); + break; + } + + if (!func) return NETWORK_SOCKET_SUCCESS; + + if (!con->plugin_con_state && con->proxy_state == ST_PROXY_QUIT) { + g_critical("%s: %p quit because of proxy state not zero", + G_STRLOC, con); + return NETWORK_SOCKET_SUCCESS; + } + + ret = (*func)(srv, con); + + return ret; +} + +/** + * reset the command-response parsing + * + * some commands needs state information and we have to + * reset the parsing as soon as we add a new command to the send-queue + */ +void network_mysqld_con_reset_command_response_state(network_mysqld_con *con) { + con->parse.command = -1; + if (con->parse.data && con->parse.data_free) { + con->parse.data_free(con->parse.data); + + con->parse.data = NULL; + con->parse.data_free = NULL; + } +} + +/** + * reset per-query states + */ +void network_mysqld_con_reset_query_state(network_mysqld_con *con) +{ + con->sql_modified = 0; + con->hav_condi.rel_type = 0; + if (con->hav_condi.condition_value) { + g_free(con->hav_condi.condition_value); + con->hav_condi.condition_value = NULL; + } + + if (con->modified_sql) { + g_string_free(con->modified_sql, TRUE); + con->modified_sql = NULL; + } + g_string_truncate(con->orig_sql, 0); +} + +/** + * get the name of a connection state + */ +const char * +network_mysqld_con_st_name(network_mysqld_con_state_t state) +{ + switch (state) { + case ST_INIT: return "ST_INIT"; + case ST_CONNECT_SERVER: return "ST_CONNECT_SERVER"; + case ST_SEND_HANDSHAKE: return "ST_SEND_HANDSHAKE"; + case ST_READ_AUTH: return "ST_READ_AUTH"; + case ST_SEND_AUTH_RESULT: return "ST_SEND_AUTH_RESULT"; + case ST_READ_QUERY: return "ST_READ_QUERY"; + case ST_GET_SERVER_CONNECTION_LIST: return "ST_GET_SERVER_CONNECTION_LIST"; + case ST_READ_M_QUERY_RESULT: return "ST_READ_M_QUERY_RESULT"; + case ST_SEND_QUERY: return "ST_SEND_QUERY"; + case ST_READ_QUERY_RESULT: return "ST_READ_QUERY_RESULT"; + case ST_SEND_QUERY_RESULT: return "ST_SEND_QUERY_RESULT"; + + case ST_CLIENT_QUIT: return "ST_CLIENT_QUIT"; + case ST_CLOSE_CLIENT: return "ST_CLOSE_CLIENT"; + case ST_CLOSE_SERVER: return "ST_CLOSE_SERVER"; + case ST_ERROR: return "ST_ERROR"; + case ST_SEND_ERROR: return "ST_SEND_ERROR"; + } + + return "unknown"; +} + +static void +check_query_status(network_mysqld_con *con, network_socket *server, + network_mysqld_com_query_result_t *com_query) +{ + g_debug("%s: visit check_query_status:%p", G_STRLOC, con); + if (!com_query || com_query->query_status != MYSQLD_PACKET_OK) { + g_debug("%s: no check for query status", G_STRLOC); + return; + } + if (com_query->server_status & SERVER_STATUS_IN_TRANS) { + con->is_in_transaction = 1; + server->is_in_tran_context = 1; + } else { + con->is_in_transaction = 0; + server->is_in_tran_context = 0; + g_debug("%s: set is_in_transaction false", G_STRLOC); + } + + if (!con->is_in_transaction) { + if (!con->is_auto_commit) { + g_debug("%s: set is_in_transaction true here:%p", G_STRLOC, con); + con->is_in_transaction = 1; + con->client->is_server_conn_reserved = 1; + } else { + if (!con->is_prepared && !con->is_in_sess_context && !con->last_warning_met) { + con->client->is_server_conn_reserved = 0; + g_debug("%s: set is_server_conn_reserved false:%p", G_STRLOC, con); + } else { + con->client->is_server_conn_reserved = 1; + g_debug("%s: set is_server_conn_reserved true:%p", G_STRLOC, con); + } + } + } else { + con->client->is_server_conn_reserved = 1; + g_debug("%s: is_in_transaction true here:%p", G_STRLOC, con); + } + + if (com_query->insert_id > 0) { + con->last_insert_id = com_query->insert_id; + g_debug("%s: set last insert id:%llu", G_STRLOC, + (unsigned long long) con->last_insert_id); + } +} + +void set_conn_attr(network_mysqld_con *con, network_socket *server) +{ + enum enum_server_command cur_command = con->parse.command; + if (con->attr_adj_state == ATTR_DIF_CHANGE_USER) { + cur_command = COM_CHANGE_USER; + } else if (con->attr_adj_state == ATTR_DIF_DEFAULT_DB) { + cur_command = COM_INIT_DB; + } else if (con->attr_adj_state == ATTR_DIF_SET_OPTION) { + cur_command = COM_SET_OPTION; + } + + switch(cur_command) { + case COM_QUERY: + check_query_status(con, server, con->parse.data); + break; + case COM_CHANGE_USER: + g_string_assign_len(server->response->username, + S(con->client->response->username)); + g_debug("%s: save username for server:%p, con:%p", G_STRLOC, server, con); + break; + case COM_INIT_DB: + /* TODO: make sure we get OK result packet */ + g_string_assign(server->default_db, con->client->default_db->str); + break; + case COM_SET_OPTION: + break; + default: + g_message("%s: unknown command:%d, con:%p, sql:%s", + G_STRLOC, cur_command, con, con->orig_sql->str); + break; + } +} + +void record_xa_log_for_mending(network_mysqld_con *con, network_socket *sock) +{ + switch(con->dist_tran_state) { + case NEXT_ST_XA_START: + break; + case NEXT_ST_XA_QUERY: + break; + case NEXT_ST_XA_END: + break; + case NEXT_ST_XA_PREPARE: + tc_log_info(LOG_WARN, 0, "XA END %s %s@%u failed", + con->xid_str, sock->dst->name->str, + sock->challenge->thread_id); + break; + case NEXT_ST_XA_COMMIT: + tc_log_info(LOG_WARN, 0, "XA PREPARE %s %s@%u failed", + con->xid_str, sock->dst->name->str, + sock->challenge->thread_id); + break; + case NEXT_ST_XA_ROLLBACK: + break; + case NEXT_ST_XA_CANDIDATE_OVER: + if (con->dist_tran_failed) { + tc_log_info(LOG_WARN, 0, "XA ROLLBACK %s %s@%u failed", + con->xid_str, + sock->dst->name->str, + sock->challenge->thread_id); + } else { + if (con->servers->len == 1) { + tc_log_info(LOG_WARN, 0, + "XA COMMIT %s %s@%u ONE PHASE failed", + con->xid_str, + sock->dst->name->str, + sock->challenge->thread_id); + } else { + tc_log_info(LOG_WARN, 0, "XA COMMIT %s %s@%u failed", + con->xid_str, + sock->dst->name->str, + sock->challenge->thread_id); + } + } + break; + default: + break; + } + + if (con->dist_tran_state <= NEXT_ST_XA_START) { + } else if (con->dist_tran_state >= NEXT_ST_XA_CANDIDATE_OVER) { + g_warning("%s:xa tran, could not recv response:%p", + G_STRLOC, con); + } else { + con->dist_tran_failed = 1; + g_warning("%s:xa tran, set failed here:%p, xa state:%d, xid:%s", + G_STRLOC, con, con->dist_tran_state, con->xid_str); + } +} + +gboolean +shard_set_autocommit(network_mysqld_con *con) +{ + size_t i; + + g_debug("%s: set autocommit here", G_STRLOC); + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + if (!pmd->participated || pmd->attr_consistent) { + continue; + } + + pmd->attr_adjusted_now = 0; + int len = con->client->charset->len + NET_HEADER_SIZE + 1 + 32; + GString *packet = g_string_sized_new(len); + packet->len = NET_HEADER_SIZE; + g_string_append_c(packet, (char) COM_QUERY); + + char *command = "start transaction"; + if (con->is_start_trans_buffered) { + con->is_start_trans_buffered = 0; + g_debug("%s: start transaction for con:%p", G_STRLOC, con); + } else { + con->is_auto_commit_trans_buffered = 0; + } + g_string_append(packet, command); + + network_mysqld_proto_set_packet_len(packet, 1 + strlen(command)); + network_mysqld_proto_set_packet_id(packet, 0); + g_queue_push_tail(pmd->server->send_queue->chunks, packet); + + pmd->server->parse.qs_state = PARSE_COM_QUERY_INIT; + pmd->attr_adjusted_now = 1; + + con->resp_expected_num++; + } + + return TRUE; +} + +gboolean shard_set_multi_stmt_consistant(network_mysqld_con *con) +{ + enum enum_server_command command = con->parse.command; + + if (command == COM_SET_OPTION) { + return TRUE; + } + + size_t i; + gboolean result = TRUE; + + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + if (!pmd->participated || pmd->attr_consistent) { + continue; + } + + pmd->attr_adjusted_now = 0; + if ((pmd->attr_diff & ATTR_DIF_SET_OPTION) == 0) { + continue; + } + if (con->client->is_multi_stmt_set != pmd->server->is_multi_stmt_set) { + int len = NET_HEADER_SIZE + 12; + GString *new_packet = g_string_sized_new(len); + new_packet->len = NET_HEADER_SIZE; + g_string_append_c(new_packet, (char) COM_SET_OPTION); + if (con->client->is_multi_stmt_set) { + g_string_append_c(new_packet, (char) 0); + } else { + g_string_append_c(new_packet, (char) 1); + } + g_string_append_c(new_packet, (char) 0); + + network_mysqld_proto_set_packet_id(new_packet, 0); + network_mysqld_proto_set_packet_len(new_packet, 1 + 1 + 1); + + g_queue_push_tail(pmd->server->send_queue->chunks, new_packet); + g_debug("%s: adjust multi stmt", G_STRLOC); + + pmd->server->is_multi_stmt_set = con->client->is_multi_stmt_set; + pmd->attr_adjusted_now = 1; + con->resp_expected_num++; + result = FALSE; + } else { + g_warning("%s:set option adjust warning:not consistant", + G_STRLOC); + } + } + + return result; +} + +gboolean +shard_set_charset_consistant(network_mysqld_con *con) +{ + size_t i; + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + if (!pmd->participated || pmd->attr_consistent) { + continue; + } + + pmd->attr_adjusted_now = 0; + if ((pmd->attr_diff & ATTR_DIF_CHARSET) == 0) { + continue; + } + + int len = con->client->charset->len + NET_HEADER_SIZE + 1 + 16; + GString *packet = g_string_sized_new(len); + packet->len = NET_HEADER_SIZE; + g_string_append_c(packet, (char) COM_QUERY); + char *command = "SET NAMES "; + g_string_append(packet, command); + + if (strcmp(con->client->charset->str, "") == 0) { + g_warning("%s: client charset is empty:%s", + G_STRLOC, con->client->src->name->str); + g_string_append(packet, "''"); + network_mysqld_proto_set_packet_len(packet, + 1 + strlen(command) + 2); + } else { + g_string_append(packet, con->client->charset->str); + network_mysqld_proto_set_packet_len(packet, + 1 + strlen(command) + con->client->charset->len); + } + + network_mysqld_proto_set_packet_id(packet, 0); + g_queue_push_tail(pmd->server->send_queue->chunks, packet); + + pmd->attr_adjusted_now = 1; + g_debug("%s: adjust default charset for server, clt:%s, srv:%s", + G_STRLOC, con->client->charset->str, pmd->server->charset->str); + pmd->server->parse.qs_state = PARSE_COM_QUERY_INIT; + con->resp_expected_num++; + + g_string_assign(pmd->server->charset, con->client->charset->str); + } + return TRUE; +} + +gboolean shard_set_default_db_consistant(network_mysqld_con *con) +{ + enum enum_server_command command = con->parse.command; + + if (command == COM_INIT_DB) { + return TRUE; + } + + size_t i; + gboolean result = TRUE; + + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + if (!pmd->participated || pmd->attr_consistent) { + continue; + } + + pmd->attr_adjusted_now = 0; + + if ((pmd->attr_diff & ATTR_DIF_DEFAULT_DB) == 0) { + continue; + } + + GString *clt_default_db = con->client->default_db; + GString *srv_default_db = pmd->server->default_db; + + if (clt_default_db && clt_default_db->len > 0) { + if (!g_string_equal(clt_default_db, srv_default_db)) { + int len = clt_default_db->len + NET_HEADER_SIZE + 1; + GString *new_packet = g_string_sized_new(len); + new_packet->len = NET_HEADER_SIZE; + g_string_append_c(new_packet, (char) COM_INIT_DB); + g_string_append_len(new_packet, S(clt_default_db)); + network_mysqld_proto_set_packet_len(new_packet, + 1 + clt_default_db->len); + network_mysqld_proto_set_packet_id(new_packet, 0); + g_queue_push_tail(pmd->server->send_queue->chunks, new_packet); + pmd->attr_adjusted_now = 1; + g_debug("%s: adjust default db for server, clt:%s, srv:%s", + G_STRLOC, clt_default_db->str, srv_default_db->str); + + con->resp_expected_num++; + + result = FALSE; + } else { + g_warning("%s:default db adjust warning:not consistant", + G_STRLOC); + } + } + } + + return result; +} + +static session_attr_flags_t next_attribute(session_attr_flags_t flags, + session_attr_flags_t attr) +{ + if (flags == 0 || attr == 0) return 0; + uint32_t a = attr << 1; + while (a != 0 && ((a & flags) == 0)) { a <<= 1; } + + if (a > ATTR_DIF_SET_AUTOCOMMIT) { + a = ATTR_START; + } + + return a; +} + +static int +build_attr_statements(network_mysqld_con *con) +{ + g_debug("%s:build_attr_statements here, attr state:%d", + G_STRLOC, con->attr_adj_state); + + con->attr_adj_state = next_attribute(con->unmatched_attribute, + con->attr_adj_state); + + if (con->attr_adj_state == ATTR_DIF_SET_AUTOCOMMIT) { + if (!con->dist_tran && con->delay_send_auto_commit) { + con->delay_send_auto_commit = 0; + g_debug("%s:need to set autocommit for con:%p", G_STRLOC, con); + } else { + con->attr_adj_state = ATTR_START; + } + } + + con->resp_expected_num = 0; + + switch(con->attr_adj_state) { + case ATTR_DIF_DEFAULT_DB: + shard_set_default_db_consistant(con); + break; + case ATTR_DIF_CHARSET: + shard_set_charset_consistant(con); + break; + case ATTR_DIF_SET_OPTION: + shard_set_multi_stmt_consistant(con); + break; + case ATTR_DIF_SET_AUTOCOMMIT: + shard_set_autocommit(con); + break; + case ATTR_START: + break; + default: + g_message("%s:strange attr adj state:%d, conn:%p", + G_STRLOC, con->attr_adj_state, con); + break; + } + + con->state = ST_SEND_QUERY; + + if (con->resp_expected_num == 0) { + con->attr_adj_state = ATTR_START; + } + + g_debug("%s: expected num:%d", G_STRLOC, con->resp_expected_num); + g_debug("%s: attr adj state:%d", G_STRLOC, con->attr_adj_state); + + if (con->attr_adj_state > ATTR_START) { + return 1; + } else { + return 0; + } +} + + +void +shard_build_xa_query(network_mysqld_con *con, server_session_t *pmd) +{ + network_socket *recv_sock = con->client; + GList *chunk = recv_sock->recv_queue->chunks->head; + GString *packet = (GString *)(chunk->data); + + g_debug("%s:packet id:%d when get server", + G_STRLOC, pmd->server->last_packet_id); + + pmd->server->parse.qs_state = PARSE_COM_QUERY_INIT; + + if (con->parse.command == COM_QUERY) { + GString *payload = g_string_new(0); + network_mysqld_proto_append_query_packet(payload, pmd->sql->str); + network_mysqld_queue_reset(pmd->server); + network_mysqld_queue_append(pmd->server, pmd->server->send_queue, + S(payload)); + g_string_free(payload, TRUE); + } else { + network_queue_append(pmd->server->send_queue, + g_string_new_len(packet->str, packet->len)); + } + + pmd->state = NET_RW_STATE_NONE; + + con->is_xa_query_sent = 1; + con->all_participate_num++; + con->resp_expected_num++; +} + +static void +build_xa_command(network_mysqld_con *con, server_session_t *pmd, + int end, char *buffer_log) +{ + char buffer[XA_CMD_BUF_LEN]; + + switch(pmd->dist_tran_state) { + case NEXT_ST_XA_END: + snprintf(buffer, XA_CMD_BUF_LEN, "XA END %s", con->xid_str); + if (con->dist_tran_failed) { + pmd->dist_tran_state = NEXT_ST_XA_ROLLBACK; + con->is_commit_or_rollback = 1; + g_debug("%s: set is_commit_or_rollback when xa end", G_STRLOC); + } else { + pmd->dist_tran_state = NEXT_ST_XA_PREPARE; + } + + g_debug("%s:XA END %s, server:%s", G_STRLOC, + con->xid_str, pmd->server->dst->name->str); + + break; + case NEXT_ST_XA_PREPARE: + if (con->servers->len == 1) { + snprintf(buffer, XA_CMD_BUF_LEN, + "XA COMMIT %s ONE PHASE", con->xid_str); + pmd->dist_tran_state = NEXT_ST_XA_CANDIDATE_OVER; + con->dist_tran_decided = 1; + } else { + snprintf(buffer, XA_CMD_BUF_LEN, "XA PREPARE %s", con->xid_str); + pmd->dist_tran_state = NEXT_ST_XA_COMMIT; + } + if (buffer_log) { + strcpy(buffer_log, buffer); + } + break; + case NEXT_ST_XA_COMMIT: + snprintf(buffer, XA_CMD_BUF_LEN, "XA COMMIT %s", con->xid_str); + pmd->dist_tran_state = NEXT_ST_XA_CANDIDATE_OVER; + if (buffer_log) { + strcpy(buffer_log, buffer); + } + con->dist_tran_decided = 1; + break; + case NEXT_ST_XA_ROLLBACK: + snprintf(buffer, XA_CMD_BUF_LEN, "XA ROLLBACK %s", con->xid_str); + if (buffer_log) { + strcpy(buffer_log, buffer); + } + con->dist_tran_decided = 1; + pmd->dist_tran_state = NEXT_ST_XA_CANDIDATE_OVER; + break; + default: + pmd->dist_tran_state = NEXT_ST_XA_OVER; + pmd->is_xa_over = 1; + pmd->dist_tran_participated = 0; + if (end) { + con->dist_tran_state = NEXT_ST_XA_OVER; + g_debug("%s: set dist_tran_state NEXT_ST_XA_OVER for con:%p", + G_STRLOC, con); + if (con->is_start_tran_command) { + con->is_auto_commit = 1; + con->is_start_tran_command = 0; + con->is_in_transaction = 0; + con->client->is_need_q_peek_exec = 1; + con->client->is_server_conn_reserved = 0; + g_debug("%s: set is_need_q_peek_exec true", G_STRLOC); + } else { + g_debug("%s: is_start_tran_command false", G_STRLOC); + } + } + return; + } + + if (end) { + g_debug("%s: set con dist_tran_state:%d", G_STRLOC, pmd->dist_tran_state); + con->dist_tran_state = pmd->dist_tran_state; + con->state = ST_SEND_QUERY; + } + + if (pmd->server->unavailable) { + return; + } + + g_debug("%s:packet id:%d when get server", + G_STRLOC, pmd->server->last_packet_id); + + pmd->server->parse.qs_state = PARSE_COM_QUERY_INIT; + + GString *srv_packet; + + srv_packet = g_string_sized_new(64); + srv_packet->len = NET_HEADER_SIZE; + g_string_append_c(srv_packet, (char) COM_QUERY); + g_string_append(srv_packet, buffer); + network_mysqld_proto_set_packet_len(srv_packet, 1 + strlen(buffer)); + network_mysqld_proto_set_packet_id(srv_packet, 0); + + g_queue_push_tail(pmd->server->send_queue->chunks, srv_packet); + + pmd->state = NET_RW_STATE_NONE; + + con->resp_expected_num++; +} + + +static void +disp_xa_abnormal_resultset(network_mysqld_con *con, server_session_t *pmd, + int *is_xa_cmd_met, char **p_buffer, char *buffer, int end) +{ + if (con->dist_tran_state <= NEXT_ST_XA_QUERY) { + pmd->dist_tran_state = NEXT_ST_XA_END; + build_xa_command(con, pmd, end, NULL); + } else if (con->dist_tran_state <= NEXT_ST_XA_COMMIT) { + pmd->dist_tran_state = NEXT_ST_XA_ROLLBACK; + if (*is_xa_cmd_met) { + build_xa_command(con, pmd, end, NULL); + (*p_buffer)[0] = ','; + } else { + *is_xa_cmd_met = 1; + build_xa_command(con, pmd, end, *p_buffer); + *p_buffer = *p_buffer + strlen(*p_buffer); + (*p_buffer)[0] = ' '; + } + (*p_buffer)++; + snprintf(*p_buffer, XA_BUF_LEN - (*p_buffer - buffer), "%s@%d", + pmd->server->dst->name->str, + pmd->server->challenge->thread_id); + *p_buffer = *p_buffer + strlen(*p_buffer); + + } else if (con->dist_tran_state <= NEXT_ST_XA_ROLLBACK) { + if (pmd->dist_tran_state < con->dist_tran_state) { + pmd->dist_tran_state = con->dist_tran_state; + g_message("%s:adjust pmd dist state:%d to %d", G_STRLOC, + pmd->dist_tran_state, con->dist_tran_state); + } + + if (*is_xa_cmd_met) { + build_xa_command(con, pmd, end, NULL); + (*p_buffer)[0] = ','; + } else { + *is_xa_cmd_met = 1; + build_xa_command(con, pmd, end, *p_buffer); + *p_buffer = *p_buffer + strlen(*p_buffer); + (*p_buffer)[0] = ' '; + } + (*p_buffer)++; + snprintf(*p_buffer, XA_BUF_LEN - (*p_buffer - buffer), "%s@%d", + pmd->server->dst->name->str, + pmd->server->challenge->thread_id); + *p_buffer = *p_buffer + strlen(*p_buffer); + + } else if (con->dist_tran_state <= NEXT_ST_XA_CANDIDATE_OVER) { + build_xa_command(con, pmd, end, NULL); + } +} + + +static void +disp_xa_according_state(network_mysqld_con *con, server_session_t *pmd, + int *is_xa_cmd_met, int *is_xa_query, char **p_buffer, char *buffer, + int end) +{ + if (pmd->dist_tran_state == NEXT_ST_XA_QUERY) { + pmd->xa_start_already_sent = 1; + con->state = ST_SEND_QUERY; + if (*is_xa_cmd_met) { + (*p_buffer)[0] = ','; + (*p_buffer)++; + } else { + *is_xa_cmd_met = 1; + } + snprintf(*p_buffer, XA_BUF_LEN - (*p_buffer - buffer), "%s@%d", + pmd->server->dst->name->str, + pmd->server->challenge->thread_id); + *p_buffer = *p_buffer + strlen(*p_buffer); + shard_build_xa_query(con, pmd); + *is_xa_query = 1; + g_debug("%s:set is xa query true for con:%p", G_STRLOC, con); + if (con->is_auto_commit) { + pmd->dist_tran_state = NEXT_ST_XA_END; + } + } else { + switch (pmd->dist_tran_state) { + case NEXT_ST_XA_PREPARE: + case NEXT_ST_XA_COMMIT: + case NEXT_ST_XA_ROLLBACK: + if (*is_xa_cmd_met) { + build_xa_command(con, pmd, end, NULL); + (*p_buffer)[0] = ','; + } else { + *is_xa_cmd_met = 1; + build_xa_command(con, pmd, end, *p_buffer); + *p_buffer = *p_buffer + strlen(*p_buffer); + (*p_buffer)[0] = ' '; + } + (*p_buffer)++; + snprintf(*p_buffer, XA_BUF_LEN - (*p_buffer - buffer), "%s@%d", + pmd->server->dst->name->str, + pmd->server->challenge->thread_id); + *p_buffer = *p_buffer + strlen(*p_buffer); + break; + default: + build_xa_command(con, pmd, end, NULL); + break; + } + } +} + + +static void +build_xa_statements(network_mysqld_con *con) +{ + int len = con->servers->len; + + g_debug("%s: call build_xa_statements:%d, server num:%d", + G_STRLOC, con->resp_expected_num, len); + + con->last_resp_num = con->resp_expected_num; + + con->resp_expected_num = 0; + con->xa_start_phase = 0; + + int iter; + int end = 0, workers = 0; + int is_xa_query = 0; + int is_xa_cmd_met = 0; + network_mysqld_con_dist_tran_state_t global_xa_state = con->dist_tran_state; + char buffer[XA_BUF_LEN] = {0}; + char *p_buffer = buffer; + + for (iter = 0; iter < len; iter++) { + server_session_t *pmd = g_ptr_array_index(con->servers, iter); + g_debug("%s: pmd %d, xa state:%d for con:%p", G_STRLOC, iter, + pmd->dist_tran_state, con); + + if (!con->is_commit_or_rollback && !pmd->participated) { + g_debug("%s: stop processing for this server:%d", G_STRLOC, iter); + continue; + } + + if (!pmd->dist_tran_participated) { + continue; + } + + if (pmd->server->unavailable) { + g_message("%s: server unavailable and stop processing here:%d", + G_STRLOC, iter); + continue; + } + + workers++; + + if ((iter + 1) == len) { + end = 1; + } + + pmd->participated = 1; + network_socket *server = pmd->server; + int result = 0; + if (con->dist_tran_failed) { + result = -1; + if (server->recv_queue->chunks->head) { + if (check_dist_tran_resultset(server->recv_queue, con) == -1) { + pmd->xa_query_status_error_and_abort = 1; + con->xa_query_status_error_and_abort = 1; + } + } + } else { + pmd->xa_query_status_error_and_abort = 0; + + if (server->recv_queue->chunks->head) { + result = check_dist_tran_resultset(server->recv_queue, con); + } + + if (result == -1) { + pmd->xa_query_status_error_and_abort = 1; + con->dist_tran_failed = 1; + con->xa_query_status_error_and_abort = 1; + g_message("%s: query status not ok, xid:%llu", + G_STRLOC, con->xa_id); + } + + if (result != -1 && con->dist_tran_failed) { + result = -1; + } + } + + if (result == -1) { + disp_xa_abnormal_resultset(con, pmd, &is_xa_cmd_met, + &p_buffer, buffer, end); + } else { + disp_xa_according_state(con, pmd, &is_xa_cmd_met, + &is_xa_query, &p_buffer, buffer, end); + } + + global_xa_state = pmd->dist_tran_state; + } + + if (workers == 0) { + con->server_to_be_closed = 1; + con->dist_tran_state = NEXT_ST_XA_OVER; + } else { + if (!end) { + con->dist_tran_state = global_xa_state; + if (global_xa_state != NEXT_ST_XA_OVER) { + con->state = ST_SEND_QUERY; + } + } + + if (is_xa_query) { + network_queue_clear(con->client->recv_queue); + g_debug("%s:set is xa query true for con:%p", G_STRLOC, con); + if (con->srv->xa_log_detailed) { + tc_log_info(LOG_INFO, 0, "XA QUERY %s %s %s", + con->xid_str, buffer, con->orig_sql->str); + } + } else if (is_xa_cmd_met) { + if (con->srv->xa_log_detailed || con->dist_tran_decided) { + tc_log_info(LOG_INFO, 0, "%s", buffer); + } + } + } +} + + +static void +retrieve_error_info_for_xa_trans(network_mysqld_con *con) +{ + int iter; + int len = con->servers->len; + + for (iter = 0; iter < len; iter++) { + server_session_t *pmd = g_ptr_array_index(con->servers, iter); + if (!pmd->xa_query_status_error_and_abort) { + continue; + } + + GQueue *out; + network_queue *in; + + out = pmd->server->recv_queue->chunks; + in = con->client->send_queue; + + GString *packet = g_queue_pop_head(out); + if (!packet) { + g_debug("%s: retrieve null server packet", G_STRLOC); + } + while (packet) { + network_queue_append(in, packet); + g_debug("%s: retrieve mul server packet:%p", G_STRLOC, packet); + packet = g_queue_pop_head(out); + } + break; + } +} + + +static void +retrieve_one_resp_for_xa_trans(network_mysqld_con *con) +{ + GString *packet; + GQueue *out; + network_queue *in; + + int iter; + int len = con->servers->len; + + for (iter = 0; iter < len; iter++) { + server_session_t *pmd = g_ptr_array_index(con->servers, iter); + if (pmd->server->unavailable || !pmd->participated) { + continue; + } + + g_debug("%s: retrieve packets for server:%p, index:%d", + G_STRLOC, pmd->server, iter); + out = pmd->server->recv_queue->chunks; + in = con->client->send_queue; + + packet = g_queue_pop_head(out); + + if (packet == NULL) { + g_warning("%s: retrieve server packet null", G_STRLOC); + } else { + do { + network_queue_append(in, packet); + packet = g_queue_pop_head(out); + } while (packet); + break; + } + } +} + +static void +normal_result_merge(network_mysqld_con *con) +{ + int len = con->servers->len; + + GPtrArray *recv_queues = g_ptr_array_sized_new(len); + + /* get all participants' receive queues */ + int i; + for (i = 0; i < len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + if (pmd->participated) { + g_ptr_array_add(recv_queues, pmd->server->recv_queue); + } + } + + uint64_t uniq_id = 0; + result_merge_t result; + + g_debug("%s: call resultset_merge", G_STRLOC); + result.status = RM_SUCCESS; + result.detail = NULL; + + resultset_merge(con->client->send_queue, recv_queues, + con, &uniq_id, &result); + + switch (result.status) { + case RM_FAIL: + if (con->candidate_tcp_streamed) { + con->server_to_be_closed = 1; + g_critical("%s: tcp streamed resultset_merge failed:%p", G_STRLOC, con); + g_debug_hexdump(G_STRLOC, S(con->orig_sql)); + } + network_queue_clear(con->client->send_queue); + g_debug("%s: merge failed", G_STRLOC); + if (result.detail) { + network_mysqld_con_send_error_full(con->client, S(result.detail), + ER_CETUS_RESULT_MERGE, "HY000"); + g_string_free(result.detail, TRUE); + } else { + network_mysqld_con_send_error_full(con->client, C("merge failed"), + ER_CETUS_RESULT_MERGE, "HY000"); + } + break; + case RM_CALL_FAIL: { + char msg[128] = {0}; + snprintf(msg, sizeof(msg), "id:%lu '%s' failed", + uniq_id, con->orig_sql->str); + network_mysqld_con_send_error_full(con->client, msg, + strlen(msg), ER_CETUS_RESULT_MERGE, "HY000"); + break; + } + default: + break; + } + + if (!con->data) { + g_ptr_array_free(recv_queues, TRUE); + } +} + +#define DISP_STOP 1 +#define DISP_CONTINUE 2 +#define WAIT_FOR_EVENT(ev_struct, ev_type, timeout) \ + event_set(&(ev_struct->event), ev_struct->fd, ev_type, network_mysqld_con_handle, con); \ + g_debug("%s:call WAIT_FOR_EVENT, ev:%p", G_STRLOC, &(ev_struct->event)); \ + chassis_event_add_with_timeout(con->srv, &(ev_struct->event), timeout); + +static void disp_query_after_consistant_attr(network_mysqld_con *con) { + network_socket *recv_sock = con->client; + GList *chunk = recv_sock->recv_queue->chunks->head; + GString *packet = (GString *)(chunk->data); + + con->is_attr_adjust = 0; + if (con->dist_tran && !con->dist_tran_xa_start_generated) { + /* append xa query to send queue */ + chassis *srv = con->srv; + con->dist_tran_state = NEXT_ST_XA_QUERY; + con->xa_id = srv->dist_tran_id++; + snprintf(con->xid_str, XID_LEN, "'%s_%02d_%llu'", + srv->dist_tran_prefix, tc_get_log_hour(), con->xa_id); + + con->dist_tran_xa_start_generated = 1; + con->is_start_trans_buffered = 0; + con->is_auto_commit_trans_buffered = 0; + g_debug("%s:xa start:%s for con:%p", G_STRLOC, con->xid_str, con); + } + + size_t i; + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + if (!con->is_commit_or_rollback && !pmd->participated) { + continue; + } + g_debug("%s:packet id:%d when get server, pmd state:%d", + G_STRLOC, pmd->server->last_packet_id, pmd->dist_tran_state); + + pmd->server->parse.qs_state = PARSE_COM_QUERY_INIT; + + if (con->dist_tran) { + pmd->xa_start_already_sent = 1; + if (pmd->dist_tran_state == NEXT_ST_XA_START) { + network_mysqld_send_xa_start(pmd->server, con->xid_str); + g_debug("%s: %s, server:%s", G_STRLOC, + con->xid_str, pmd->server->dst->name->str); + con->resp_expected_num++; + pmd->dist_tran_state = NEXT_ST_XA_QUERY; + pmd->xa_start_already_sent = 0; + con->xa_start_phase = 1; + } else { + continue; + } + } else { + if (con->parse.command == COM_QUERY) { + GString *payload = g_string_new(0); + network_mysqld_proto_append_query_packet(payload, pmd->sql->str); + network_mysqld_queue_reset(pmd->server); + network_mysqld_queue_append(pmd->server, pmd->server->send_queue, + S(payload)); + g_string_free(payload, TRUE); + } else { + network_queue_append(pmd->server->send_queue, + g_string_new_len(packet->str, packet->len)); + } + con->resp_expected_num++; + } + + pmd->state = NET_RW_STATE_NONE; + } + + if (!con->dist_tran) { + g_string_free(packet, TRUE); + g_queue_delete_link(recv_sock->recv_queue->chunks, chunk); + } +} + +static void handle_query_time_stats(network_mysqld_con *con) { + int diff = (con->resp_send_time.tv_sec - con->req_recv_time.tv_sec) * 1000; + diff += (con->resp_send_time.tv_usec - con->req_recv_time.tv_usec) / 1000; + + diff = MAX(0, diff); + if (diff >= con->srv->long_query_time) { + g_log("slowquery", G_LOG_LEVEL_MESSAGE, + "time: %dms, client: %s, user: %s, sql: %s", + diff, con->client->src->name->str, + con->client->response->username->str, con->orig_sql->str); + diff = con->srv->long_query_time - 1; + } + con->srv->query_stats.query_time_table[diff]++; +} + +static void handle_query_wait_stats(network_mysqld_con *con) { + struct timeval cur; + gettimeofday(&cur, NULL); + + int diff = (cur.tv_sec - con->req_recv_time.tv_sec) * 1000; + diff += (cur.tv_usec - con->req_recv_time.tv_usec) / 1000; + + if (diff < 0 || diff >= MAX_WAIT_TIME) { + g_message("%s: query waits too long:%d for con:%p", G_STRLOC, diff, con); + diff = MAX_WAIT_TIME - 1; + } + + con->srv->query_stats.query_wait_table[diff]++; +} + + +static int +handle_read_query(network_mysqld_con *con, network_mysqld_con_state_t ostate) +{ + struct timeval timeout; + network_socket *recv_sock; + network_packet last_packet; + + chassis *srv = con->srv; + recv_sock = con->client; + + recv_sock->total_output = 0; + + recv_sock->compressed_packet_id = 1; + recv_sock->do_strict_compress = 0; + + con->client->do_query_cache = 0; + con->client->query_cache_too_long = 0; + con->query_cache_judged = 0; + con->is_read_ro_server_allowed = 0; + + gettimeofday(&(con->req_recv_time), NULL); + + if (!con->is_wait_server) { + do { + switch (network_mysqld_read(srv, recv_sock)) { + case NETWORK_SOCKET_SUCCESS: + break; + case NETWORK_SOCKET_WAIT_FOR_EVENT: + if (con->is_commit_or_rollback) { + if (con->is_start_tran_command) {/* is prev sql START */ + con->is_start_tran_command = 0; + con->is_auto_commit = 1; + con->is_in_transaction = 0; + con->client->is_need_q_peek_exec = 1; + con->client->is_server_conn_reserved = 0; + g_debug("%s: set is_need_q_peek_exec true", G_STRLOC); + } + } + if (con->client->is_need_q_peek_exec) { + timeout = con->wait_clt_next_sql; + con->client->is_need_q_peek_exec = 0; + g_debug("%s: set a short timeout:%p", G_STRLOC, con); + } else { + timeout = con->read_timeout; + g_debug("%s: set a long timeout:%p", G_STRLOC, con); + } + + WAIT_FOR_EVENT(con->client, EV_READ, &timeout); + return DISP_STOP; + case NETWORK_SOCKET_ERROR_RETRY: + case NETWORK_SOCKET_ERROR: + g_critical("%s: network_mysqld_read error", G_STRLOC); + con->prev_state = con->state; + con->state = ST_ERROR; + return DISP_CONTINUE; + } + + if (con->state != ostate) { + break; + } + + GQueue *chunks = recv_sock->recv_queue->chunks; + last_packet.data = g_queue_peek_tail(chunks); + } while (last_packet.data->len == (PACKET_LEN_MAX + NET_HEADER_SIZE)); + } else { + g_debug("%s:wait server.", G_STRLOC); + } + + con->resp_too_long = 0; + g_debug("%s:call read query", G_STRLOC); + switch (plugin_call(srv, con, con->state)) { + case NETWORK_SOCKET_SUCCESS: + if (con->retry_serv_cnt > 0 && con->is_wait_server) { + g_message("%s: wait successful:%d, con:%p", + G_STRLOC, con->retry_serv_cnt, con); + handle_query_wait_stats(con); + } + con->is_wait_server = 0; + con->retry_serv_cnt = 0; + break; + case NETWORK_SOCKET_ERROR_RETRY: + if (con->retry_serv_cnt < con->max_retry_serv_cnt) { + if (con->retry_serv_cnt == 0 || con->retry_serv_cnt == 8) { + network_connection_pool_create_conn(con); + } + con->retry_serv_cnt++; + con->is_wait_server = 1; + timeout = network_mysqld_con_retry_timeout(con); + + g_debug(G_STRLOC ": wait again:%d, con:%p, l:%d", + con->retry_serv_cnt, con, (int) timeout.tv_usec); + WAIT_FOR_EVENT(con->client, EV_TIMEOUT, &timeout); + return DISP_STOP; + } + /* fall through */ + default: + g_critical("%s: wait failed and no server backend for user:%s", + G_STRLOC, con->client->response->username->str); + + handle_query_wait_stats(con); + con->state = ST_SEND_ERROR; + network_mysqld_con_send_error_full(con->client, C("service unavailable"), + ER_SERVER_SHUTDOWN, "08S01"); + con->is_wait_server = 0; + network_queue_clear(con->client->recv_queue); + network_mysqld_queue_reset(con->client); + break; + } + + /** + * there should be 3 possible next states from here: + * + * - ST_ERROR + * (if something went wrong and we want to + * close the connection + * - ST_SEND_QUERY + * (if we want to send data to the con->server) + * - ST_SEND_QUERY_RESULT (if we want to send data + * to the con->client) + * + * @todo verify this with a clean switch () + */ + + /* reset the tracked command + */ + if (con->state == ST_SEND_QUERY) { + network_mysqld_con_reset_command_response_state(con); + g_debug("%s: call reset_command_response_state for con:%p", + G_STRLOC, con); + } + return DISP_CONTINUE; +} + + +static void +process_write_to_server(network_mysqld_con *con, server_session_t *pmd, + int *write_wait) +{ + network_socket_retval_t ret; + + con->num_write_pending++; + ret = network_mysqld_write(con->srv, pmd->server); + + switch (ret) + { + case NETWORK_SOCKET_SUCCESS: + con->num_pending_servers++; + con->num_servers_visited++; + con->num_read_pending++; + pmd->read_cal_flag = 0; + g_debug("%s:num_read_pending:%d, pmd->index:%d, reset 0 for con:%p", + G_STRLOC, con->num_read_pending, pmd->index, con); + con->num_write_pending--; + pmd->state = NET_RW_STATE_READ; + break; + case NETWORK_SOCKET_WAIT_FOR_EVENT: + pmd->state = NET_RW_STATE_WRITE; + + g_debug("%s:write waits for con:%p", G_STRLOC, con); + server_sess_wait_for_event(pmd, EV_WRITE, &con->write_timeout); + + *write_wait = 1; + break; + + default: + { + char *msg = "write error"; + con->state = ST_SEND_QUERY_RESULT; + con->server_to_be_closed = 1; + g_warning("%s:write error for con:%p, ret:%d", G_STRLOC, con, ret); + network_mysqld_con_send_error_full(con->client, L(msg), + ER_NET_ERROR_ON_WRITE, "08S01"); + break; + } + } +} + + +static int +process_shard_write(network_mysqld_con *con, int *disp_flag) +{ + switch (con->parse.command) { + case COM_STMT_SEND_LONG_DATA: + case COM_STMT_CLOSE: + g_debug("%s:set ST_READ_QUERY for con:%p", G_STRLOC, con); + con->state = ST_READ_QUERY; + break; + case COM_QUERY: + default: + con->state = ST_READ_M_QUERY_RESULT; + break; + } + + con->num_read_pending = 0; + con->num_pending_servers = 0; + con->num_servers_visited = 0; + con->num_write_pending = 0; + + int i, write_wait = 0; + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + pmd->index = i; + pmd->fresh = 0; + pmd->server->compressed_packet_id = 0; + pmd->server->resp_len = 0; + pmd->server->is_read_finished = 0; + pmd->server->is_waiting = 0; + + if (!pmd->participated || pmd->server->unavailable) { + g_debug("%s:not participated or unavailable:%d for con%p", G_STRLOC, i, con); + continue; + } + + if (con->attr_adj_state != ATTR_START) { + if (!pmd->attr_adjusted_now) { + g_debug("%s:skip here for con:%p", G_STRLOC, con); + continue; + } + } + pmd->server->parse.command = con->parse.command; + pmd->state = NET_RW_STATE_NONE; + pmd->server->resp_len = 0; + + if (!g_queue_is_empty(pmd->server->send_queue->chunks)) { + process_write_to_server(con, pmd, &write_wait); + } + } /* for each server */ + + if (con->state == ST_READ_M_QUERY_RESULT) { + if (write_wait) { + *disp_flag = DISP_STOP; + return 0; + } + if (con->resp_expected_num != con->num_pending_servers) { + g_critical("%s:expected resp num:%d, pending:%d, con:%p", + G_STRLOC, con->resp_expected_num, + con->num_pending_servers, con); + } + } + + return 1; +} + + +static int +process_rw_write(network_mysqld_con *con, network_mysqld_con_state_t ostate, + int *disp_flag) +{ + if (con->server->send_queue->offset == 0) { + /* only parse the packets once */ + network_packet packet; + GQueue *chunks = con->server->send_queue->chunks; + packet.data = g_queue_peek_head(chunks); + packet.offset = 0; + + if (network_mysqld_con_command_states_init(con, &packet)) { + g_warning("%s: track mysql proto states failed", G_STRLOC); + con->prev_state = con->state; + con->state = ST_ERROR; + + return DISP_CONTINUE; + } + } + + con->server->resp_len = 0; + con->server->compressed_packet_id = 0; + + switch (network_mysqld_write(con->srv, con->server)) { + case NETWORK_SOCKET_SUCCESS: + break; + case NETWORK_SOCKET_WAIT_FOR_EVENT: + g_debug("%s:write wait for con:%p", G_STRLOC, con); + WAIT_FOR_EVENT(con->server, EV_WRITE, &con->write_timeout); + *disp_flag = DISP_STOP; + return 0; + case NETWORK_SOCKET_ERROR_RETRY: + case NETWORK_SOCKET_ERROR: + g_debug("%s:write(SEND_QUERY) error", G_STRLOC); + + /** + * write() failed, close the connections + */ + con->prev_state = con->state; + con->state = ST_ERROR; + break; + } + + if (con->state != ostate) { + *disp_flag = DISP_CONTINUE; + return 0; + } + + /* some statements don't have a server response */ + switch (con->parse.command) { + case COM_STMT_SEND_LONG_DATA: /* not acked */ + case COM_STMT_CLOSE: + if (!con->server_to_be_closed) { + g_message("%s: set ST_READ_QUERY for con:%p", G_STRLOC, con); + con->state = ST_READ_QUERY; + } else { + g_message("%s: set ST_CLOSE_SERVER for con:%p", G_STRLOC, con); + con->state = ST_CLOSE_SERVER; + } + if (con->client) { + network_mysqld_queue_reset(con->client); + } + if (con->server) { + network_mysqld_queue_reset(con->server); + } + + con->prepare_stmt_count--; + g_debug("%s: conn:%p, sub, now prepare_stmt_count:%d", + G_STRLOC, con, + con->prepare_stmt_count); + + if (con->prepare_stmt_count == 0) { + if (!con->is_in_transaction) { + if (network_pool_add_conn(con, 0)) { + g_message("%s,con:%p:->pool failed", G_STRLOC, con); + } + } + } + break; + default: + con->state = ST_READ_QUERY_RESULT; + break; + } + + return 1; +} + + +static int +handle_send_query_to_servers(network_mysqld_con *con, + network_mysqld_con_state_t ostate) +{ + int disp_flag = 0; + + /* + * send the query to the server + * this state will loop until all the packets + * from the send-queue are flushed + */ + if (con->srv->sharding_mode && con->servers != NULL) { + if (!process_shard_write(con, &disp_flag)) { + return disp_flag; + } + + } else {/* RW-edition */ + + if (!process_rw_write(con, ostate, &disp_flag)) { + return disp_flag; + } + } + + return DISP_CONTINUE; +} + + +static int +shard_read_response(network_mysqld_con *con, server_session_t *pmd) +{ + if (pmd->server->resp_len == 0 && pmd->server->to_read == 0) { + switch(network_socket_to_read(pmd->server)) { + case NETWORK_SOCKET_SUCCESS: + if (pmd->server->to_read == 0) { + pmd->state = NET_RW_STATE_READ; + g_debug("%s:read wait here for con:%p", G_STRLOC, con); + server_sess_wait_for_event(pmd, EV_READ, &con->read_timeout); + return DISP_CONTINUE; + } + break; + default: + g_message("%s:read not success", G_STRLOC); + return DISP_STOP; + } + } + + if (con->srv->query_cache_enabled) { + g_debug("%s: check if query can be cached, attr state:%d", G_STRLOC, con->attr_adj_state); + if (con->is_read_ro_server_allowed && con->attr_adj_state == ATTR_START) { + if (con->query_cache_judged == 0) { + do_check_qeury_cache(con); + } + } + } + + int is_finished = 0; + switch (network_mysqld_read_mul_packets(con->srv, con, pmd->server, &is_finished)) { + case NETWORK_SOCKET_SUCCESS: + if (is_finished) { + con->num_pending_servers--; + if (pmd->read_cal_flag == 0) { + con->num_read_pending--; + pmd->read_cal_flag = 1; + } + + set_conn_attr(con, pmd->server); + pmd->state = NET_RW_STATE_FINISHED; + pmd->server->is_read_finished = 1; + pmd->server->is_waiting = 0; + break; + } else { + pmd->state = NET_RW_STATE_READ; + g_debug("%s:server_sess_wait_for_event for con:%p", G_STRLOC, con); + server_sess_wait_for_event(pmd, EV_READ, &con->read_timeout); + con->num_read_pending++; + pmd->read_cal_flag = 0; + g_debug("%s:num_read_pending:%d, pmd->index:%d for con:%p", + G_STRLOC, con->num_read_pending, pmd->index, con); + + break; + } + case NETWORK_SOCKET_WAIT_FOR_EVENT: + if (con->candidate_tcp_streamed) { + if (!pmd->server->max_header_size_reached) { + pmd->state = NET_RW_STATE_READ; + server_sess_wait_for_event(pmd, EV_READ, &con->read_timeout); + } else { + if (con->num_servers_visited > 1) { + pmd->state = NET_RW_STATE_PART_FINISHED; + if (pmd->read_cal_flag == 0) { + con->num_read_pending--; + pmd->read_cal_flag = 1; + } + g_debug("%s:num read pending:%d for fd:%d", G_STRLOC, + con->num_read_pending, pmd->server->fd); + if (con->num_read_pending == 0) { + set_conn_attr(con, pmd->server); + } + } else { + g_message("%s: here, we send_part_content_to_client", G_STRLOC); + send_part_content_to_client(con); + pmd->state = NET_RW_STATE_READ; + g_debug("%s:num_read_pending:%d, pmd->index:%d for con:%p", + G_STRLOC, con->num_read_pending, pmd->index, con); + server_sess_wait_for_event(pmd, EV_READ, &con->read_timeout); + } + } + } else { + pmd->state = NET_RW_STATE_READ; + g_debug("%s:num_read_pending:%d for fd:%d, pmd->index:%d", + G_STRLOC, con->num_read_pending, pmd->server->fd, pmd->index); + server_sess_wait_for_event(pmd, EV_READ, &con->read_timeout); + } + break; + case NETWORK_SOCKET_ERROR: + default: + g_critical("%s:network_mysqld_read_mul_packets error", G_STRLOC); + con->num_read_pending--; + con->state = ST_ERROR; + return DISP_CONTINUE; + } + + return DISP_CONTINUE; +} + +static int +handle_dist_tran_after_read_mul_resp(network_mysqld_con *con, int *result_reserve, int *skip, int *disp_flag) +{ + g_debug("%s: con dist tran here:%p", G_STRLOC, con); + if (con->is_timeout || con->is_auto_commit || !con->is_xa_query_sent || + con->xa_query_status_error_and_abort) + { + g_debug("%s: call before, con dist tan state:%d for con:%p", + G_STRLOC, con->dist_tran_state, con); + build_xa_statements(con); + g_debug("%s: call after, con dist tan state:%d for con:%p", + G_STRLOC, con->dist_tran_state, con); + if (con->dist_tran_state != NEXT_ST_XA_PREPARE) { + if (con->state == ST_SEND_QUERY) { + g_debug("%s: visit here", G_STRLOC); + if (con->dist_tran_failed && + con->dist_tran_state == NEXT_ST_XA_ROLLBACK) + { + g_debug("%s: visit here", G_STRLOC); + if (con->xa_query_status_error_and_abort) { + retrieve_error_info_for_xa_trans(con); + } else { + retrieve_one_resp_for_xa_trans(con); + } + } + remove_mul_server_recv_packets(con); + *disp_flag = DISP_CONTINUE; + return 1; + } else if (con->dist_tran_state == NEXT_ST_XA_OVER) { + *skip = 1; + if (con->xa_query_status_error_and_abort) { + remove_mul_server_recv_packets(con); + network_queue_clear(con->client->recv_queue); + if (con->client->send_queue->chunks->head) { + network_mysqld_queue_reset(con->client); + } + } else if (con->is_commit_or_rollback) { + if (con->dist_tran_failed) { + g_message("%s: service unavailable for con:%p", G_STRLOC, con); + network_mysqld_con_send_error_full(con->client, + C("service unavailable"),ER_SERVER_SHUTDOWN, "08S01"); + if (con->dist_tran_state != NEXT_ST_XA_OVER) { + g_message("%s: dist state not NEXT_ST_XA_OVER for con:%p", + G_STRLOC, con); + } + con->dist_tran_state = NEXT_ST_XA_OVER; + g_debug("%s: set dist_tran_state NEXT_ST_XA_OVER for con:%p", + G_STRLOC, con); + } else if (con->is_timeout) { + network_mysqld_con_send_ok_full(con->client, 0, 0, 0, 0); + g_message("%s: send ok to client when timeout:%p", + G_STRLOC, con); + } else { + retrieve_one_resp_for_xa_trans(con); + } + remove_mul_server_recv_packets(con); + network_queue_clear(con->client->recv_queue); + network_mysqld_queue_reset(con->client); + } else if (con->is_timeout) { + remove_mul_server_recv_packets(con); + network_queue_clear(con->client->recv_queue); + if (con->client->send_queue->chunks->head) { + network_mysqld_queue_reset(con->client); + g_warning("%s: no packets sent to client for con:%p", G_STRLOC, con); + } + g_debug("%s: call here for con:%p", G_STRLOC, con); + } + + con->dist_tran = 0; + con->dist_tran_failed = 0; + g_debug("%s: set dist tran 0:%p", G_STRLOC, con); + } else { + g_debug("%s: call here for con:%p, xa state:%d", + G_STRLOC, con, con->dist_tran_state); + } + } else { + /* Record affected rows */ + if (con->is_auto_commit) { + *result_reserve = 1; + } + } + } + + return 0; +} + + +static int +read_server_resp(network_mysqld_con *con, int *disp_flag) +{ + int i; + + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + + if (!pmd->participated || (con->dist_tran && con->xa_start_phase && pmd->xa_start_already_sent)) { + g_debug("%s: server not participated", G_STRLOC); + continue; + } + network_socket *server = pmd->server; + + if (server->unavailable || server->is_waiting) { + g_debug("%s: omit here", G_STRLOC); + continue; + } + + if (con->attr_adj_state >= ATTR_DIF_CHANGE_USER) { + if (pmd->attr_adjusted_now == 0 || pmd->attr_consistent) { + g_debug("%s: server attr consistent, adj state:%d for con:%p", + G_STRLOC, con->attr_adj_state, con); + continue; + } + } + + if (pmd->state == NET_RW_STATE_FINISHED || pmd->state == NET_RW_STATE_WRITE) { + g_debug("%s: omit here", G_STRLOC); + continue; + } + + if (pmd->state == NET_RW_STATE_ERROR) { + g_message("%s: read server error", G_STRLOC); + if (con->dist_tran) { + record_xa_log_for_mending(con, server); + server->unavailable = 1; + g_message("%s: fail at %d server", G_STRLOC, i + 1); + continue; + } else { + int j; + for (j = 0; j < i; j++) { + pmd = g_ptr_array_index(con->servers, j); + network_mysqld_con_send_command( + pmd->server, + COM_STMT_RESET, + "\x00\x00"); + } + + network_mysqld_con_send_error(con->client, C("server error")); + + if (!con->server_to_be_closed) { + g_message("%s: set ST_READ_QUERY for con:%p", G_STRLOC, con); + con->state = ST_READ_QUERY; + } else { + g_message("%s: set ST_CLOSE_SERVER for con:%p", G_STRLOC, con); + con->state = ST_CLOSE_SERVER; + } + + *disp_flag = DISP_STOP; + return 0; + } + } + + if (shard_read_response(con, pmd) == DISP_STOP) { + *disp_flag = DISP_STOP; + return 0; + } + + if (con->state == ST_ERROR) { + *disp_flag = DISP_CONTINUE; + return 0; + } + } + + return 1; +} + +static void +check_server_status(network_mysqld_con *con, int *srv_down_count, int *srv_response_count) +{ + int i; + + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + + if (!pmd->participated) { + g_debug("%s: server not participated", G_STRLOC); + continue; + } + + network_socket *server = pmd->server; + if (server->unavailable) { + (*srv_down_count)++; + continue; + } + + if (con->attr_adj_state >= ATTR_DIF_CHANGE_USER) { + if (pmd->attr_consistent) { + continue; + } + if (!pmd->attr_adjusted_now) { + continue; + } + } + + if (pmd->state != NET_RW_STATE_FINISHED && pmd->state != NET_RW_STATE_PART_FINISHED) { + if (pmd->state == NET_RW_STATE_ERROR) { + while (con->servers->len > 0) { + pmd = g_ptr_array_remove_index_fast(con->servers, 0); + network_socket_free(pmd->server); + pmd->sql = NULL; + pmd->server = NULL; + server_session_free(pmd); + } + + network_mysqld_con_send_error(con->client, + C("server error")); + + if (!con->server_to_be_closed) { + con->state = ST_READ_QUERY; + g_debug("%s: set ST_READ_QUERY for con:%p", G_STRLOC, con); + } else { + con->state = ST_CLOSE_SERVER; + g_debug("%s: set ST_CLOSE_SERVER for con:%p", G_STRLOC, con); + } + break; + } + } else { + if (g_queue_is_empty(server->recv_queue->chunks)) { + g_debug("fd(%d) multi resp, no data in queue", server->fd); + } + g_debug("fd(%d) srv_response_count added", server->fd); + (*srv_response_count)++; + } + } /* for each pmd server */ +} + +static void disp_no_workers(network_mysqld_con *con) { + if (con->dist_tran) { + network_queue_clear(con->client->send_queue); + con->dist_tran_state = NEXT_ST_XA_OVER; + con->dist_tran = 0; + con->dist_tran_failed = 0; + } + + con->server_to_be_closed = 1; + con->state = ST_SEND_QUERY_RESULT; + g_message("%s: send dist trans error to client, dist state:%d", + G_STRLOC, con->dist_tran_state); + network_mysqld_con_send_error_full(con->client, C("service unavailable"), + ER_SERVER_SHUTDOWN, "08S01"); + network_queue_clear(con->client->recv_queue); + network_mysqld_queue_reset(con->client); +} + + +static int disp_resp_workers_not_matched(network_mysqld_con *con, int *disp_flag) { + int result = 1; + + if (con->dist_tran_failed && con->dist_tran) { + if (con->num_pending_servers) { + g_debug("%s: not recv all resp:%d", G_STRLOC, con->state); + *disp_flag = DISP_STOP; + result = 0; + } else { + g_debug("%s: check dist tran query status:%d, dist tran state:%d, attr state:%d", + G_STRLOC, con->state, con->dist_tran_state, con->attr_adj_state); + if (con->xa_query_status_error_and_abort) { + g_debug("%s: dist tran query status error:%d", G_STRLOC, con->state); + } else { + if (con->is_attr_adjust) { + g_critical("%s: attr adj met problems here for con:%p", G_STRLOC, con); + con->server_to_be_closed = 1; + } else { + if (con->dist_tran_state < NEXT_ST_XA_CANDIDATE_OVER) { + g_debug("%s: build xa stmt for failure, state:%d", + G_STRLOC, con->state); + build_xa_statements(con); + if (con->dist_tran_state != NEXT_ST_XA_OVER) { + *disp_flag = DISP_CONTINUE; + return 0; + } + } + } + + con->state = ST_SEND_QUERY_RESULT; + g_message("%s: send dist trans error to client, dist state:%d", + G_STRLOC, con->dist_tran_state); + if (con->dist_tran_state != NEXT_ST_XA_OVER) { + g_message("%s: dist state not NEXT_ST_XA_OVER for con:%p", + G_STRLOC, con); + } + g_debug("%s: set dist_tran_state NEXT_ST_XA_OVER for con:%p", + G_STRLOC, con); + + con->dist_tran_state = NEXT_ST_XA_OVER; + con->dist_tran = 0; + con->dist_tran_failed = 0; + + network_mysqld_con_send_error_full(con->client, + C("service unavailable"), ER_SERVER_SHUTDOWN, "08S01"); + + remove_mul_server_recv_packets(con); + network_queue_clear(con->client->send_queue); + network_queue_clear(con->client->recv_queue); + network_mysqld_queue_reset(con->client); + + *disp_flag = DISP_CONTINUE; + result = 0; + } + } + } else { + if (con->num_pending_servers) { + if (con->candidate_tcp_streamed) { + if (con->num_read_pending != 0) { + g_debug("%s: we will stop for a while:%p", G_STRLOC, con); + *disp_flag = DISP_STOP; + result = 0; + } + } else { + g_debug("%s: we will stop for a while:%p", G_STRLOC, con); + *disp_flag = DISP_STOP; + result = 0; + } + } + } + + return result; +} + + +static void +disp_single_resp(network_mysqld_con *con) +{ + server_session_t *pmd = NULL; + + if (con->dist_tran) { + int iter; + for (iter = 0; iter < con->servers->len; iter++) { + server_session_t *candidate_pmd = g_ptr_array_index(con->servers, iter); + if (candidate_pmd->participated && !candidate_pmd->server->unavailable) { + pmd = candidate_pmd; + break; + } + } + + } else { + pmd = g_ptr_array_index(con->servers, 0); + } + + if (pmd != NULL) { + GQueue *out; + network_queue *in; + + out = pmd->server->recv_queue->chunks; + in = con->client->send_queue; + + GString *packet = g_queue_pop_head(out); + while (packet) { + network_queue_append(in, packet); + packet = g_queue_pop_head(out); + } + } else { + g_warning("%s: not recv resp for con:%p", G_STRLOC, con); + } +} + + +static int disp_not_skipped(network_mysqld_con *con, int srv_response_count, + int *single_response, int *disp_flag) +{ + switch(con->parse.command) { + case COM_STMT_EXECUTE: + case COM_QUERY: + if (srv_response_count > 1) { + normal_result_merge(con); + if (con->partially_merged) { + g_debug("%s:partially_merged here:%d", + G_STRLOC, srv_response_count); + *disp_flag = DISP_STOP; + return 0; + } + } else { + if (con->partially_merged) { + g_message("%s:part read here for con:%p", G_STRLOC, con); + *disp_flag = DISP_STOP; + return 0; + } else { + *single_response = 1; + } + } + break; + default: + *single_response = 1; + break; + } + + return 1; +} + +static void disp_result_not_reserved(network_mysqld_con *con) +{ + GQueue *in = con->client->send_queue->chunks; + if (in == NULL || in->length == 0) { + if (con->dist_tran_decided) { + network_mysqld_con_send_ok_full(con->client, 0, 0, 0, 0); + g_warning("%s:needs to check xa recover", G_STRLOC); + } else { + g_warning("%s:send queue empty when sending result to client", + G_STRLOC); + network_mysqld_con_send_error_full(con->client, C("server error"), + ER_UNKNOWN_ERROR, "HY000"); + } + con->server_to_be_closed = 1; + if (con->dist_tran) { + con->is_commit_or_rollback = 1; + con->dist_tran = 0; + con->dist_tran_state = NEXT_ST_XA_OVER; + } + } + + g_debug("%s:set send result to client", G_STRLOC); + con->resultset_is_finished = TRUE; + con->state = ST_SEND_QUERY_RESULT; +} + + +static int disp_attr(network_mysqld_con *con, int srv_down_count, int *disp_flag) +{ + if (con->num_pending_servers) { + g_debug("%s: not recv all resp:%d", G_STRLOC, con->state); + *disp_flag = DISP_STOP; + return 0; + } else { + if (con->attr_adj_state <= ATTR_DIF_SET_AUTOCOMMIT) { + if (srv_down_count > 0) { + con->state = ST_SEND_QUERY_RESULT; + if (con->dist_tran) { + con->dist_tran = 0; + con->dist_tran_failed = 0; + con->dist_tran_state = NEXT_ST_XA_OVER; + } + + network_mysqld_con_send_error_full(con->client, + C("MySQL server down"), ER_SERVER_SHUTDOWN, "08S01"); + + remove_mul_server_recv_packets(con); + network_queue_clear(con->client->send_queue); + network_queue_clear(con->client->recv_queue); + network_mysqld_queue_reset(con->client); + + con->server_to_be_closed = 1; + + *disp_flag = DISP_CONTINUE; + return 0; + + } else { + if (build_attr_statements(con)) { + g_debug("%s: continue here:%d", G_STRLOC, con->state); + remove_mul_server_recv_packets(con); + *disp_flag = DISP_CONTINUE; + return 0; + } + } + } + + if (con->attr_adj_state == ATTR_START) { + g_debug("%s: before disp_query_after_consistant_attr:%d, expected resp:%d", + G_STRLOC, con->state, con->resp_expected_num); + /* now the attrs of all server connections are the same */ + if (con->could_be_tcp_streamed) { + con->candidate_tcp_streamed = 1; + } + disp_query_after_consistant_attr(con); + g_debug("%s: disp_query_after_consistant_attr:%d, expected resp:%d", + G_STRLOC, con->state, con->resp_expected_num); + remove_mul_server_recv_packets(con); + *disp_flag = DISP_CONTINUE; + return 0; + } + } + + return 1; +} + + +static int +disp_after_resp(network_mysqld_con *con, int srv_down_count, int srv_response_count, int *disp_flag) +{ + int result_reserve = 0, skip = 0; + + g_debug("%s: att adj state:%d", G_STRLOC, con->attr_adj_state); + if (con->attr_adj_state >= ATTR_DIF_CHANGE_USER) { + if (!disp_attr(con, srv_down_count, disp_flag)) { + return 0; + } + } + + if (con->dist_tran) { + if (handle_dist_tran_after_read_mul_resp(con, &result_reserve, &skip, disp_flag)) { + return 0; + } + } + + int single_response = 0; + + if (!skip) { + if (!disp_not_skipped(con, srv_response_count, &single_response, disp_flag)) { + return 0; + } + } + + if (single_response) { + disp_single_resp(con); + } + + remove_mul_server_recv_packets(con); + + if (!result_reserve) { + disp_result_not_reserved(con); + } + + return 1; +} + + +static int +handle_read_mul_servers_resp(network_mysqld_con *con) +{ + g_debug("%s: visit handle_read_mul_servers_resp for con:%p", G_STRLOC, con); + int disp_flag = 0; + + int srv_down_count = 0, srv_response_count = 0; + + if (con->num_read_pending != 0) { + if (!read_server_resp(con, &disp_flag)) { + return disp_flag; + } + } + + check_server_status(con, &srv_down_count, &srv_response_count); + + if (srv_down_count > 0) { + g_warning("%s: server down num:%d for con:%p", G_STRLOC, srv_down_count, con); + } + + g_debug("%s:resp count:%d, server num:%d, num pending:%d, state:%d for con:%p", + G_STRLOC, srv_response_count, con->servers->len, + con->num_pending_servers, con->state, con); + + int workers = con->servers->len - srv_down_count; + + if (workers == 0) { + disp_no_workers(con); + return DISP_CONTINUE; + } + + if (con->state == ST_READ_QUERY) { + return DISP_CONTINUE; + } else if (srv_response_count != workers) { + if (!disp_resp_workers_not_matched(con, &disp_flag)) { + return disp_flag; + } + } + + if (srv_response_count != 0) { + if (!disp_after_resp(con, srv_down_count, srv_response_count, &disp_flag)) { + return disp_flag; + } + } else { + if (con->state != ST_READ_QUERY) { + g_debug("%s: stop here for con state:%d", G_STRLOC, con->state); + return DISP_STOP; + } + } + + network_mysqld_queue_reset(con->client); + + return DISP_CONTINUE; +} + +void send_part_content_to_client(network_mysqld_con *con) +{ + g_debug("%s: call send_part_content_to_client, and queue len:%llu, con client:%p", + G_STRLOC, (unsigned long long) con->client->send_queue->chunks->length, con->client); + switch (network_mysqld_write(con->srv, con->client)) { + case NETWORK_SOCKET_SUCCESS: + break; + case NETWORK_SOCKET_WAIT_FOR_EVENT: + g_debug("%s: write wait for event", G_STRLOC); + break; + case NETWORK_SOCKET_ERROR_RETRY: + case NETWORK_SOCKET_ERROR: + con->prev_state = con->state; + con->state = ST_ERROR; + break; + } + + g_debug("%s: call send_part_content_to_client over, and queue len:%llu, con client:%p", + G_STRLOC, (unsigned long long) con->client->send_queue->chunks->length, con->client); + +} + + +static void process_service_unavailable(network_mysqld_con *con) { + + con->state = ST_SEND_QUERY_RESULT; + g_message("%s: service unavailable for con:%p", G_STRLOC, con); + + if (con->servers != NULL) { + g_debug("%s: server num :%d for con:%p", G_STRLOC, + (int) con->servers->len, con); + size_t i; + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + if (pmd->fresh) { + network_pool_add_idle_conn(pmd->backend->pool, con->srv, pmd->server); + pmd->backend->connected_clients--; + g_message("%s: connected_clients sub:%d, %d ndx for con:%p", G_STRLOC, + pmd->backend->connected_clients, (int) i, con); + g_ptr_array_remove(con->servers, pmd); + pmd->server = NULL; + server_session_free(pmd); + i--; + } + } + + if (con->servers->len == 0) { + con->server_to_be_closed = 1; + } + } + + network_mysqld_con_send_error_full(con->client, + C("service unavailable"), ER_TOO_MANY_USER_CONNECTIONS, "42000"); + con->is_wait_server = 0; + network_queue_clear(con->client->recv_queue); + network_mysqld_queue_reset(con->client); +} + + +static int +send_result_to_client(network_mysqld_con *con, network_mysqld_con_state_t ostate) +{ + chassis *srv = con->srv; + struct timeval timeout; + + /* only for sharding */ + if (con->partially_merged) { + if (con->servers) { + remove_mul_server_recv_packets(con); + } + network_queue_clear(con->client->recv_queue); + network_mysqld_queue_reset(con->client); + con->partially_merged = 0; + } + + g_debug("%s: send server result to client", G_STRLOC); + /** + * send the query result-set to the client + */ + switch (network_mysqld_write(srv, con->client)) { + case NETWORK_SOCKET_SUCCESS: + break; + case NETWORK_SOCKET_WAIT_FOR_EVENT: + g_debug("%s: write wait and add event", G_STRLOC); + timeout = con->write_timeout; + + WAIT_FOR_EVENT(con->client, EV_WRITE, &timeout); + + return DISP_STOP; + case NETWORK_SOCKET_ERROR_RETRY: + case NETWORK_SOCKET_ERROR: + /** + * client is gone away + * + * close the connection and clean up + */ + con->prev_state = con->state; + con->state = ST_ERROR; + g_message("%s: client is gone away for con:%p", G_STRLOC, con); + break; + } + + /* if the write failed, don't call the plugin handlers */ + if (con->state != ostate) { + return DISP_CONTINUE; + } + + /* + * in case we havn't read the full resultset + * from the server yet, + * go back and read more + */ + if (!con->resultset_is_finished && con->server) { + con->state = ST_READ_QUERY_RESULT; + return DISP_CONTINUE; + } + + switch (plugin_call(srv, con, con->state)) { + case NETWORK_SOCKET_SUCCESS: + break; + default: + con->prev_state = con->state; + con->state = ST_ERROR; + break; + } + + if (con->data) { + cetus_clean_conn_data(con); + } + + gettimeofday(&(con->resp_send_time), NULL); + handle_query_time_stats(con); + + if (con->client->do_query_cache) { + if (con->client->query_cache_too_long) { + network_queue_free(con->client->cache_queue); + con->client->cache_queue = NULL; + } else { + GString *key = g_string_new(NULL); + g_string_append(key, con->orig_sql->str); + g_string_append(key, con->client->response->username->str); + g_string_append(key, con->client->default_db->str); + g_message("%s:key for cache:%s", G_STRLOC, key->str); + gchar *md5_key = g_compute_checksum_for_string(G_CHECKSUM_MD5, S(key)); + g_string_free(key, TRUE); + + query_cache_item *item = g_hash_table_lookup(srv->query_cache_table, md5_key); + if (item) { + g_warning("%s:key for cache exists:%s", G_STRLOC, md5_key); + g_hash_table_remove(srv->query_cache_table, md5_key); + network_queue_free(con->client->cache_queue); + g_free(md5_key); + } else { + g_debug("%s:put content to cache:%s", G_STRLOC, md5_key); + item = g_new0(query_cache_item, 1); + g_hash_table_insert(srv->query_cache_table, md5_key, item); + gchar *dup_key = g_strdup(md5_key); + query_cache_index_item *index = g_new0(query_cache_index_item, 1); + index->key = dup_key; + unsigned long long access_ms; + access_ms = con->resp_send_time.tv_sec * 1000 + con->resp_send_time.tv_usec / 1000; + index->expire_ms = access_ms + srv->default_query_cache_timeout; + g_queue_push_tail(srv->cache_index, index); + + item->queue = con->client->cache_queue; + } + con->client->cache_queue = NULL; + } + } + + if (con->resultset_is_finished) { + con->client->last_visit_time = time(0); + if (!con->client->is_server_conn_reserved) { + con->client->is_need_q_peek_exec = 1; + g_debug("%s: set is_need_q_peek_exec true, state:%d", + G_STRLOC, con->state); + } else { + con->client->is_need_q_peek_exec = 0; + g_debug("%s: set is_need_q_peek_exec false", + G_STRLOC); + } + } + + if (con->slave_conn_shortaged) { + time_t cur = time(0); + if (con->last_check_conn_supplement_time != cur) { + g_debug("%s: slave conn shortaged, try to add more conns ", G_STRLOC); + network_connection_pool_create_conn(con); + con->last_check_conn_supplement_time = cur; + } else { + g_debug("%s: slave conn shortaged, but time is the same", G_STRLOC); + } + } + + return DISP_CONTINUE; +} + + +static int +normal_read_query_result(network_mysqld_con *con, network_mysqld_con_state_t ostate) +{ + struct timeval timeout; + chassis *srv = con->srv; + /* read all packets of the resultset + * + * depending on the backend we may forward the data + * to the client right away + */ + do { + network_socket *recv_sock; + + recv_sock = con->server; + + if (recv_sock == NULL) { + con->prev_state = con->state; + con->state = ST_ERROR; + return DISP_CONTINUE; + } + + int read_len = recv_sock->to_read; + + if (srv->query_cache_enabled && read_len > 0) { + g_debug("%s: check if query can be cached", G_STRLOC); + proxy_plugin_con_t *st = con->plugin_con_state; + if (con->is_read_ro_server_allowed && st->injected.queries->length == 0) { + if (con->query_cache_judged == 0) { + do_check_qeury_cache(con); + } + } + } + + switch (network_mysqld_read(srv, recv_sock)) { + case NETWORK_SOCKET_SUCCESS: + recv_sock->resp_len += read_len; + if (recv_sock->resp_len > con->srv->max_resp_len) { + network_queue_clear(con->server->recv_queue); + network_mysqld_queue_reset(con->server); + g_message("%s: resp too long:%p, src port:%s, sql:%s", + G_STRLOC, con, recv_sock->src->name->str, + con->orig_sql->str); + network_mysqld_con_send_error_full(con->client, + C("response too long for proxy"), ER_CETUS_LONG_RESP, "HY000"); + con->server_to_be_closed = 1; + con->resultset_is_finished = TRUE; + con->resp_too_long = 1; + } + break; + case NETWORK_SOCKET_WAIT_FOR_EVENT: + timeout = con->read_timeout; + g_debug("%s: set read query timeout, already read:%d", + G_STRLOC, (int) recv_sock->resp_len); + GString *packet; + while ((packet = g_queue_pop_head(con->server->recv_queue->chunks)) != NULL) { + network_mysqld_queue_append_raw(con->client, con->client->send_queue, packet); + } + if (con->client->send_queue->len > 0) { + g_debug("%s: send_part_content_to_client", G_STRLOC); + send_part_content_to_client(con); + } + WAIT_FOR_EVENT(con->server, EV_READ, &timeout); + return DISP_STOP; + case NETWORK_SOCKET_ERROR_RETRY: + case NETWORK_SOCKET_ERROR: + g_critical("%s: read(READ_QUERY_RESULT) error:%p", + G_STRLOC, con); + con->prev_state = con->state; + con->state = ST_ERROR; + break; + } + + switch (plugin_call(srv, con, con->state)) { + case NETWORK_SOCKET_SUCCESS: + /* + * if we don't need the resultset, + * forward it to the client + */ + if (!con->resultset_is_finished && + !con->resultset_is_needed) + { + if (con->client->send_queue->len > 65536) { + con->state = ST_SEND_QUERY_RESULT; + g_warning("%s: send queue len is too big for sock:%p", + G_STRLOC, con->client); + } + } + break; + case NETWORK_SOCKET_ERROR: + /* + * something nasty happend, + * let's close the connection + */ + con->prev_state = con->state; + con->state = ST_ERROR; + break; + default: + g_critical("%s: ...", G_STRLOC); + con->prev_state = con->state; + con->state = ST_ERROR; + break; + } + } while (con->state == ST_READ_QUERY_RESULT); + + return DISP_CONTINUE; +} + +static int +read_query_result_for_sharding(network_mysqld_con *con, network_mysqld_con_state_t ostate) +{ + g_debug("%s: read query result", G_STRLOC); + if (con->resp_too_long) { + g_message("%s: recording resp too long:%d", G_STRLOC, con->state); + } else if (con->server_to_be_closed) { + remove_mul_server_recv_packets(con); + con->state = ST_SEND_QUERY_RESULT; + g_message("%s: send server error to client, state:%d", + G_STRLOC, con->state); + network_mysqld_con_send_error(con->client, + C("MySQL server connection closed")); + network_queue_clear(con->client->recv_queue); + network_mysqld_queue_reset(con->client); + } else { + if (con->partially_merged) { + g_debug("%s: set ST_SEND_QUERY_RESULT here", G_STRLOC); + remove_mul_server_recv_packets(con); + con->state = ST_SEND_QUERY_RESULT; + network_queue_clear(con->client->recv_queue); + network_mysqld_queue_reset(con->client); + con->partially_merged = 0; + } else { + if (handle_read_mul_servers_resp(con) == DISP_STOP) { + return DISP_STOP; + } + } + } + + return DISP_CONTINUE; +} + + +static void +process_read_event(network_mysqld_con *con, int event_fd) +{ + int b = -1; + + /** + * check how much data there is to read + * + * ioctl() + * - returns 0 if connection is closed + * - or -1 and ECONNRESET on solaris + * or -1 and EPIPE on HP/UX + */ + if (ioctl(event_fd, FIONREAD, &b)) { + g_critical("ioctl(%d, FIONREAD, ...) failed: %s", + event_fd, g_strerror(errno)); + con->prev_state = con->state; + con->state = ST_ERROR; + } else if (b != 0) { + if (event_fd == con->client->fd) { + con->client->to_read = b; + g_debug("%s:client to read:%d for con:%p", G_STRLOC, b, con); + } else if (con->server && event_fd == con->server->fd) { + con->server->to_read = b; + g_debug("%s:server to read:%d for con:%p", G_STRLOC, b, con); + } else { + g_error("%s: neither nor", G_STRLOC); + } + } else { /* Linux */ + if (event_fd == con->client->fd) { + /* + * the client closed the connection, + * let's keep the server side open + */ + con->prev_state = con->state; + con->state = ST_CLOSE_CLIENT; + g_debug("%s:client needs to closed for con:%p", G_STRLOC, con); + } else if (con->server && event_fd == con->server->fd && + con->com_quit_seen) + { + con->state = ST_CLOSE_SERVER; + } else { + g_message("%s:server closed prematurely, op: %s", + G_STRLOC, network_mysqld_con_st_name(con->state)); + + network_mysqld_con_send_error_full(con->client, + C("server closed prematurely"), ER_CETUS_UNKNOWN, "HY000"); + con->server_to_be_closed = 1; + con->server_closed = 1; + con->resultset_is_finished = TRUE; + + con->state = ST_SEND_QUERY_RESULT; + } + } +} + + +static void +process_timeout_event(network_mysqld_con *con) +{ + if (con->is_wait_server) { + g_debug("%s:now get a chance to get server connection", G_STRLOC); + } else { + /* + * if we got a timeout on ST_CONNECT_SERVER + * we should pick another backend + */ + switch (plugin_call_timeout(con->srv, con)) { + case NETWORK_SOCKET_SUCCESS: + /* the plugin did set a reasonable next state */ + break; + default: + con->prev_state = con->state; + con->state = ST_ERROR; + break; + } + } +} + + +/** + * handle the different states of the MySQL protocol + * + * @param event_fd fd on which the event was fired + * @param events the event that was fired + * @param user_data the connection handle + */ +void network_mysqld_con_handle(int event_fd, short events, void *user_data) { + network_mysqld_con_state_t ostate; + network_mysqld_con *con = user_data; + chassis *srv = con->srv; + int retval; + + if (events == EV_READ) { + process_read_event(con, event_fd); + } else if (events == EV_TIMEOUT) { + process_timeout_event(con); + } + + /** + * loop on the same connection as long as we don't end up in a stable state + */ + + do { + struct timeval timeout; + + ostate = con->state; +#if NETWORK_DEBUG_TRACE_STATE_CHANGES + /* + * if you need the state-change information without dtrace, + * enable this + */ + g_debug("%s: [%d] %s, con:%p", + G_STRLOC, + getpid(), + network_mysqld_con_st_name(con->state), + con); +#endif + switch (con->state) { + case ST_ERROR: + /* we can't go on, close the connection */ + con->server_to_be_closed = 1; + plugin_call_cleanup(srv, con); + g_debug("%s: client conn %p released", G_STRLOC, con); + network_mysqld_con_free(con); + return; + case ST_CLOSE_CLIENT: + case ST_CLIENT_QUIT: + case ST_CLOSE_SERVER: + /* FIXME: this comment has nothing to do with reality... + * the server connection is still fine, + * let's keep it open for reuse */ + plugin_call_cleanup(srv, con); + g_debug("%s: client conn %p released, state:%d", G_STRLOC, + con, con->state); + + network_mysqld_con_free(con); + + con = NULL; + + return; + case ST_INIT: + /* + * if we are a proxy ask the remote server + * for the hand-shake packet + * if not, we generate one + */ + switch (plugin_call(srv, con, con->state)) { + case NETWORK_SOCKET_SUCCESS: + break; + default: + /** + * no luck, let's close the connection + */ + g_critical("%s: ST_INIT not successful", G_STRLOC); + + con->prev_state = con->state; + con->state = ST_ERROR; + + break; + } + + break; + case ST_CONNECT_SERVER: + switch ((retval = plugin_call(srv, con, con->state))) { + case NETWORK_SOCKET_SUCCESS: + + /** + * hmm, if this is success and we have something + * in the clients send-queue + * we just send it out ... who needs a server ? + */ + + if (con->client->send_queue->chunks->length > 0 && con->server == NULL) { + /* we want to send something to the client */ + con->state = ST_SEND_HANDSHAKE; + } + + break; + default: + g_critical("%s: hook for CONNECT_SERVER invalid: %d", + G_STRLOC, + retval); + + con->prev_state = con->state; + con->state = ST_ERROR; + + break; + } + + break; + case ST_SEND_HANDSHAKE: + /* send the hand-shake to the client and + * wait for a response + */ + switch (network_mysqld_write(srv, con->client)) { + case NETWORK_SOCKET_SUCCESS: + break; + case NETWORK_SOCKET_WAIT_FOR_EVENT: + timeout = con->write_timeout; + + WAIT_FOR_EVENT(con->client, EV_WRITE, &timeout); + + return; + case NETWORK_SOCKET_ERROR_RETRY: + case NETWORK_SOCKET_ERROR: + /** + * writing failed, closing connection + */ + con->prev_state = con->state; + con->state = ST_ERROR; + break; + } + + if (con->state != ostate) break; + + switch (plugin_call(srv, con, con->state)) { + case NETWORK_SOCKET_SUCCESS: + break; + default: + g_critical("%s: plugin_call(SEND_HANDSHAKE) failed", + G_STRLOC); + con->prev_state = con->state; + con->state = ST_ERROR; + break; + } + break; + case ST_READ_AUTH: { + /* read auth from client */ + network_socket *recv_sock; + + recv_sock = con->client; + + g_assert(events == 0 || event_fd == recv_sock->fd); + + switch (network_mysqld_read(srv, recv_sock)) { + case NETWORK_SOCKET_SUCCESS: + break; + case NETWORK_SOCKET_WAIT_FOR_EVENT: + timeout = con->read_timeout; + g_debug("%s: set read query timeout for con:%p", G_STRLOC, con); + + WAIT_FOR_EVENT(con->client, EV_READ, &timeout); + + return; + case NETWORK_SOCKET_ERROR_RETRY: + case NETWORK_SOCKET_ERROR: + con->state = ST_ERROR; + g_message("%s: ST_READ_AUTH: read error for con:%p", + G_STRLOC, con); + break; + } + + if (con->state != ostate) break; + + switch (plugin_call(srv, con, con->state)) { + case NETWORK_SOCKET_SUCCESS: + break; + case NETWORK_SOCKET_ERROR: + con->state = ST_SEND_ERROR; + break; + default: + g_critical("%s: plugin_call(READ_AUTH) failed", + G_STRLOC); + con->prev_state = con->state; + con->state = ST_ERROR; + break; + } + + break; + } + case ST_SEND_AUTH_RESULT: + switch (network_mysqld_write(srv, con->client)) { + case NETWORK_SOCKET_SUCCESS: + break; + case NETWORK_SOCKET_WAIT_FOR_EVENT: + timeout = con->write_timeout; + + WAIT_FOR_EVENT(con->client, EV_WRITE, &timeout); + return; + case NETWORK_SOCKET_ERROR_RETRY: + case NETWORK_SOCKET_ERROR: + g_debug("%s: write(AUTH_RESULT) error", + G_STRLOC); + con->prev_state = con->state; + con->state = ST_ERROR; + break; + } + + if (con->state != ostate) break; + + switch (plugin_call(srv, con, con->state)) { + case NETWORK_SOCKET_SUCCESS: + break; + default: + g_critical("%s: ...", G_STRLOC); + con->prev_state = con->state; + con->state = ST_ERROR; + break; + } + + break; + case ST_GET_SERVER_CONNECTION_LIST: + switch (plugin_call(srv, con, con->state)) { + case NETWORK_SOCKET_SUCCESS: + con->state = ST_SEND_QUERY; + if (con->retry_serv_cnt > 0 && con->is_wait_server) { + g_message("%s: wait successful:%d, con:%p, state:%d", + G_STRLOC, con->retry_serv_cnt, con, con->state); + } + con->is_wait_server = 0; + con->retry_serv_cnt = 0; + break; + case NETWORK_SOCKET_WAIT_FOR_EVENT: + if (con->retry_serv_cnt < con->max_retry_serv_cnt) { + con->master_conn_shortaged = 1; + con->retry_serv_cnt++; + con->is_wait_server = 1; + g_debug("%s:PROXY_NO_CONNECTION", G_STRLOC); + if (con->retry_serv_cnt == 0 || con->retry_serv_cnt == 8) { + network_connection_pool_create_conn(con); + } + struct timeval timeout = + network_mysqld_con_retry_timeout(con); + + g_debug(G_STRLOC ": wait again:%d, con:%p, l:%d", + con->retry_serv_cnt, con, (int) timeout.tv_usec); + WAIT_FOR_EVENT(con->client, EV_TIMEOUT, &timeout); + return; + } else { + con->is_wait_server = 0; + con->retry_serv_cnt = 0; + process_service_unavailable(con); + } + + break; + + default: + process_service_unavailable(con); + break; + } + break; + case ST_READ_QUERY: + g_debug("%s:call here.", G_STRLOC); + CHECK_PENDING_EVENT(&(con->client->event)); + + /* TODO If config is reloaded, close all current cons */ + g_assert(events == 0 || event_fd == con->client->fd); + if (handle_read_query(con, ostate) == DISP_STOP) { + return; + } + break; + case ST_SEND_QUERY: + if (handle_send_query_to_servers(con, ostate) == DISP_STOP) { + return; + } + break; + case ST_READ_QUERY_RESULT: + if (normal_read_query_result(con, ostate) == DISP_STOP) { + return; + } + break; + case ST_READ_M_QUERY_RESULT: + if (read_query_result_for_sharding(con, ostate) == DISP_STOP) { + return; + } + break; + + case ST_SEND_QUERY_RESULT: + g_debug("%s: send query result for con:%p", G_STRLOC, con); + if (send_result_to_client(con, ostate) == DISP_STOP) { + return; + } + break; + case ST_SEND_ERROR: + /** + * send error to the client + * and close the connections afterwards + */ + switch (network_mysqld_write(srv, con->client)) { + case NETWORK_SOCKET_SUCCESS: + break; + case NETWORK_SOCKET_WAIT_FOR_EVENT: + timeout = con->write_timeout; + + WAIT_FOR_EVENT(con->client, EV_WRITE, &timeout); + return; + case NETWORK_SOCKET_ERROR_RETRY: + case NETWORK_SOCKET_ERROR: + g_critical("%s: write(SEND_ERROR) error", G_STRLOC); + + con->prev_state = con->state; + con->state = ST_ERROR; + break; + } + + con->prev_state = con->state; + con->state = ST_CLOSE_CLIENT; + g_debug("%s:client needs to closed for con:%p", G_STRLOC, con); + + break; + default: + con->prev_state = con->state; + con->state = ST_ERROR; + break; + } + + event_fd = -1; + events = 0; + } while (ostate != con->state); + + return; +} + +static gboolean update_accept_event(network_mysqld_con *con, const int new_flags) { + g_assert(con != NULL); + struct event *ev = &(con->server->event); + struct event_base *base = ev->ev_base; + if (ev->ev_flags == new_flags) + return TRUE; + if (event_del(ev) == -1) + return FALSE; + event_set(ev, con->server->fd, new_flags, network_mysqld_con_accept, con); + event_base_set(base, ev); + event_add(ev, 0); + return TRUE; +} + +static void accept_new_conns(chassis *chas, const gboolean do_accept); + +static struct event maxconnsevent; +static void maxconns_handler(const int fd, const short which, void *arg) +{ + struct timeval t = {.tv_sec = 0, .tv_usec = 10000}; + + chassis* chas = arg; + if (fd == -42 || chas->allow_new_conns == FALSE) { + /* reschedule in 10ms if we need to keep polling */ + evtimer_set(&maxconnsevent, maxconns_handler, chas); + event_base_set(chas->event_base, &maxconnsevent); + evtimer_add(&maxconnsevent, &t); + } else { + evtimer_del(&maxconnsevent); + accept_new_conns(chas, TRUE); + g_message("Got vacant fd, start accept"); + } +} + +/* + * Sets whether we are listening for new connections or not. + */ +static void accept_new_conns(chassis *chas, const gboolean do_accept) +{ + GList *l; + for (l = chas->priv->listen_conns; l; l = l->next) { + network_mysqld_con *con = l->data; + if (do_accept) { + update_accept_event(con, EV_READ | EV_PERSIST); + if (listen(con->server->fd, 128) != 0) { + g_warning("listen errno: %d", errno); + } + } else { + update_accept_event(con, 0); + if (listen(con->server->fd, 0) != 0) { + g_warning("listen errno: %d", errno); + } + } + } + if (!do_accept) { + chas->allow_new_conns = FALSE; + maxconns_handler(-42, 0, chas); + } +} + +/** + * accept a connection + * + * event handler for listening connections + * + * @param event_fd fd on which the event was fired + * @param events the event that was fired + * @param user_data the listening connection handle + * + */ +void network_mysqld_con_accept(int G_GNUC_UNUSED event_fd, short events, + void *user_data) +{ + network_mysqld_con *listen_con = user_data; + network_mysqld_con *client_con; + network_socket *client; + + g_assert(events == EV_READ); + g_assert(listen_con->server); + int reason = 0; + client = network_socket_accept(listen_con->server, &reason); + if (!client) { + if (reason == EMFILE) { /* if reach max fd, stop accepting */ + g_warning("EMFILE (Too many open files), stop accept"); + accept_new_conns(listen_con->srv, FALSE); + } + return; + } + + /* looks like we open a client connection */ + client_con = network_mysqld_con_new(); + client_con->client = client; + + g_debug("%s: add a new client connection: %p", + G_STRLOC, client_con); + + network_mysqld_add_connection(listen_con->srv, client_con, FALSE); + + client_con->key = client_con->srv->sess_key++; + + /** + * inherit the config to the new connection + */ + + client_con->plugins = listen_con->plugins; + client_con->config = listen_con->config; + + network_mysqld_con_handle(-1, 0, client_con); + + return; +} + +/** + * @TODO move to network_mysqld_proto + */ +int network_mysqld_con_send_resultset(network_socket *con, GPtrArray *fields, + GPtrArray *rows) +{ + GString *s; + gsize i, j; + + g_assert(fields->len > 0); + + s = g_string_new(NULL); + + /* - len = 99 + * \1\0\0\1 + * \1 - one field + * \'\0\0\2 + * \3def + * \0 + * \0 + * \0 + * \21@@version_comment + * \0 - org-name + * \f - filler + * \10\0 - charset + * \34\0\0\0 - length + * \375 - type + * \1\0 - flags + * \37 - decimals + * \0\0 - filler + * \5\0\0\3 + * \376\0\0\2\0 + * \35\0\0\4 + * \34MySQL Community Server (GPL) + * \5\0\0\5 + * \376\0\0\2\0 + */ + + /* the field-count */ + network_mysqld_proto_append_lenenc_int(s, fields->len); + network_mysqld_queue_append(con, con->send_queue, S(s)); + + for (i = 0; i < fields->len; i++) { + MYSQL_FIELD *field = fields->pdata[i]; + + g_string_truncate(s, 0); + + network_mysqld_proto_append_lenenc_str(s, + field->catalog ? field->catalog : "def"); + network_mysqld_proto_append_lenenc_str(s, + field->db ? field->db : ""); + network_mysqld_proto_append_lenenc_str(s, + field->table ? field->table : ""); + network_mysqld_proto_append_lenenc_str(s, + field->org_table ? field->org_table : ""); + network_mysqld_proto_append_lenenc_str(s, + field->name ? field->name : ""); + network_mysqld_proto_append_lenenc_str(s, + field->org_name ? field->org_name : ""); + + /* length of the following block, 12 byte */ + g_string_append_c(s, '\x0c'); + g_string_append_len(s, "\x08\x00", 2); /* charset */ + g_string_append_c(s, (field->length >> 0) & 0xff); /* len */ + g_string_append_c(s, (field->length >> 8) & 0xff); /* len */ + g_string_append_c(s, (field->length >> 16) & 0xff); /* len */ + g_string_append_c(s, (field->length >> 24) & 0xff); /* len */ + g_string_append_c(s, field->type); /* type */ + g_string_append_c(s, field->flags & 0xff); /* flags */ + g_string_append_c(s, (field->flags >> 8) & 0xff); /* flags */ + g_string_append_c(s, 0); /* decimals */ + g_string_append_len(s, "\x00\x00", 2); /* filler */ + network_mysqld_queue_append(con, con->send_queue, S(s)); + } + + g_string_truncate(s, 0); + + /* EOF */ + g_string_append_len(s, "\xfe", 1); /* EOF */ + g_string_append_len(s, "\x00\x00", 2); /* warning count */ + g_string_append_len(s, "\x02\x00", 2); /* flags */ + + network_mysqld_queue_append(con, con->send_queue, S(s)); + + for (i = 0; i < rows->len; i++) { + GPtrArray *row = rows->pdata[i]; + + g_string_truncate(s, 0); + + for (j = 0; j < row->len; j++) { + network_mysqld_proto_append_lenenc_str(s, row->pdata[j]); + } + + network_mysqld_queue_append(con, con->send_queue, S(s)); + } + + g_string_truncate(s, 0); + + /* EOF */ + g_string_append_len(s, "\xfe", 1); /* EOF */ + g_string_append_len(s, "\x00\x00", 2); /* warning count */ + g_string_append_len(s, "\x02\x00", 2); /* flags */ + + network_mysqld_queue_append(con, con->send_queue, S(s)); + network_mysqld_queue_reset(con); + + g_string_free(s, TRUE); + + return 0; +} + +int network_mysqld_con_send_current_date(network_socket *con, const char *name) +{ + GPtrArray *fields = g_ptr_array_new_with_free_func( + (void *) network_mysqld_proto_fielddef_free); + MYSQL_FIELD *field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup(name); + field->type = MYSQL_TYPE_VAR_STRING; + g_ptr_array_add(fields, field); + + char date[32] = {0}; + time_t tepoch = time(0); + struct tm tsep = {0}; + localtime_r(&tepoch, &tsep); + strftime(date, sizeof(date), "%Y-%m-%d", &tsep); + + GPtrArray *rows = g_ptr_array_new_with_free_func( + (void *) network_mysqld_mysql_field_row_free); + GPtrArray *row = g_ptr_array_new(); + g_ptr_array_add(row, date); + g_ptr_array_add(rows, row); + + network_mysqld_con_send_resultset(con, fields, rows); + g_ptr_array_free(fields, TRUE); + g_ptr_array_free(rows, TRUE); + return 0; +} + +int network_mysqld_con_send_cetus_version(network_socket *con) +{ + GPtrArray *fields = g_ptr_array_new_with_free_func( + (void *) network_mysqld_proto_fielddef_free); + MYSQL_FIELD *field = network_mysqld_proto_fielddef_new(); + field->name = g_strdup("cetus version"); + field->type = MYSQL_TYPE_VAR_STRING; + g_ptr_array_add(fields, field); + + char version[128] = {0}; +#ifdef CHASSIS_BUILD_TAG + snprintf(version, sizeof(version), "%s (build:%s)", PACKAGE_STRING, CHASSIS_BUILD_TAG); +#else + strncpy(version, PACKAGE_STRING, sizeof(version)); +#endif + GPtrArray *rows = g_ptr_array_new_with_free_func( + (void *) network_mysqld_mysql_field_row_free); + GPtrArray *row = g_ptr_array_new(); + g_ptr_array_add(row, version); + g_ptr_array_add(rows, row); + + network_mysqld_con_send_resultset(con, fields, rows); + g_ptr_array_free(fields, TRUE); + g_ptr_array_free(rows, TRUE); + return 0; +} + +void network_mysqld_send_xa_start(network_socket *sock, const char *xid) +{ + GString *packet = g_string_sized_new(64); + packet->len = NET_HEADER_SIZE; + g_string_append_c(packet, (char) COM_QUERY); + g_string_append(packet, "XA START "); + g_string_append(packet, xid); + network_mysqld_proto_set_packet_len(packet, 10 + strlen(xid)); + network_mysqld_proto_set_packet_id(packet, 0); + network_queue_append(sock->send_queue, packet); +} + +static retval_t proxy_self_read_handshake(chassis *srv, server_connection_state_t *con) +{ + /* server_connection_state_t *con */ + int err = 0; + guint8 status = 0; + network_packet packet; + network_socket *recv_sock; + network_mysqld_auth_challenge *challenge; + + recv_sock = con->server; + + packet.data = g_queue_peek_tail(recv_sock->recv_queue->chunks); + packet.offset = 0; + + err = err || network_mysqld_proto_skip_network_header(&packet); + if (err) return RET_ERROR; + + err = err || network_mysqld_proto_peek_int8(&packet, &status); + if (err) return RET_ERROR; + + if (status == 0xff) { + return RET_ERROR; + } + + challenge = network_mysqld_auth_challenge_new(); + if (network_mysqld_proto_get_auth_challenge(&packet, challenge)) { + g_string_free(g_queue_pop_tail(recv_sock->recv_queue->chunks), TRUE); + + network_mysqld_auth_challenge_free(challenge); + + return RET_ERROR; + } + + if (con->srv->sharding_mode) { + if (challenge->server_version < 50707) { + g_warning("%s: for xa, server:%s, mysql version:%s is lower than 5.7.7", + G_STRLOC, recv_sock->dst->name->str, challenge->server_version_str); + } + } + + if (!con->srv->client_found_rows) { + challenge->capabilities &= ~(CLIENT_FOUND_ROWS); + } + + if (!con->srv->compress_support) { + challenge->capabilities &= ~(CLIENT_COMPRESS); + } + + con->server->challenge = challenge; + network_backend_save_challenge(con->backend, challenge); + + return RET_SUCCESS; +} + +static retval_t proxy_self_create_auth(chassis *srv, server_connection_state_t *con) +{ + network_socket *send_sock = con->server; + + const network_mysqld_auth_challenge *challenge = send_sock->challenge; + network_mysqld_auth_response *auth + = network_mysqld_auth_response_new(challenge->capabilities); + + auth->client_capabilities = CETUS_DEFAULT_FLAGS; + + if (srv->is_back_compressed) { + auth->client_capabilities |= CLIENT_COMPRESS; + } + + if (send_sock->default_db->len == 0) { + auth->client_capabilities &= ~CLIENT_CONNECT_WITH_DB; + } + if (!srv->client_found_rows) { + auth->client_capabilities &= ~CLIENT_FOUND_ROWS; + } + + auth->max_packet_size = 0x01000000; + auth->charset = con->charset_code; + con->is_multi_stmt_set = 1; + g_debug("%s:set multi stmt true for con:%p", G_STRLOC, con); + + g_string_truncate(auth->auth_plugin_data, 0); + network_mysqld_proto_password_scramble(auth->auth_plugin_data, + S(challenge->auth_plugin_data), + S(con->hashed_pwd)); + + g_string_assign_len(challenge->scrambled_password, S(auth->auth_plugin_data)); + + g_string_append_len(auth->database, S(send_sock->default_db)); + g_string_assign_len(auth->username, S(send_sock->username)); + g_debug("%s:username: %s ", G_STRLOC, send_sock->username->str); + + GString *packet = g_string_new(NULL); + network_mysqld_proto_append_auth_response(packet, auth); + send_sock->last_packet_id = 0; + network_mysqld_queue_append(send_sock, send_sock->send_queue, S(packet)); + g_string_free(packet, TRUE); + + send_sock->response = auth; + return RET_SUCCESS; +} + +server_connection_state_t *network_mysqld_self_con_init(chassis *srv) { + server_connection_state_t *con; + + con = g_new0(server_connection_state_t, 1); + + con->srv = srv; + con->server = network_socket_new(); + con->state = ST_ASYNC_CONN; + con->hashed_pwd = g_string_new(0); + + return con; +} + +void network_mysqld_self_con_free(server_connection_state_t *con) { + if (!con) return; + + if (con->server) { + network_socket_free(con->server); + g_debug("%s: connection server free:%p", G_STRLOC, con); + con->server = NULL; + } else { + g_debug("%s: connections server is null:%p", G_STRLOC, con); + } + g_string_free(con->hashed_pwd, TRUE); + g_debug("%s: connections free :%p", G_STRLOC, con); + + g_free(con); +} + +#define ASYNC_WAIT_FOR_EVENT(sock, ev_type, timeout, user_data) \ +event_set(&(sock->event), sock->fd, ev_type, network_mysqld_self_con_handle, user_data); \ +chassis_event_add_with_timeout(srv, &(sock->event), timeout); + +static int +process_self_event(server_connection_state_t *con, int events, int event_fd) +{ + if (events == EV_READ) { + int b = -1; + if (ioctl(con->server->fd, FIONREAD, &b)) { + g_warning("ioctl(%d, FIONREAD, ...) failed: %s", + event_fd, strerror(errno)); + con->state = ST_ASYNC_ERROR; + } else if (b != 0) { + con->server->to_read = b; + } else { + if (errno == 0 || errno == EWOULDBLOCK) { + return 0; + } else { + g_warning("%s:ERROR EV_READ errno=%d error:%s, state:%d, con:%p, server:%p", + G_STRLOC, errno, strerror(errno), con->state, con, con->server); + con->state = ST_ASYNC_ERROR; + } + } + } else if (events == EV_TIMEOUT) { + g_debug("%s:timeout, ev:%p",G_STRLOC, (&con->server->event)); + if (con->state == ST_ASYNC_CONN) { + g_message("%s: self conn timeout, state:%d, con:%p, server:%p", + G_STRLOC, con->state, con, con->server); + con->state = ST_ASYNC_ERROR; + con->backend->state = BACKEND_STATE_DOWN; + g_message("%s: set backend:%p down", G_STRLOC, con->backend); + } + } + + return 1; +} + +static int process_self_server_read(server_connection_state_t *con) +{ + chassis *srv = con->srv; + switch (network_mysqld_read(con->srv, con->server)) { + case NETWORK_SOCKET_SUCCESS: + break; + case NETWORK_SOCKET_WAIT_FOR_EVENT: { + /* call us again when you have a event */ + ASYNC_WAIT_FOR_EVENT(con->server, EV_READ, NULL, con); + return 0; + } + case NETWORK_SOCKET_ERROR: + g_warning("%s:plugin_call(ASYNC_READ_HANDSHAKE) error", + G_STRLOC); + con->state = ST_ASYNC_ERROR; + break; + default: + g_warning("%s:unexpected state", G_STRLOC); + con->state = ST_ASYNC_ERROR; + break; + } + + return 1; +} + +static int process_self_read_auth_result(server_connection_state_t *con) +{ + + if (!process_self_server_read(con)) { + return 0; + } + + if (con->state == ST_ASYNC_ERROR) { + g_warning("%s: con:%p auth failed,crazy here", + G_STRLOC, con); + return 1; + } + + GString *packet = g_queue_peek_head(con->server->recv_queue->chunks); + int type = packet->str[NET_HEADER_SIZE]; + switch (type) { + case MYSQLD_PACKET_OK: + break; + case MYSQLD_PACKET_ERR: + g_warning("%s: error send AUTH_RESULT: %02x", + G_STRLOC, packet->str[NET_HEADER_SIZE]); + con->state = ST_ASYNC_ERROR; + break; + case MYSQLD_PACKET_EOF: + con->state = ST_ASYNC_ERROR; + g_warning("%s: the MySQL 4.0 hash in a MySQL 4.1+ connection", + G_STRLOC); + break; + default: { + network_packet pkt; + pkt.data = packet; + pkt.offset = NET_HEADER_SIZE; + network_mysqld_err_packet_t *err_packet; + err_packet = network_mysqld_err_packet_new(); + if (!network_mysqld_proto_get_err_packet(&pkt, err_packet)) { + g_warning("%s:READ_AUTH_RESULT:%d, server:%s, errinfo:%s", + G_STRLOC, type, + con->server->response->username->str, + err_packet->errmsg->str); + } + network_mysqld_err_packet_free(err_packet); + con->state = ST_ASYNC_ERROR; + break; + } + } + + if (con->state != ST_ASYNC_ERROR) { + con->backend->connected_clients--; + g_debug("%s: connected_clients sub, now:%d for con:%p", + G_STRLOC, con->backend->connected_clients, con); + g_message("%s: backend:%s, new connection:%p", + G_STRLOC, con->backend->addr->name->str, con->server); + network_mysqld_queue_reset(con->server); + network_queue_clear(con->server->recv_queue); + con->server->is_multi_stmt_set = con->is_multi_stmt_set; +#if NETWORK_DEBUG_TRACE_EVENT + CHECK_PENDING_EVENT(&(con->server->event)); +#endif + if (con->srv->is_back_compressed) { + con->server->do_compress = 1; + } + network_pool_add_idle_conn(con->pool, con->srv, con->server); + con->server = NULL; /* tell _self_con_free we succeed */ + network_mysqld_self_con_free(con); + + return 0; + } + + return 1; +} + + +static void network_mysqld_self_con_handle(int event_fd, short events, + void *user_data) +{ + int ostate; + server_connection_state_t *con = (server_connection_state_t *) user_data; + chassis *srv = con->srv; + + if (!process_self_event(con, events, event_fd)) { + return; + } + + do { + ostate = con->state; + + switch (con->state) { + case ST_ASYNC_ERROR: + g_warning("%s: con:%p failed for server:%p", G_STRLOC, + con, con->server); + con->backend->connected_clients--; + g_debug("%s: connected_clients sub, now:%d for con:%p", + G_STRLOC, con->backend->connected_clients, con); + network_mysqld_self_con_free(con); + return; + case ST_ASYNC_CONN: + switch (network_socket_connect_finish(con->server)) { + case NETWORK_SOCKET_SUCCESS: + if (con->backend->state != BACKEND_STATE_UP) { + con->backend->state = BACKEND_STATE_UP; + g_get_current_time(&(con->backend->state_since)); + g_message(G_STRLOC ": set backend: %s (%p) up", + con->backend->addr->name->str, con->backend); + } + con->state = ST_ASYNC_READ_HANDSHAKE; + break; + default: + con->state = ST_ASYNC_ERROR; + con->backend->state = BACKEND_STATE_DOWN; + g_message(G_STRLOC ": set backend: %s (%p) down", + con->backend->addr->name->str, con->backend); + break; + } + break; + case ST_ASYNC_READ_HANDSHAKE: + g_assert(events == 0 || event_fd == con->server->fd); + + if (!process_self_server_read(con)) { + return; + } + + switch(proxy_self_read_handshake(srv, con)) { + case RET_SUCCESS: + break; + default: + con->state = ST_ASYNC_ERROR; + g_warning("%s: ...", G_STRLOC); + break; + } + + if (con->state != ST_ASYNC_ERROR) { + con->state = ST_ASYNC_SEND_AUTH; + } + break; + case ST_ASYNC_SEND_AUTH: + proxy_self_create_auth(srv, con); + network_queue_clear(con->server->recv_queue); + + switch (network_mysqld_write(con->srv, con->server)) { + case NETWORK_SOCKET_SUCCESS: + con->state = ST_ASYNC_READ_AUTH_RESULT; + break; + case NETWORK_SOCKET_WAIT_FOR_EVENT: { + ASYNC_WAIT_FOR_EVENT(con->server, EV_WRITE, NULL, con); + return; + } + case NETWORK_SOCKET_ERROR: + con->state = ST_ASYNC_ERROR; + break; + default: + g_warning("%s:unexpected state", G_STRLOC); + break; + } + break; + case ST_ASYNC_READ_AUTH_RESULT: + if (!process_self_read_auth_result(con)) { + return; + } + break; + } + + event_fd = -1; + events = 0; + } while (ostate != con->state); + + return; +} + +void network_connection_pool_create_conn(network_mysqld_con *con) { + chassis *srv = con->srv; + chassis_private *g = srv->priv; + + int i; + char *username; + + g_debug("%s: call network_connection_pool_create_conn", G_STRLOC); + + if (con->client->response == NULL) { + username = srv->default_username; + } else { + username = con->client->response->username->str; + } + + if (!cetus_users_contains(g->users, username)) { + g_message("%s: hashed password is null for user:%s", G_STRLOC, username); + return; + } + + time_t cur = time(0); + + int back_num = network_backends_count(g->backends); + for (i = 0; i < back_num; i++) { + network_backend_t *backend = network_backends_get(g->backends, i); + if (backend != NULL) { + + if (backend->state != BACKEND_STATE_UP) { + if (backend->last_check_time == cur) { + g_debug("%s: omit create, backend:%d state:%d", + G_STRLOC, i, backend->state); + continue; + } + backend->last_check_time = cur; + } else { + if (cur == backend->last_check_time) { + continue; + } + } + + network_connection_pool *pool = backend->pool; + int max_allowed_conn_num; + if (backend->config) { + if (pool->max_idle_connections > + backend->config->max_conn_pool) + { + backend->config->max_conn_pool = pool->max_idle_connections; + g_message("%s: set max conn pool size:%d", + G_STRLOC, backend->config->max_conn_pool); + + } + max_allowed_conn_num = backend->config->max_conn_pool; + } else { + max_allowed_conn_num = pool->max_idle_connections; + } + + int total = network_backend_conns_count(backend); + g_debug("%s: backend ndx:%d total conn:%d, max allowed:%d", + G_STRLOC, i, total, max_allowed_conn_num); + + if (total >= max_allowed_conn_num) { + g_message("%s: backend ndx:%d reach max conn num:%d", + G_STRLOC, i, max_allowed_conn_num); + backend->last_check_time = cur; + continue; + } else if (total >= pool->mid_idle_connections) { + int is_need_to_create = 0; + switch(backend->type) { + case BACKEND_TYPE_RW: + if (con->master_conn_shortaged) { + is_need_to_create = 1; + } + break; + case BACKEND_TYPE_RO: + if (con->slave_conn_shortaged) { + is_need_to_create = 1; + } + break; + default: + break; + } + if (!is_need_to_create) { + continue; + } + } + + server_connection_state_t *scs = network_mysqld_self_con_init(srv); + + g_message("%s: create %s connection for backend ndx:%d, ptr:%p", + G_STRLOC, username, i, backend); + + scs->charset_code = con->client->charset_code; + if (srv->disable_dns_cache) + network_address_set_address(scs->server->dst, backend->address->str); + else + network_address_copy(scs->server->dst, backend->addr); + + scs->pool = backend->pool; + scs->backend = backend; + cetus_users_get_hashed_server_pwd(con->srv->priv->users, username, scs->hashed_pwd); + + /* Avoid the event base's time cache problems */ + scs->connect_timeout.tv_sec = 60; + scs->connect_timeout.tv_usec = 0; + + g_string_append(scs->server->username, username); + + if (con->client->default_db && con->client->default_db->len > 0) { + g_string_append(scs->server->default_db, + con->client->default_db->str); + g_debug("%s:set server default db:%s for con:%p", + G_STRLOC, scs->server->default_db->str, con); + } + + scs->backend->connected_clients++; + g_message("%s: connected_clients add, backend ndx:%d, for server:%p, faked con:%p", + G_STRLOC, i, scs->server, scs); + + switch (network_socket_connect(scs->server)) { + case NETWORK_SOCKET_ERROR_RETRY: { + scs->state = ST_ASYNC_CONN; + struct timeval timeout = scs->connect_timeout; + g_debug("%s: set timeout:%d for new conn", G_STRLOC, + (int) scs->connect_timeout.tv_sec); + ASYNC_WAIT_FOR_EVENT(scs->server, EV_WRITE, &timeout, scs); + break; + } + case NETWORK_SOCKET_SUCCESS: + if (backend->state != BACKEND_STATE_UP) { + backend->state = BACKEND_STATE_UP; + g_get_current_time(&(backend->state_since)); + g_message("%s: set backend:%p, ndx:%d up", G_STRLOC, backend, i); + } + ASYNC_WAIT_FOR_EVENT(scs->server, EV_READ, 0, scs); + scs->state = ST_ASYNC_READ_HANDSHAKE; + break; + default: + scs->backend->connected_clients--; + backend->state = BACKEND_STATE_DOWN; + g_get_current_time(&(backend->state_since)); + network_mysqld_self_con_free(scs); + g_message("%s: set backend ndx:%d down", G_STRLOC, i); + break; + } + } + } +} + + +void network_connection_pool_create_conns(chassis *srv) { + int i, j; + chassis_private *g = srv->priv; + + for (i = 0; i < network_backends_count(g->backends); i++) { + network_backend_t *backend = network_backends_get(g->backends, i); + if (backend != NULL) { + for (j = 0; j < backend->config->mid_conn_pool; j++) { + server_connection_state_t *scs = network_mysqld_self_con_init(srv); + if (srv->disable_dns_cache) + network_address_set_address(scs->server->dst, backend->address->str); + else + network_address_copy(scs->server->dst, backend->addr); + + scs->backend = backend; + scs->pool = backend->pool; + scs->charset_code = backend->config->charset; + g_string_append(scs->server->username, + backend->config->default_username->str); + cetus_users_get_hashed_server_pwd(g->users, scs->server->username->str, + scs->hashed_pwd); + + scs->connect_timeout.tv_sec = 3; + scs->connect_timeout.tv_usec = 0; + + if (backend->config->default_db && + backend->config->default_db->len > 0) + { + g_string_append(scs->server->default_db, + backend->config->default_db->str); + g_debug("%s:set server default db:%s for con:%p", + G_STRLOC, scs->server->default_db->str, scs); + + } + + g_message("%s: connected_clients add, backend ndx:%d, for server:%p, faked con:%p", + G_STRLOC, i, scs->server, scs); + + scs->backend->connected_clients++; + switch(network_socket_connect(scs->server)) { + case NETWORK_SOCKET_ERROR_RETRY: { + scs->state = ST_ASYNC_CONN; + struct timeval timeout = scs->connect_timeout; + ASYNC_WAIT_FOR_EVENT(scs->server, EV_WRITE, &timeout, scs); + break; + } + case NETWORK_SOCKET_SUCCESS: + if (backend->state != BACKEND_STATE_UP) { + backend->state = BACKEND_STATE_UP; + g_message("%s: set backend:%p, ndx:%d up", G_STRLOC, backend, i); + g_get_current_time(&(backend->state_since)); + } + ASYNC_WAIT_FOR_EVENT(scs->server, EV_READ, 0, scs); + scs->state = ST_ASYNC_READ_HANDSHAKE; + g_message("%s: set backend conn:%p read handshake", G_STRLOC, scs); + break; + default: + scs->backend->connected_clients--; + network_mysqld_self_con_free(scs); + backend->state = BACKEND_STATE_DOWN; + g_get_current_time(&(backend->state_since)); + g_message("%s: set backend ndx:%d down, connected_clients sub", G_STRLOC, i); + break; + } + } + } + } +} + +void network_mysqld_con_set_sharding_plan(network_mysqld_con *con, sharding_plan_t *plan) +{ + if (con->sharding_plan) { + sharding_plan_free(con->sharding_plan); + } + con->sharding_plan = plan; +} diff --git a/src/network-mysqld.h b/src/network-mysqld.h new file mode 100644 index 0000000..497e801 --- /dev/null +++ b/src/network-mysqld.h @@ -0,0 +1,759 @@ +#ifndef _NETWORK_MYSQLD_H_ +#define _NETWORK_MYSQLD_H_ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_SYS_TIME_H +/** + * event.h needs struct timeval and doesn't include sys/time.h itself + */ +#include +#endif + +#include + +#include + +#include + +#include + +#include "network-exports.h" + +#include "network-injection.h" +#include "network-socket.h" +#include "network-conn-pool.h" +#include "chassis-plugin.h" +#include "chassis-mainloop.h" +#include "chassis-timings.h" +#include "sys-pedantic.h" +#include "network-backend.h" +#include "cetus-error.h" + +#define XID_LEN 64 +#define COMPRESS_BUF_SIZE 1048576 + +typedef enum { + PROXY_NO_DECISION, + PROXY_NO_CONNECTION, /* TODO: this one shouldn't be here, it's not a dicsion */ + PROXY_SEND_QUERY, + PROXY_SEND_RESULT, + PROXY_SEND_INJECTION, + PROXY_SEND_NONE, + PROXY_IGNORE_RESULT /** for read_query_result */ +} network_mysqld_stmt_ret; + +typedef struct network_mysqld_con network_mysqld_con; /* forward declaration */ + +/** + * A macro that produces a plugin callback function pointer declaration. + */ +#define NETWORK_MYSQLD_PLUGIN_FUNC(x) network_socket_retval_t (*x)(chassis *, network_mysqld_con *) +/** + * The prototype for plugin callback functions. + * + * Some plugins don't use the global "chas" pointer, thus it is marked "unused" for GCC. + */ +#define NETWORK_MYSQLD_PLUGIN_PROTO(x) static network_socket_retval_t x(chassis G_GNUC_UNUSED *chas, network_mysqld_con *con) + +/** + * The function pointers to plugin callbacks + * for each customizable state in the MySQL Protocol. + * + * Any of these callbacks can be NULL, + * in which case the default pass-through behavior will be used. + * + * The function prototype is defined by #NETWORK_MYSQLD_PLUGIN_PROTO, + * which is used in each plugin to define the callback. + * #NETWORK_MYSQLD_PLUGIN_FUNC can be used to create a function pointer declaration. + */ +typedef struct { + /** + * Called when a new client connection to cetus was created. + */ + NETWORK_MYSQLD_PLUGIN_FUNC(con_init); + /** + * Called when cetus needs to establish a connection to a backend server + * + * Returning a handshake response packet from this callback will + * cause the con_read_handshake step to be skipped. + * + * The next state then is con_send_handshake. + */ + NETWORK_MYSQLD_PLUGIN_FUNC(con_connect_server); + /** + * Called when cetus has read the handshake packet from the server. + */ + NETWORK_MYSQLD_PLUGIN_FUNC(con_read_handshake); + /** + * Called when cetus wants to send the handshake packet to the client. + * + * @note No known plugins actually implement this step right now, + * but rather return a handshake challenge from con_init instead. + */ + NETWORK_MYSQLD_PLUGIN_FUNC(con_send_handshake); + /** + * Called when cetus has read the authentication packet from the client. + */ + NETWORK_MYSQLD_PLUGIN_FUNC(con_read_auth); + /** + * Called when cetus wants to send the authentication packet to the server. + * + * @note No known plugins actually implement this step. + */ + NETWORK_MYSQLD_PLUGIN_FUNC(con_send_auth); + /** + * Called when cetus has read the authentication result + * from the backend server, in response to con_send_auth. + */ + NETWORK_MYSQLD_PLUGIN_FUNC(con_read_auth_result); + /** + * Called when cetus wants to send the authentication response packet to the client. + * + * @note No known plugins implement this callback, but the default + * implementation deals with the important case that + * + * the authentication response used the pre-4.1 password hash method, but the client didn't. + * @see network_mysqld_con::auth_result_state + */ + NETWORK_MYSQLD_PLUGIN_FUNC(con_send_auth_result); + /** + * Called when cetus receives a COM_QUERY packet from a client. + */ + NETWORK_MYSQLD_PLUGIN_FUNC(con_read_query); + NETWORK_MYSQLD_PLUGIN_FUNC(con_get_server_conn_list); + /** + * Called when cetus receives a result set from the server. + */ + NETWORK_MYSQLD_PLUGIN_FUNC(con_read_query_result); + /** + * Called when cetus sends a result set to the client. + * + * The proxy plugin, for example, uses this state to inject more queries + * into the connection, possibly in response to a result set received from a server. + * + * This callback should not cause multiple result sets to be sent to the client. + * @see network_mysqld_con_injection::sent_resultset + */ + NETWORK_MYSQLD_PLUGIN_FUNC(con_send_query_result); + /** + * Called when either side of a connection was either + * closed or some network error occurred. + * + * Usually this is called because a client has disconnected. + * Plugins might want to preserve the server connection in this case + * and reuse it later. In this case the connection state will be ::ST_CLOSE_CLIENT. + * + * When an error on the server connection occurred, this callback is + * usually used to close the client connection as well. + * + * In this case the connection state will be ::ST_CLOSE_SERVER. + * + * @note There are no two separate callback functions for the two possibilities, + * which probably is a deficiency. + */ + NETWORK_MYSQLD_PLUGIN_FUNC(con_cleanup); + + NETWORK_MYSQLD_PLUGIN_FUNC(con_read_auth_old_password); + NETWORK_MYSQLD_PLUGIN_FUNC(con_send_auth_old_password); + + NETWORK_MYSQLD_PLUGIN_FUNC(con_timeout); +} network_mysqld_hooks; + +/** + * A structure containing the parsed packet for a command packet as well + * as the common parts necessary to find the correct packet parsing function. + * + * The correct parsing function is chose by looking at both the current state + * as well as the command in this structure. + * + * @todo Currently the plugins are responsible for setting the first two + * fields of this structure. We have to investigate + * how we can refactor this into a more generic way. + */ +struct network_mysqld_con_parse { + /**< The command indicator from the MySQL Protocol */ + enum enum_server_command command; + + /**< An opaque pointer to a parsed command structure, parsed resultset */ + gpointer data; + + /**< A function pointer to the appropriate "free" function of data */ + void (*data_free)(gpointer); +}; + +/** + * The possible states in the MySQL Protocol. + * + * Not all of the states map directly to plugin callbacks. + * Those states that have no corresponding plugin callbacks are marked as + * + * internal state. + */ +typedef enum { + /**< A new client connection was established */ + ST_INIT = 0, + + /**< A connection to a backend is about to be made */ + ST_CONNECT_SERVER = 1, + + /**< A handshake packet is to be sent to a client */ + ST_SEND_HANDSHAKE = 2, + + /**< An authentication packet is to be read from a client */ + ST_READ_AUTH = 3, + + /**< The result of an authentication attempt is to be sent to a client */ + ST_SEND_AUTH_RESULT = 4, + + /**< COM_QUERY packets are to be read from a client */ + ST_READ_QUERY = 5, + + ST_GET_SERVER_CONNECTION_LIST = 6, + + /**< COM_QUERY packets are to be sent to a server */ + ST_SEND_QUERY = 7, + + /**< Result set packets are to be read from a server */ + ST_READ_QUERY_RESULT = 8, + + ST_READ_M_QUERY_RESULT = 9, + + /**< Result set packets are to be sent to a client */ + ST_SEND_QUERY_RESULT = 10, + + /**< The client connection should be closed */ + ST_CLOSE_CLIENT = 11, + + ST_CLIENT_QUIT = 12, + + /** + * < An unrecoverable error occurred, leads to sending a MySQL ERR packet + * to the client and closing the client connection + */ + ST_SEND_ERROR = 13, + + /** + * < An error occurred (malformed/unexpected packet, + * unrecoverable network error), internal state + */ + ST_ERROR = 14, + + /**< The server connection should be closed */ + ST_CLOSE_SERVER = 15, + +} network_mysqld_con_state_t; + +typedef enum { + /* XA state */ + NEXT_ST_XA_START = 1, + NEXT_ST_XA_QUERY = 2, + NEXT_ST_XA_END = 3, + NEXT_ST_XA_PREPARE = 4, + NEXT_ST_XA_COMMIT = 5, + NEXT_ST_XA_ROLLBACK = 6, + NEXT_ST_XA_CANDIDATE_OVER = 7, + NEXT_ST_XA_OVER = 8 +} network_mysqld_con_dist_tran_state_t; + +typedef enum { + RET_SUCCESS = 10, + RET_ERROR +} retval_t; + + +typedef struct server_connection_state_t server_connection_state_t; + +typedef enum { + ST_ASYNC_CONN, + ST_ASYNC_READ_HANDSHAKE, + ST_ASYNC_SEND_AUTH, + ST_ASYNC_READ_AUTH_RESULT, + ST_ASYNC_ERROR, +} self_con_state_t; + +typedef enum { + /* connection attribute adjuestment */ + ATTR_START = 0, + ATTR_DIF_CHANGE_USER = 1, + ATTR_DIF_DEFAULT_DB = 2, + ATTR_DIF_SQL_MODE = 4, + ATTR_DIF_CHARSET = 8, + ATTR_DIF_SET_OPTION = 16, + ATTR_DIF_SET_AUTOCOMMIT = 32, /* TODO: START TRANSACTION */ +} session_attr_flags_t; + +typedef struct { + gint64 offset; + gint64 row_count; +} LIMIT; + +typedef struct merge_parameters_s { + void *heap; + void *elements; + network_queue *send_queue; + GPtrArray *recv_queues; + GList **candidates; + GString *err_pack; + LIMIT limit; + guint pkt_count; + int pack_err_met; + int is_distinct; + int row_cnter; + int off_pos; + int is_pack_err; + int aggr_output_len; + +} merge_parameters_t; + +struct server_connection_state_t { + + self_con_state_t state; + + struct timeval connect_timeout; + struct timeval read_timeout; + struct timeval write_timeout; + + GString *hashed_pwd; + network_backend_t *backend; + network_socket *server; + chassis *srv; + network_connection_pool *pool; + unsigned int is_multi_stmt_set:1; + guint8 charset_code; +}; + +typedef enum { + ST_PROXY_OK = 0, + ST_PROXY_QUIT = 1 +} proxy_session_state_t; + +typedef enum { + RM_SUCCESS, + RM_FAIL, + RM_CALL_FAIL +} result_merge_status_t; + +typedef struct result_merge_t { + result_merge_status_t status; + GString *detail; +} result_merge_t; + +typedef struct having_condition_t { + int rel_type; + int data_type; + char *condition_value; +} having_condition_t; + +typedef struct mysqld_query_attr_t { + unsigned int sql_mode_set:1; + unsigned int charset_set:1; + unsigned int charset_connection_set:1; + unsigned int charset_results_set:1; + unsigned int charset_client_set:1; + unsigned int charset_reset:1; + unsigned int conn_reserved:1; +} mysqld_query_attr_t; + +typedef struct query_cache_index_item { + gchar *key; + unsigned long long expire_ms; +} query_cache_index_item; + +typedef struct query_cache_item { + network_queue *queue; +} query_cache_item; + +struct query_queue_t; +/** + * get the name of a connection state + */ +NETWORK_API const char *network_mysqld_con_st_name(network_mysqld_con_state_t state); + +/** + * Encapsulates the state and callback functions for a MySQL protocol-based + * connection to and from cetus. + * + * New connection structures are created by the function responsible for + * handling the accept on a listen socket, which + * also is a network_mysqld_con structure, but only has a server set + * - there is no "client" for connections that we listen on. + * + * The chassis itself does not listen on any sockets, this is left to each plugin. + * Plugins are free to create any number of + * connections to listen on, but most of them will only create one and + * reuse the network_mysqld_con_accept function to set up an + * incoming connection. + * + * Each plugin can register callbacks for the various states + * in the MySQL Protocol, these are set in the member plugins. + * A plugin is not required to implement any callbacks at all, + * but only those that it wants to customize. Callbacks that + * are not set, will cause the cetus core to simply forward the received data. + */ +struct network_mysqld_con { + /** + * The current/next state of this connection. + * + * When the protocol state machine performs a transition, + * this variable will contain the next state, + * otherwise, while performing the action at state, + * it will be set to the connection's current state + * in the MySQL protocol. + * + * Plugins may update it in a callback to cause an arbitrary + * state transition, however, this may result + * reaching an invalid state leading to connection errors. + * + * @see network_mysqld_con_handle + */ + network_mysqld_con_state_t state; + + /* + * Save the state before client closed, Then check the state, + * If the state is valid, reuse it. + */ + network_mysqld_con_state_t prev_state; + + network_mysqld_con_dist_tran_state_t dist_tran_state; + + session_attr_flags_t attr_adj_state; + + proxy_session_state_t proxy_state; + + having_condition_t hav_condi; + + /** + * The server side of the connection as it pertains to + * the low-level network implementation. + * The current working server + */ + network_socket *server; /* default = NULL */ + GString *orig_sql; + GString *modified_sql; + + GPtrArray *servers; + + /** + * The client side of the connection as it pertains + * to the low-level network implementation. + */ + network_socket *client; + + /** + * Function pointers to the plugin's callbacks. + * + * Plugins don't need set any of these, but if unset, + * the plugin will not have the opportunity to + * alter the behavior of the corresponding protocol state. + * + * @note In theory you could use functions from different plugins + * to handle the various states, but there is no guarantee that + * this will work. Generally the plugins will assume that + * config is their own chassis_plugin_config (a plugin-private struct) + * and violating this constraint may lead to a crash. + * @see chassis_plugin_config + */ + network_mysqld_hooks plugins; + + /** + * A pointer to a plugin-private struct describing configuration parameters. + * + * @note The actual struct definition used is private to each plugin. + */ + chassis_plugin_config *config; + + /** + * A pointer back to the global, singleton chassis structure. + */ + chassis *srv; /* our srv object */ + + session_attr_flags_t unmatched_attribute; + /** + * A boolean flag indicating that this connection should + * only be used to accept incoming connections. + * + * It does not follow the MySQL protocol by itself + * and its client network_socket will always be NULL. + */ + int retry_serv_cnt; + int max_retry_serv_cnt; + int prepare_stmt_count; + int resp_expected_num; + int last_resp_num; + int num_pending_servers; + int num_servers_visited; + int num_write_pending; + int num_read_pending; + unsigned int key; + + mysqld_query_attr_t query_attr; + + unsigned int is_wait_server:1; /* first connect to backend failed, retrying */ + unsigned int is_calc_found_rows:1; + unsigned int login_failed:1; + unsigned int is_auto_commit:1; + unsigned int is_start_tran_command:1; + unsigned int is_prepared:1; + unsigned int is_in_transaction:1; + unsigned int is_timeout:1; + unsigned int is_in_sess_context:1; + unsigned int is_changed_user_when_quit:1; + unsigned int is_changed_user_failed:1; + unsigned int is_start_trans_buffered:1; + unsigned int is_auto_commit_trans_buffered:1; + unsigned int is_commit_or_rollback:1; + unsigned int is_rollback:1; + unsigned int xa_start_phase:1; + unsigned int use_slave_forced:1; + unsigned int multiple_server_mode:1; + unsigned int could_be_tcp_streamed:1; + unsigned int candidate_tcp_streamed:1; + unsigned int is_new_server_added:1; + unsigned int is_attr_adjust:1; + unsigned int sql_modified:1; + unsigned int dist_tran:1; + unsigned int is_tran_not_distributed_by_comment:1; + unsigned int dist_tran_xa_start_generated:1; + unsigned int dist_tran_failed:1; + unsigned int dist_tran_decided:1; + unsigned int server_to_be_closed:1; + unsigned int server_closed:1; + unsigned int last_warning_met:1; + unsigned int conn_attr_check_omit:1; + unsigned int buffer_and_send_fake_resp:1; + unsigned int delay_send_auto_commit:1; + unsigned int resp_too_long:1; + unsigned int rob_other_conn:1; + unsigned int master_unavailable:1; + unsigned int master_conn_shortaged:1; + unsigned int slave_conn_shortaged:1; + unsigned int auth_next_packet_is_from_server:1; + unsigned int is_xa_query_sent:1; + unsigned int is_read_ro_server_allowed:1; + unsigned int is_read_op_for_cached:1; + unsigned int xa_query_status_error_and_abort:1; + unsigned int use_all_prev_servers:1; + unsigned int partially_merged:1; + unsigned int query_cache_judged:1; + unsigned int is_client_compressed:1; + unsigned int last_backend_type:2; + unsigned int all_participate_num:8; + + unsigned long long xa_id; + + time_t last_check_conn_supplement_time; + + struct timeval req_recv_time; + struct timeval resp_recv_time; + struct timeval resp_send_time; + + guint64 resp_cnt; + guint64 last_insert_id; + + /** + * An integer indicating the result received from a server + * after sending an authentication request. + * + * This is used to differentiate between the old, pre-4.1 + * authentication and the new, 4.1+ one based on the response. + */ + guint8 auth_result_state; + + /* track the auth-method-switch state */ + GString *auth_switch_to_method; + GString *auth_switch_to_data; + guint32 auth_switch_to_round; + + /** Flag indicating if we the plugin doesn't need the resultset itself. + * + * If set to TRUE, the plugin needs to see + * the entire resultset and we will buffer it. + * If set to FALSE, the plugin is not interested + * in the content of the resultset and we'll + * try to forward the packets to the client directly, + * even before the full resultset is parsed. + */ + gboolean resultset_is_needed; + /** + * Flag indicating whether we have seen all parts belonging to one resultset. + */ + gboolean resultset_is_finished; + + /** + * Flag indicating that we have received a COM_QUIT command. + * + * This is mainly used to differentiate between the case + * where the server closed the connection because of some error + * or if the client asked it to close its side of the connection. + * cetus would report spurious errors for the latter case, + * if we failed to track this command. + */ + gboolean com_quit_seen; + + /** + * Contains the parsed packet. + */ + struct network_mysqld_con_parse parse; + + enum enum_server_command cur_command; + + /** + * An opaque pointer to a structure describing extra + * connection state needed by the plugin. + * + * The content and meaning is completely up to each plugin and + * the chassis will not access this in any way. + * + * in proxy-plugin, proxy_plugin_con_t + * in shard-plugin, shard_plugin_con_t + * in admin-plugin, not used + */ + void *plugin_con_state; + /* connection specific timeouts */ + struct timeval connect_timeout;/* default = 2 s */ + struct timeval read_timeout; /* default = 10 min */ + struct timeval write_timeout; /* default = 10 min */ + struct timeval wait_clt_next_sql; + char xid_str[XID_LEN]; + char last_backends_type[MAX_SERVER_NUM]; + + struct sharding_plan_t *sharding_plan; + struct query_queue_t *recent_queries; + void *data; +}; + + +struct network_mysqld_con_injection { + network_injection_queue *queries; + int sent_resultset; +}; + +typedef enum { + NET_RW_STATE_NONE, + NET_RW_STATE_WRITE, + NET_RW_STATE_READ, + NET_RW_STATE_ERROR, + NET_RW_STATE_PART_FINISHED, + NET_RW_STATE_FINISHED +} read_write_state; + + +/** + * A server side session, wraps server side socket + * For sharding plugin: + * proxy-session = 1 * client-socket + n *server-session + */ +typedef struct server_session_t +{ + unsigned int fresh:1; + unsigned int participated:1; + unsigned int xa_start_already_sent:1; + unsigned int dist_tran_participated:1; + unsigned int xa_query_status_error_and_abort:1; + unsigned int is_in_xa:1; + unsigned int is_xa_over:1; + unsigned int attr_consistent:1; + unsigned int attr_consistent_checked:1; + unsigned int attr_adjusted_now:1; + unsigned int read_cal_flag:1; + unsigned int index:6; + + network_socket *server; + const GString *sql; + network_mysqld_con *con; + network_backend_t *backend; + network_mysqld_con_dist_tran_state_t dist_tran_state; + read_write_state state; + session_attr_flags_t attr_diff; +} server_session_t; + + +typedef struct { + /**< A list of queries to send to the backend.*/ + struct network_mysqld_con_injection injected; + + network_backend_t *backend; + /**< index into the backend-array, start from 0 */ + int backend_ndx; + + short *backend_ndx_array; /* rw-only: map backend index to connections in con->servers */ + + struct sql_context_t *sql_context; + int trx_read_write; /* default TF_READ_WRITE */ + int trx_isolation_level; /* default TF_REPEATABLE_READ */ + +} proxy_plugin_con_t; + +NETWORK_API network_mysqld_con *network_mysqld_con_new(void); +NETWORK_API void network_mysqld_con_free(network_mysqld_con *con); + +NETWORK_API void network_mysqld_con_accept(int event_fd, short events, void *user_data); + +NETWORK_API int network_mysqld_con_send_ok(network_socket *con); +NETWORK_API int network_mysqld_con_send_ok_full(network_socket *con, guint64 affected_rows, + guint64 insert_id, guint16 server_status, guint16 warnings); +NETWORK_API int network_mysqld_con_send_error(network_socket *con, const gchar *errmsg, gsize errmsg_len); +NETWORK_API int network_mysqld_con_send_error_full(network_socket *con, const char *errmsg, + gsize errmsg_len, guint errorcode, const gchar *sqlstate); +NETWORK_API int network_mysqld_con_send_error_pre41(network_socket *con, const gchar *errmsg, + gsize errmsg_len); +NETWORK_API int network_mysqld_con_send_error_full_pre41(network_socket *con, const char *errmsg, + gsize errmsg_len, guint errorcode); +NETWORK_API int network_mysqld_con_send_resultset(network_socket *con, GPtrArray *fields, GPtrArray *rows); +int network_mysqld_con_send_current_date(network_socket *, const char *); +int network_mysqld_con_send_cetus_version(network_socket *); +void network_mysqld_send_xa_start(network_socket *, const char *xid); + +NETWORK_API void network_mysqld_con_reset_command_response_state(network_mysqld_con *con); +NETWORK_API void network_mysqld_con_reset_query_state(network_mysqld_con *con); + +/** + * set groups, delete if already exists + */ +void network_mysqld_con_set_sharding_plan(network_mysqld_con *con, struct sharding_plan_t *); + +/** + * should be socket + */ +NETWORK_API network_socket_retval_t network_mysqld_read(chassis *srv, network_socket *con); +NETWORK_API network_socket_retval_t network_mysqld_write(chassis *srv, network_socket *con); +NETWORK_API network_socket_retval_t network_mysqld_con_get_packet(chassis G_GNUC_UNUSED *chas, + network_socket *con); + +struct chassis_private { + GPtrArray *cons; /**< array(network_mysqld_con) */ + GList *listen_conns; + network_backends_t *backends; + struct cetus_users_t *users; + struct cetus_variable_t *stats_variables; + struct cetus_monitor_t *monitor; +}; + +NETWORK_API network_socket_retval_t +network_mysqld_read_mul_packets(chassis G_GNUC_UNUSED *chas, network_mysqld_con *con, + network_socket *server, int *is_finished); + +NETWORK_API void send_part_content_to_client(network_mysqld_con *con); +NETWORK_API void set_conn_attr(network_mysqld_con *con, network_socket *server); +NETWORK_API int network_mysqld_init(chassis *srv); +NETWORK_API void network_mysqld_add_connection(chassis *srv, network_mysqld_con *con, gboolean listen); +NETWORK_API void network_mysqld_con_handle(int event_fd, short events, void *user_data); +NETWORK_API int network_mysqld_queue_append(network_socket *sock, network_queue *queue, + const char *data, size_t len); +NETWORK_API int network_mysqld_queue_append_raw(network_socket *sock, network_queue *queue, GString *data); +NETWORK_API int network_mysqld_queue_reset(network_socket *sock); + +NETWORK_API void network_connection_pool_create_conn(network_mysqld_con *con); +NETWORK_API void network_connection_pool_create_conns(chassis *srv); + +NETWORK_API void record_xa_log_for_mending(network_mysqld_con *con, network_socket *sock); +NETWORK_API gboolean shard_set_autocommit(network_mysqld_con *con); +NETWORK_API gboolean shard_set_charset_consistant(network_mysqld_con *con); +NETWORK_API gboolean shard_set_default_db_consistant(network_mysqld_con *con); +NETWORK_API gboolean shard_set_multi_stmt_consistant(network_mysqld_con *con); +NETWORK_API void shard_build_xa_query(network_mysqld_con *con, server_session_t *pmd); + +#endif diff --git a/src/network-queue.c b/src/network-queue.c new file mode 100644 index 0000000..afe70c0 --- /dev/null +++ b/src/network-queue.c @@ -0,0 +1,160 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "glib-ext.h" +#include "network-queue.h" +#include "network-mysqld-proto.h" + +network_queue *network_queue_new() { + network_queue *queue; + + queue = g_new0(network_queue, 1); + + queue->chunks = g_queue_new(); + + return queue; +} + +void network_queue_free(network_queue *queue) { + GString *packet; + + if (!queue) return; + + while ((packet = g_queue_pop_head(queue->chunks))) { + g_string_free(packet, TRUE); + } + + g_queue_free(queue->chunks); + + g_free(queue); +} + +void network_queue_clear(network_queue *queue) +{ + if (!queue) return; + GString *packet; + while ((packet = g_queue_pop_head(queue->chunks)) != NULL) { + g_string_free(packet, TRUE); + } + queue->len = queue->offset = 0; +} + +int network_queue_append(network_queue *queue, GString *s) { + if (s->len == 0) { + g_warning("%s: packet len:%d", G_STRLOC, (int) s->len); + } + + if (s->len > PACKET_LEN_MAX) { + g_warning("%s: packet len:%d is too long", G_STRLOC, (int) s->len); + } + + queue->len += s->len; + + g_queue_push_tail(queue->chunks, s); + + return 0; +} + +/** + * get a string from the head of the queue and leave the queue unchanged + * + * @param queue the queue to read from + * @param peek_len bytes to collect + * @param dest GString to write it. If NULL, we allow a new one and return it + * @return NULL if not enough data + * if dest is not NULL, dest, otherwise a new GString containing the data + */ +GString *network_queue_peek_str(network_queue *queue, gsize peek_len, GString *dest) { + gsize we_want = peek_len; + GList *node; + + + /* + * TODO: convert to DTrace probe + * g_debug("[%s] looking for %d bytes, queue has %d", G_STRLOC, peek_len, queue->len); + */ + if (queue->len < peek_len) { + return NULL; + } + + if (!dest) { + /* no define */ + dest = g_string_sized_new(peek_len); + } + + g_assert_cmpint(dest->allocated_len, >, peek_len); + + for (node = queue->chunks->head; node && we_want; node = node->next) { + GString *chunk = node->data; + + if (node == queue->chunks->head) { + gsize we_have = we_want < (chunk->len - queue->offset) ? + we_want : (chunk->len - queue->offset); + + g_string_append_len(dest, chunk->str + queue->offset, we_have); + + we_want -= we_have; + } else { + gsize we_have = we_want < chunk->len ? we_want : chunk->len; + + g_string_append_len(dest, chunk->str, we_have); + + we_want -= we_have; + } + } + + return dest; +} + +/** + * get a string from the head of the queue and remove the chunks from the queue + */ +GString * +network_queue_pop_str(network_queue *queue, gsize steal_len, GString *dest) +{ + gsize we_want = steal_len; + GString *chunk; + + if (queue->len < steal_len) { + return NULL; + } + + while ((chunk = g_queue_peek_head(queue->chunks))) { + gsize we_have = we_want < (chunk->len - queue->offset) ? + we_want : (chunk->len - queue->offset); + + if (!dest && (queue->offset == 0) && (chunk->len == steal_len)) { + /* optimize the common case that we want to have to full chunk + * + * if dest is null, we can remove the GString from the queue and + * return it directly without copying it + */ + dest = g_queue_pop_head(queue->chunks); + queue->len -= we_have; + return dest; + } + + if (!dest) { + /* if we don't have a dest-buffer yet, create one */ + dest = g_string_sized_new(steal_len); + } + g_string_append_len(dest, chunk->str + queue->offset, we_have); + + queue->offset += we_have; + queue->len -= we_have; + we_want -= we_have; + + if (chunk->len == queue->offset) { + /* the chunk is done, remove it */ + g_string_free(g_queue_pop_head(queue->chunks), TRUE); + queue->offset = 0; + } else { + break; + } + } + + return dest; +} + + diff --git a/src/network-queue.h b/src/network-queue.h new file mode 100644 index 0000000..d175173 --- /dev/null +++ b/src/network-queue.h @@ -0,0 +1,23 @@ +#ifndef _NETWORK_QUEUE_H_ +#define _NETWORK_QUEUE_H_ + +#include "network-exports.h" + +#include + +/* a input or output stream */ +typedef struct { + GQueue *chunks; + + size_t len; /* len in all chunks (w/o the offset) */ + size_t offset; /* offset in the first chunk */ +} network_queue; + +NETWORK_API network_queue *network_queue_new(void); +NETWORK_API void network_queue_free(network_queue *queue); +void network_queue_clear(network_queue *queue); +NETWORK_API int network_queue_append(network_queue *queue, GString *chunk); +NETWORK_API GString *network_queue_pop_str(network_queue *queue, gsize steal_len, GString *dest); +NETWORK_API GString *network_queue_peek_str(network_queue *queue, gsize peek_len, GString *dest); + +#endif diff --git a/src/network-socket.c b/src/network-socket.c new file mode 100644 index 0000000..5c60325 --- /dev/null +++ b/src/network-socket.c @@ -0,0 +1,1028 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include /* writev */ +#ifdef HAVE_SYS_TYPES_H +#include +#endif + +#ifdef HAVE_SYS_FILIO_H +#include +#endif + + +#include /** inet_ntoa */ +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +#ifdef HAVE_WRITEV +#define USE_BUFFERED_NETIO +#else +#undef USE_BUFFERED_NETIO +#endif + +#define E_NET_CONNRESET ECONNRESET +#define E_NET_CONNABORTED ECONNABORTED +#define E_NET_INPROGRESS EINPROGRESS +#if EWOULDBLOCK == EAGAIN +/** + * some system make EAGAIN == EWOULDBLOCK which would lead to a + * error in the case handling + * + * set it to -1 as this error should never happen + */ +#define E_NET_WOULDBLOCK -1 +#else +#define E_NET_WOULDBLOCK EWOULDBLOCK +#endif + +#define MAX_QUERY_CACHE_SIZE 65536 +#include "network-socket.h" +#include "network-mysqld-proto.h" +#include "network-mysqld-packet.h" +#include "cetus-util.h" +#include "network-compress.h" +#include "glib-ext.h" + +network_socket *network_socket_new() { + network_socket *s; + + s = g_new0(network_socket, 1); + + s->send_queue = network_queue_new(); + s->recv_queue = network_queue_new(); + s->recv_queue_raw = network_queue_new(); + s->recv_queue_uncompress_raw = network_queue_new(); + + s->default_db = g_string_new(NULL); + s->username = g_string_new(NULL); + s->charset = g_string_new(NULL); + s->charset_client = g_string_new(NULL); + s->charset_connection = g_string_new(NULL); + s->charset_results = g_string_new(NULL); + s->sql_mode = g_string_new(NULL); + + s->fd = -1; + s->socket_type = SOCK_STREAM; /* let's default to TCP */ + s->packet_id_is_reset = TRUE; + + s->src = network_address_new(); + s->dst = network_address_new(); + s->last_visit_time = time(0); + + return s; +} + +void network_socket_free(network_socket *s) { + if (!s) return; + + g_debug("%s: network_socket_free:%p", G_STRLOC, s); + + if (s->last_compressed_packet) { + g_string_free(s->last_compressed_packet, TRUE); + s->last_compressed_packet = NULL; + } + network_queue_free(s->send_queue); + network_queue_free(s->recv_queue); + network_queue_free(s->recv_queue_raw); + network_queue_free(s->recv_queue_uncompress_raw); + + if (s->response) network_mysqld_auth_response_free(s->response); + if (s->challenge) network_mysqld_auth_challenge_free(s->challenge); + + network_address_free(s->dst); + network_address_free(s->src); + + if (s->event.ev_base) { /* if .ev_base isn't set, the event never got added */ + g_debug("%s:event del, ev:%p",G_STRLOC, &(s->event)); + event_del(&(s->event)); + } + + if (s->fd != -1) { + closesocket(s->fd); + } + + g_string_free(s->default_db, TRUE); + g_string_free(s->charset_client, TRUE); + g_string_free(s->charset_connection, TRUE); + g_string_free(s->charset, TRUE); + g_string_free(s->charset_results, TRUE); + g_string_free(s->username, TRUE); + g_string_free(s->sql_mode, TRUE); + + g_free(s); +} + +/** + * portable 'set non-blocking io' + * + * @param sock a socket + * @return NETWORK_SOCKET_SUCCESS on success, NETWORK_SOCKET_ERROR on error + */ +network_socket_retval_t network_socket_set_non_blocking(network_socket *sock) { + int ret; + ret = fcntl(sock->fd, F_SETFL, O_NONBLOCK | O_RDWR); + if (ret != 0) { + g_critical("%s: set_non_blocking() failed: %s (%d)", + G_STRLOC, + g_strerror(errno), errno); + return NETWORK_SOCKET_ERROR; + } + return NETWORK_SOCKET_SUCCESS; +} + +network_socket_retval_t network_socket_set_send_buffer_size(network_socket *sock, int size) { + + if (setsockopt(sock->fd, SOL_SOCKET, SO_SNDBUF, &size, sizeof(size)) != 0) { + g_critical("%s: setsockopt SO_SNDBUF failed: %s (%d)", + G_STRLOC, + g_strerror(errno), errno); + return NETWORK_SOCKET_ERROR; + } + + return NETWORK_SOCKET_SUCCESS; +} + +/** + * accept a connection + * + * event handler for listening connections + * + * @param srv a listening socket + * + */ +network_socket *network_socket_accept(network_socket *srv, int *reason) { + network_socket *client; + + g_return_val_if_fail(srv, NULL); + /* accept() only works on stream sockets */ + g_return_val_if_fail(srv->socket_type == SOCK_STREAM, NULL); + + client = network_socket_new(); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28) + g_debug("%s: call accept4", G_STRLOC); + if (-1 == (client->fd = accept4(srv->fd, &client->src->addr.common, + &(client->src->len), SOCK_NONBLOCK))) + { + *reason = errno; + network_socket_free(client); + return NULL; + } +#else + if (-1 == (client->fd = accept(srv->fd, &client->src->addr.common, &(client->src->len)))) { + *reason = errno; + network_socket_free(client); + return NULL; + } + network_socket_set_non_blocking(client); +#endif + + if (network_address_refresh_name(client->src)) { + network_socket_free(client); + return NULL; + } + + /* + * the listening side may be INADDR_ANY, + * let's get which address the client really connected to + */ + if (-1 == getsockname(client->fd, &client->dst->addr.common, + &(client->dst->len))) + { + network_address_reset(client->dst); + } else if (network_address_refresh_name(client->dst)) { + network_address_reset(client->dst); + } + + return client; +} + +static network_socket_retval_t +network_socket_connect_setopts(network_socket *sock) +{ + int val; + /** + * set the same options as the mysql client + */ +#ifdef IP_TOS + val = 8; + if (setsockopt(sock->fd, IPPROTO_IP, IP_TOS, &val, sizeof(val)) != 0) { + g_critical("%s: setsockopt IP_TOS failed: %s (%d)", + G_STRLOC, g_strerror(errno), errno); + } +#endif + val = 1; + if (setsockopt(sock->fd, IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val)) != 0) { + g_critical("%s: setsockopt TCP_NODELAY failed: %s (%d)", + G_STRLOC, g_strerror(errno), errno); + } + val = 1; + if (setsockopt(sock->fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) != 0) { + g_critical("%s: setsockopt SO_KEEPALIVE failed: %s (%d)", + G_STRLOC, g_strerror(errno), errno); + } + + /* + * the listening side may be INADDR_ANY, + * let's get which address the client really connected to + */ + if (-1 == getsockname(sock->fd, &sock->src->addr.common, &(sock->src->len))) { + g_debug("%s: getsockname() failed: %s (%d)", + G_STRLOC, + g_strerror(errno), + errno); + network_address_reset(sock->src); + } else if (network_address_refresh_name(sock->src)) { + g_debug("%s: network_address_refresh_name() failed", + G_STRLOC); + network_address_reset(sock->src); + } + + return NETWORK_SOCKET_SUCCESS; +} + +/** + * finish the non-blocking connect() + * + * sets 'errno' as if connect() would have failed + * + */ +network_socket_retval_t network_socket_connect_finish(network_socket *sock) { + int so_error = 0; + network_socklen_t so_error_len = sizeof(so_error); + + /** + * we might get called a 2nd time after a connect() == EINPROGRESS + */ + if (getsockopt(sock->fd, SOL_SOCKET, SO_ERROR, &so_error, &so_error_len)) { + /* getsockopt failed */ + g_critical("%s: getsockopt(%s) failed: %s (%d)", + G_STRLOC, + sock->dst->name->str, g_strerror(errno), errno); + return NETWORK_SOCKET_ERROR; + } + + switch (so_error) { + case 0: + network_socket_connect_setopts(sock); + + return NETWORK_SOCKET_SUCCESS; + default: + errno = so_error; + + return NETWORK_SOCKET_ERROR_RETRY; + } +} + +/** + * connect a socket + * + * the sock->addr has to be set before + * + * @param sock a socket + * @return NETWORK_SOCKET_SUCCESS on connected, + * NETWORK_SOCKET_ERROR on error, + * NETWORK_SOCKET_ERROR_RETRY for try again + * @see network_address_set_address() + */ +network_socket_retval_t network_socket_connect(network_socket *sock) { + /* our _new() allocated it already */ + g_return_val_if_fail(sock->dst, NETWORK_SOCKET_ERROR); + /* we want to use the ->name in the error-msgs */ + g_return_val_if_fail(sock->dst->name->len, NETWORK_SOCKET_ERROR); + /* we already have a valid fd, we don't want to leak it */ + g_return_val_if_fail(sock->fd < 0, NETWORK_SOCKET_ERROR); + g_return_val_if_fail(sock->socket_type == SOCK_STREAM, NETWORK_SOCKET_ERROR); + + /** + * create a socket for the requested address + * + * if the dst->addr isn't set yet, socket() will fail with unsupported type + */ + if (-1 == (sock->fd = socket(sock->dst->addr.common.sa_family, + sock->socket_type, 0))) + { + g_critical("%s: socket(%s) failed: %s (%d)", + G_STRLOC, + sock->dst->name->str, g_strerror(errno), errno); + return NETWORK_SOCKET_ERROR; + } + + /** + * make the connect() call non-blocking + * + */ + network_socket_set_non_blocking(sock); + + if (-1 == connect(sock->fd, &sock->dst->addr.common, sock->dst->len)) { + /** + * in most TCP cases we connect() will return with + * EINPROGRESS ... 3-way handshake + */ + switch (errno) { + case E_NET_INPROGRESS: + case E_NET_WOULDBLOCK: + return NETWORK_SOCKET_ERROR_RETRY; + default: + g_critical("%s: connect(%s) failed: %s (%d)", + G_STRLOC, + sock->dst->name->str, + g_strerror(errno), errno); + return NETWORK_SOCKET_ERROR; + } + } + + network_socket_connect_setopts(sock); + + return NETWORK_SOCKET_SUCCESS; +} + +/** + * connect a socket + * + * the con->dst->addr has to be set before + * + * @param con a socket + * @return NETWORK_SOCKET_SUCCESS on connected, NETWORK_SOCKET_ERROR on error + * + * @see network_address_set_address() + */ +network_socket_retval_t network_socket_bind(network_socket *con) { + /* + * HPUX: int setsockopt(int s, int level, int optname, + * const void *optval, int optlen); + * all others: int setsockopt(int s, int level, int optname, + * const void *optval, socklen_t optlen); + */ +#define SETSOCKOPT_OPTVAL_CAST (void *) + + /* socket is already bound */ + g_return_val_if_fail(con->fd < 0, NETWORK_SOCKET_ERROR); + g_return_val_if_fail((con->socket_type == SOCK_DGRAM) || + (con->socket_type == SOCK_STREAM), NETWORK_SOCKET_ERROR); + + if (con->socket_type == SOCK_STREAM) { + g_return_val_if_fail(con->dst, NETWORK_SOCKET_ERROR); + g_return_val_if_fail(con->dst->name->len > 0, NETWORK_SOCKET_ERROR); + + if (-1 == (con->fd = socket(con->dst->addr.common.sa_family, + con->socket_type, 0))) + { + g_critical("%s: socket(%s) failed: %s (%d)", + G_STRLOC, + con->dst->name->str, + g_strerror(errno), errno); + return NETWORK_SOCKET_ERROR; + } + + if (con->dst->addr.common.sa_family == AF_INET || + con->dst->addr.common.sa_family == AF_INET6) { + int val; + + val = 1; + if (0 != setsockopt(con->fd, IPPROTO_TCP, TCP_NODELAY, + SETSOCKOPT_OPTVAL_CAST &val, sizeof(val))) + { + g_critical("%s: setsockopt(%s, IPPROTO_TCP, TCP_NODELAY) failed: %s (%d)", + G_STRLOC, + con->dst->name->str, + g_strerror(errno), errno); + return NETWORK_SOCKET_ERROR; + } + + if (0 != setsockopt(con->fd, SOL_SOCKET, SO_REUSEADDR, + SETSOCKOPT_OPTVAL_CAST &val, sizeof(val))) + { + g_critical("%s: setsockopt(%s, SOL_SOCKET, SO_REUSEADDR) failed: %s (%d)", + G_STRLOC, + con->dst->name->str, + g_strerror(errno), errno); + return NETWORK_SOCKET_ERROR; + } + } + + if (con->dst->addr.common.sa_family == AF_INET6) { +#ifdef IPV6_V6ONLY + /* disable dual-stack IPv4-over-IPv6 sockets + * + * ... if it is supported: + * - Linux + * - Mac OS X + * - FreeBSD + * - Solaris 10 and later + * + * no supported on: + * - Solaris 9 and earlier + */ + + /* IPV6_V6ONLY is int on unix */ + int val; + + val = 0; + if (0 != setsockopt(con->fd, IPPROTO_IPV6, IPV6_V6ONLY, + SETSOCKOPT_OPTVAL_CAST &val, sizeof(val))) + { + g_critical("%s: setsockopt(%s, IPPROTO_IPV6, IPV6_V6ONLY) failed: %s (%d)", + G_STRLOC, + con->dst->name->str, + g_strerror(errno), errno); + return NETWORK_SOCKET_ERROR; + } +#endif + } + + if (-1 == bind(con->fd, &con->dst->addr.common, con->dst->len)) { + /* binding failed so the address/socket is already being used + * let's check if we can connect to it so we check if is being used + * by some app + */ + if (-1 == connect(con->fd, &con->dst->addr.common, con->dst->len)) { + g_debug("%s: connect(%s) failed: %s (%d)", + G_STRLOC, + con->dst->name->str, + g_strerror(errno), errno); + /* we can't connect to the socket so no one is listening on it. We need + * to unlink it (delete the name from the file system) to be able to + * re-use it. + * network_address_free does the unlink, but to re-use it we need + * to store the pathname associated with the socket before unlink it and + * create a new socket with it. + */ + gchar *address_copy = g_strdup(con->dst->name->str); + con->dst->can_unlink_socket = TRUE; + network_address_free(con->dst); + + con->dst = network_address_new(); + + if (network_address_set_address(con->dst, address_copy) == -1) { + g_free(address_copy); + return NETWORK_SOCKET_ERROR; + } + + /* we can now free the address copy */ + g_free(address_copy); + + g_debug("%s: retrying to bind(%s)", + G_STRLOC, + con->dst->name->str); + + /* let's bind again with the new socket */ + if (-1 == bind(con->fd, &con->dst->addr.common, con->dst->len)) { + g_critical("%s: bind(%s) failed: %s (%d)", + G_STRLOC, + con->dst->name->str, + g_strerror(errno), errno); + + return NETWORK_SOCKET_ERROR; + } + } else { + g_critical("%s: bind(%s) failed: %s (%d)", + G_STRLOC, + con->dst->name->str, + g_strerror(errno), errno); + + return NETWORK_SOCKET_ERROR; + } + } + + if (con->dst->addr.common.sa_family == AF_INET && + con->dst->addr.ipv4.sin_port == 0) { + struct sockaddr_in a; + socklen_t a_len = sizeof(a); + + if (0 != getsockname(con->fd, (struct sockaddr *)&a, &a_len)) { + g_critical("%s: getsockname(%s) failed: %s (%d)", + G_STRLOC, + con->dst->name->str, + g_strerror(errno), errno); + return NETWORK_SOCKET_ERROR; + } + con->dst->addr.ipv4.sin_port = a.sin_port; + } else if (con->dst->addr.common.sa_family == AF_INET6 && + con->dst->addr.ipv6.sin6_port == 0) { + struct sockaddr_in6 a; + socklen_t a_len = sizeof(a); + + if (0 != getsockname(con->fd, (struct sockaddr *)&a, &a_len)) { + g_critical("%s: getsockname(%s) failed: %s (%d)", + G_STRLOC, + con->dst->name->str, + g_strerror(errno), errno); + return NETWORK_SOCKET_ERROR; + } + con->dst->addr.ipv6.sin6_port = a.sin6_port; + } + + if (-1 == listen(con->fd, 128)) { + g_critical("%s: listen(%s, 128) failed: %s (%d)", + G_STRLOC, + con->dst->name->str, + g_strerror(errno), errno); + return NETWORK_SOCKET_ERROR; + } + } else { + /* UDP sockets bind the ->src address */ + g_return_val_if_fail(con->src, NETWORK_SOCKET_ERROR); + g_return_val_if_fail(con->src->name->len > 0, NETWORK_SOCKET_ERROR); + + if (-1 == (con->fd = socket(con->src->addr.common.sa_family, + con->socket_type, 0))) + { + g_critical("%s: socket(%s) failed: %s (%d)", + G_STRLOC, + con->src->name->str, + g_strerror(errno), errno); + return NETWORK_SOCKET_ERROR; + } + + if (-1 == bind(con->fd, &con->src->addr.common, con->src->len)) { + g_critical("%s: bind(%s) failed: %s (%d)", + G_STRLOC, + con->src->name->str, + g_strerror(errno), errno); + return NETWORK_SOCKET_ERROR; + } + } + + con->dst->can_unlink_socket = TRUE; + return NETWORK_SOCKET_SUCCESS; +} + +/** + * read a data from the socket + * + * @param sock the socket + */ +network_socket_retval_t network_socket_read(network_socket *sock) { + gssize len; + + if (sock->to_read > 0) { + GString *packet = g_string_sized_new(sock->to_read); + + g_queue_push_tail(sock->recv_queue_raw->chunks, packet); + + g_debug("%s: recv queue length:%d, sock:%p, client addr:%s, to read:%d", + G_STRLOC, sock->recv_queue_raw->chunks->length, + sock, sock->src->name->str, (int) sock->to_read); + + g_debug("%s: tcp read:%d for fd:%d", G_STRLOC, (int) sock->to_read, sock->fd); + len = recv(sock->fd, packet->str, sock->to_read, 0); + + if (-1 == len) { + switch (errno) { + case E_NET_CONNABORTED: + /** nothing to read, let's let ioctl() handle the close for us */ + case E_NET_CONNRESET: + case E_NET_WOULDBLOCK: /** the buffers are empty, try again later */ + case EAGAIN: + g_message("%s:server to read:%d, but empty", + G_STRLOC, (int) sock->to_read); + return NETWORK_SOCKET_WAIT_FOR_EVENT; + default: + g_message("%s: recv() failed: %s (errno=%d), to read:%d", G_STRLOC, + g_strerror(errno), errno, (int) sock->to_read); + return NETWORK_SOCKET_ERROR; + } + } else if (len == 0) { + /** + * connection close + * + * let's call the ioctl() and let it handle it for use + */ + return NETWORK_SOCKET_WAIT_FOR_EVENT; + } + + sock->to_read -= len; + sock->recv_queue_raw->len += len; + packet->len = len; + } + + return NETWORK_SOCKET_SUCCESS; +} + +static network_socket_retval_t +network_socket_compressed_write(network_socket *con, int send_chunks) +{ + gssize len; + int os_errno; + + if (con->last_compressed_packet) { + char *str = con->last_compressed_packet->str + con->compressed_unsend_offset; + int unsend_len = con->last_compressed_packet->len - con->compressed_unsend_offset; + + len = send(con->fd, str, unsend_len, 0); + + g_debug("%s: tcp write, comp:%d, write len:%d", + G_STRLOC, (int) unsend_len, (int) len); + + os_errno = errno; + + if (len > 0) { + if (len != unsend_len) { + g_debug("%s: tcp write len not equal to compressed packet, comp:%d, len:%d", + G_STRLOC, unsend_len, (int) len); + con->compressed_unsend_offset = len + con->compressed_unsend_offset; + return NETWORK_SOCKET_WAIT_FOR_EVENT; + } else { + g_string_free(con->last_compressed_packet, TRUE); + con->last_compressed_packet = NULL; + con->compressed_unsend_offset = 0; + } + } + + if (-1 == len) { + switch (os_errno) { + case E_NET_WOULDBLOCK: + case EAGAIN: + return NETWORK_SOCKET_WAIT_FOR_EVENT; + case EPIPE: + case E_NET_CONNRESET: + case E_NET_CONNABORTED: + /** remote side closed the connection */ + return NETWORK_SOCKET_ERROR; + default: + g_message("%s: send(%s, ...) failed: %s", + G_STRLOC, + con->dst->name->str, + g_strerror(errno)); + return NETWORK_SOCKET_ERROR; + } + } else if (len == 0) { + return NETWORK_SOCKET_ERROR; + } + } + + if (send_chunks == 0) return NETWORK_SOCKET_SUCCESS; + + gint chunk_count; + chunk_count = send_chunks > 0 ? send_chunks : (gint) con->send_queue->chunks->length; + + if (chunk_count == 0) return NETWORK_SOCKET_SUCCESS; + + g_assert_cmpint(chunk_count, >, 0); /* make sure it is never negative */ + + z_stream strm; + cetus_compress_init(&strm); + + GString *compress_packet = g_string_sized_new(16384); + + network_mysqld_proto_append_packet_len(compress_packet, 0); + network_mysqld_proto_append_packet_id(compress_packet, con->compressed_packet_id); + con->compressed_packet_id++; + network_mysqld_proto_append_packet_len(compress_packet, 0); + + int uncompressed_len = 0, is_too_large = 0; + + GList *chunk; + gint chunk_id; + + int need_more_write = 0; + + for (chunk = con->send_queue->chunks->head, chunk_id = 0; + chunk && chunk_id < chunk_count; chunk_id++, chunk = chunk->next) + { + GString *s = chunk->data; + + int end = 0; + + if (!chunk->next || (chunk_id + 1 >= chunk_count)) { + end = 1; + } + + char *str; + int str_len; + if (chunk_id == 0) { + str = s->str + con->send_queue->offset; + str_len = s->len - con->send_queue->offset; + } else { + str = s->str; + str_len = s->len; + } + + g_debug("%s: offset:%d, str_len:%d, s->len:%d", G_STRLOC, + (int) con->send_queue->offset, str_len, (int) s->len); + + if (!con->do_strict_compress && uncompressed_len < MIN_COMPRESS_LENGTH && end) { + g_string_append_len(compress_packet, str, str_len); + if (con->send_queue->offset != 0) { + g_warning("%s:con->send_queue->offset is not zero here", G_STRLOC); + } + } else { + + con->do_strict_compress = 1; + uncompressed_len += str_len; + if (uncompressed_len > PACKET_LEN_MAX) { + g_message("%s:too large packet:%d, s->len:%d", G_STRLOC, + uncompressed_len, (int) str_len); + uncompressed_len -= str_len; + str_len = PACKET_LEN_MAX - uncompressed_len; + if (str_len == 0) { + cetus_compress(&strm, compress_packet, NULL, 0, 1); + con->send_queue->offset = 0; + } else { + uncompressed_len = PACKET_LEN_MAX; + cetus_compress(&strm, compress_packet, str, str_len, 1); + con->send_queue->offset += str_len; + } + is_too_large = 1; + } else { + cetus_compress(&strm, compress_packet, str, str_len, end); + con->send_queue->offset = 0; + } + } + + if (end || is_too_large) { + cetus_compress_end(&strm); + } + + con->total_output += str_len; + + if (is_too_large) { + need_more_write = 1; + break; + } + } + + int compressed_len = compress_packet->len - NET_HEADER_SIZE - COMP_HEADER_SIZE; + + if (compressed_len > PACKET_LEN_MAX || uncompressed_len > PACKET_LEN_MAX + || (is_too_large && uncompressed_len == 0)) + { + g_warning("%s:too large for compression:%d, compressed len:%d", + G_STRLOC, uncompressed_len, compressed_len); + g_string_free(compress_packet, TRUE); + return NETWORK_SOCKET_ERROR; + } + + int i; + char *s = compress_packet->str; + for (i = 0; i < 3; i++) { + s[i] = compressed_len & 0xff; + compressed_len >>= 8; + } + + s = s + 4; + for (i = 0; i < 3; i++) { + s[i] = uncompressed_len & 0xff; + uncompressed_len >>= 8; + } + + i = 0; + for (chunk = con->send_queue->chunks->head; chunk; ) { + if (i >= chunk_id) { + break; + } + GString *s = chunk->data; + g_string_free(s, TRUE); + g_queue_delete_link(con->send_queue->chunks, chunk); + chunk = con->send_queue->chunks->head; + i++; + } + + g_debug("%s: network socket:%p, send (src:%s, dst:%s) fd:%d", + G_STRLOC, + con, + con->src->name->str, + con->dst->name->str, + con->fd); + + len = send(con->fd, compress_packet->str, compress_packet->len, 0); + os_errno = errno; + + g_debug("%s: tcp write, comp:%d, write len:%d, total len:%d", + G_STRLOC, (int) compress_packet->len, (int) len, con->total_output); + + if (len > 0) { + if (len != compress_packet->len) { + g_debug("%s: tcp write len not equal to compressed packet, comp:%d, len:%d", + G_STRLOC, (int) compress_packet->len, (int) len); + con->last_compressed_packet = compress_packet; + con->compressed_unsend_offset = len; + return NETWORK_SOCKET_WAIT_FOR_EVENT; + } else { + g_string_free(compress_packet, TRUE); + con->last_compressed_packet = NULL; + con->compressed_unsend_offset = 0; + if (need_more_write) { + return NETWORK_SOCKET_WAIT_FOR_EVENT; + } + } + } else { + con->last_compressed_packet = compress_packet; + con->compressed_unsend_offset = 0; + } + + if (-1 == len) { + switch (os_errno) { + case E_NET_WOULDBLOCK: + case EAGAIN: + return NETWORK_SOCKET_WAIT_FOR_EVENT; + case EPIPE: + case E_NET_CONNRESET: + case E_NET_CONNABORTED: + /** remote side closed the connection */ + return NETWORK_SOCKET_ERROR; + default: + g_message("%s: writev(%s, ...) failed: %s", + G_STRLOC, + con->dst->name->str, + g_strerror(errno)); + return NETWORK_SOCKET_ERROR; + } + } else if (len == 0) { + return NETWORK_SOCKET_ERROR; + } + + return NETWORK_SOCKET_SUCCESS; +} + +/** + * write data to the socket + * + */ +static network_socket_retval_t +network_socket_write_writev(network_socket *con, int send_chunks) +{ + /* send the whole queue */ + GList *chunk; + struct iovec *iov; + gint chunk_id; + gint chunk_count; + gssize len; + int os_errno; + gint max_chunk_count; + + if (send_chunks == 0) return NETWORK_SOCKET_SUCCESS; + + chunk_count = send_chunks > 0 ? send_chunks : (gint)con->send_queue->chunks->length; + + if (chunk_count == 0) return NETWORK_SOCKET_SUCCESS; + + max_chunk_count = UIO_MAXIOV; + + chunk_count = chunk_count > max_chunk_count ? max_chunk_count : chunk_count; + + g_assert_cmpint(chunk_count, >, 0); /* make sure it is never negative */ + + iov = g_new0(struct iovec, chunk_count); + + for (chunk = con->send_queue->chunks->head, chunk_id = 0; + chunk && chunk_id < chunk_count; + chunk_id++, chunk = chunk->next) { + GString *s = chunk->data; + + if (chunk_id == 0) { + g_assert(con->send_queue->offset < s->len); + + iov[chunk_id].iov_base = s->str + con->send_queue->offset; + iov[chunk_id].iov_len = s->len - con->send_queue->offset; + } else { + iov[chunk_id].iov_base = s->str; + iov[chunk_id].iov_len = s->len; + } + + if (s->len == 0) { + g_warning("%s: s->len is zero", G_STRLOC); + } + } + + g_debug("%s: network socket:%p, send (src:%s, dst:%s) fd:%d", + G_STRLOC, + con, + con->src->name->str, + con->dst->name->str, + con->fd); + + len = writev(con->fd, iov, chunk_count); + g_debug("%s: tcp write:%d, chunk count:%d", G_STRLOC, (int) len, (int) chunk_count); + os_errno = errno; + + g_free(iov); + + if (-1 == len) { + switch (os_errno) { + case E_NET_WOULDBLOCK: + case EAGAIN: + return NETWORK_SOCKET_WAIT_FOR_EVENT; + case EPIPE: + case E_NET_CONNRESET: + case E_NET_CONNABORTED: + /** remote side closed the connection */ + return NETWORK_SOCKET_ERROR; + default: + g_message("%s: writev(%s, ...) failed: %s", + G_STRLOC, + con->dst->name->str, + g_strerror(errno)); + return NETWORK_SOCKET_ERROR; + } + } else if (len == 0) { + return NETWORK_SOCKET_ERROR; + } + + con->send_queue->offset += len; + con->send_queue->len -= len; + + /* check all the chunks which we have sent out */ + for (chunk = con->send_queue->chunks->head; chunk; ) { + GString *s = chunk->data; + + if (s->len == 0) { + g_warning("%s: s->len is zero", G_STRLOC); + } + + if (con->send_queue->offset >= s->len) { + con->send_queue->offset -= s->len; +#if NETWORK_DEBUG_TRACE_IO + g_debug("%s:output for sock:%p", G_STRLOC, con); + /* to trace the data we sent to the socket, enable this */ + g_debug_hexdump(G_STRLOC, S(s)); +#endif + if (!con->do_query_cache) { + g_string_free(s, TRUE); + } else { + size_t len = con->cache_queue->len + s->len; + if (len > MAX_QUERY_CACHE_SIZE) { + if (!con->query_cache_too_long) { + g_message("%s:too long for cache queue:%p, len:%d", + G_STRLOC, con, (int) len); + con->query_cache_too_long = 1; + } + g_string_free(s, TRUE); + } else { + g_debug("%s:append packet to cache queue:%p, len:%d, total:%d", + G_STRLOC, con, (int) s->len, (int) len); + network_queue_append(con->cache_queue, s); + } + } + + g_queue_delete_link(con->send_queue->chunks, chunk); + + chunk = con->send_queue->chunks->head; + } else { + g_debug("%s:wait for event", G_STRLOC); + return NETWORK_SOCKET_WAIT_FOR_EVENT; + } + } + + return NETWORK_SOCKET_SUCCESS; +} + +/** + * write a content of con->send_queue to the socket + * + * @param con socket to read from + * @param send_chunks number of chunks to send, if < 0 send all + * + * @returns NETWORK_SOCKET_SUCCESS on success, + * NETWORK_SOCKET_ERROR on error and + * NETWORK_SOCKET_WAIT_FOR_EVENT if the call would have blocked + */ +network_socket_retval_t network_socket_write(network_socket *sock, int send_chunks) { + if (sock->socket_type == SOCK_STREAM) { + if (sock->do_compress) { + return network_socket_compressed_write(sock, send_chunks); + } else { + return network_socket_write_writev(sock, send_chunks); + } + } else { + g_critical("%s: udp write is not supported", G_STRLOC); + return NETWORK_SOCKET_ERROR; + } +} + +network_socket_retval_t network_socket_to_read(network_socket *sock) { + int b = -1; + + if (0 != ioctl(sock->fd, FIONREAD, &b)) { + g_critical("%s: ioctl(%d, FIONREAD, ...) failed: %s (%d)", + G_STRLOC, + sock->fd, + g_strerror(errno), errno); + return NETWORK_SOCKET_ERROR; + } else if (b < 0) { + g_critical("%s: ioctl(%d, FIONREAD, ...) succeeded, but is negative: %d", + G_STRLOC, + sock->fd, + b); + + return NETWORK_SOCKET_ERROR; + } else { + sock->to_read = b; + return NETWORK_SOCKET_SUCCESS; + } + +} diff --git a/src/network-socket.h b/src/network-socket.h new file mode 100644 index 0000000..e014fcf --- /dev/null +++ b/src/network-socket.h @@ -0,0 +1,181 @@ +#ifndef _NETWORK_SOCKET_H_ +#define _NETWORK_SOCKET_H_ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "network-exports.h" +#include "network-queue.h" + +#ifdef HAVE_SYS_TIME_H +/** + * event.h needs struct timeval and doesn't include sys/time.h itself + */ +#include +#endif + +#include /** u_char */ +#include /** struct sockaddr */ + +#ifdef HAVE_NETINET_IN_H +#include /** struct sockaddr_in */ +#endif +#include + +#ifdef HAVE_SYS_UN_H +#include /** struct sockaddr_un */ +#endif +#define closesocket(x) close(x) +#include +#include + +#include "network-address.h" + +typedef enum { + NETWORK_SOCKET_SUCCESS, + NETWORK_SOCKET_WAIT_FOR_EVENT, + NETWORK_SOCKET_ERROR, + NETWORK_SOCKET_ERROR_RETRY +} network_socket_retval_t; + +typedef struct network_mysqld_auth_challenge network_mysqld_auth_challenge; +typedef struct network_mysqld_auth_response network_mysqld_auth_response; + +typedef struct server_state_data +{ + guint32 len; + int command; + + int qs_state; + + + union + { + struct + { + int want_eofs; + int first_packet; + } prepare; + + int query_type; + + struct + { + char state; + } auth_result; + + struct + { + GString *db_name; + } init_db; + } state; +} server_state_data; + +typedef struct +{ + int server_status; + int warning_count; + guint64 affected_rows; + guint64 insert_id; + + int was_resultset; + + int query_status; +} server_query_status; + +typedef struct { + int fd; /**< socket-fd */ + guint32 last_visit_time; + struct event event; /**< events for this fd */ + + network_address *src; /**< getsockname() */ + network_address *dst; /**< getpeername() */ + + int socket_type; /**< SOCK_STREAM or SOCK_DGRAM for now */ + + + /**< internal tracking of the packet_id's the automaticly set the next good packet-id */ + guint8 last_packet_id; + guint8 compressed_packet_id; + + /**< internal tracking of the packet_id sequencing */ + gboolean packet_id_is_reset; + + network_queue *recv_queue; + network_queue *recv_queue_raw; + network_queue *recv_queue_uncompress_raw; + network_queue *send_queue; + network_queue *cache_queue; + GString *last_compressed_packet; + int compressed_unsend_offset; + + off_t to_read; + off_t resp_len; + int total_output; + + /** + * data extracted from the handshake + */ + network_mysqld_auth_challenge *challenge; + network_mysqld_auth_response *response; + + unsigned int is_authed:1; /** did a client already authed this connection */ + unsigned int is_server_conn_reserved:1; + + /* if not in a trx, use a "short timeout" to trigger (proxy_timeout), + so that server conns will be returned to pool immediately */ + unsigned int is_need_q_peek_exec:1; + unsigned int is_multi_stmt_set:1; + unsigned int is_closed:1; + unsigned int unavailable:1; + unsigned int is_in_sess_context:1; + unsigned int is_in_tran_context:1; + unsigned int is_robbed:1; + unsigned int is_waiting:1; + unsigned int is_read_only:1; + unsigned int is_read_finished:1; + unsigned int query_cache_too_long:1; + unsigned int max_header_size_reached:1; + unsigned int do_compress:1; + unsigned int do_strict_compress:1; + unsigned int do_query_cache:1; + + guint8 charset_code; + + /** + * store the default-db of the socket + * + * the client might have a different default-db than the server-side due to + * statement balancing + */ + GString *default_db; /** default-db of this side of the connection */ + /* only used for server */ + GString *username; + GString *group; + + GString *charset; + GString *charset_client; + GString *charset_connection; + GString *charset_results; + GString *sql_mode; + server_state_data parse; + server_query_status qstat; + +} network_socket; + + +NETWORK_API network_socket *network_socket_new(void); +NETWORK_API void network_socket_free(network_socket *s); +NETWORK_API network_socket_retval_t network_socket_write(network_socket *con, int send_chunks); +NETWORK_API network_socket_retval_t network_socket_read(network_socket *con); +NETWORK_API network_socket_retval_t network_socket_to_read(network_socket *sock); +NETWORK_API network_socket_retval_t network_socket_set_non_blocking(network_socket *sock); +NETWORK_API network_socket_retval_t network_socket_connect(network_socket *con); +NETWORK_API network_socket_retval_t network_socket_connect_finish(network_socket *sock); +NETWORK_API network_socket_retval_t network_socket_bind(network_socket *con); +NETWORK_API network_socket *network_socket_accept(network_socket *srv, int *reason); +NETWORK_API network_socket_retval_t network_socket_set_send_buffer_size(network_socket *sock, int size); + +#endif + diff --git a/src/plugin-common.c b/src/plugin-common.c new file mode 100644 index 0000000..e17b0b6 --- /dev/null +++ b/src/plugin-common.c @@ -0,0 +1,601 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_SYS_FILIO_H +#include +#endif + +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include + +#ifndef HAVE_SOCKLEN_T +typedef int socklen_t; +#endif + + +#include /** for ER_UNKNOWN_ERROR */ + +#include "network-mysqld.h" +#include "network-mysqld-proto.h" +#include "network-mysqld-packet.h" + +#include "network-conn-pool.h" +#include "network-conn-pool-wrap.h" + +#include "sys-pedantic.h" +#include "network-injection.h" +#include "network-backend.h" +#include "sql-context.h" +#include "sql-filter-variables.h" +#include "glib-ext.h" +#include "chassis-timings.h" +#include "chassis-event.h" +#include "character-set.h" +#include "cetus-util.h" +#include "cetus-users.h" +#include "chassis-options.h" +#include "plugin-common.h" + +#define MAX_CACHED_ITEMS 65536 + +network_socket_retval_t do_read_auth(network_mysqld_con *con, GHashTable *allow_ip_table, GHashTable *deny_ip_table) +{ + /* read auth from client */ + network_packet packet; + network_socket *recv_sock; + network_mysqld_auth_response *auth; + + recv_sock = con->client; + + packet.data = g_queue_peek_tail(recv_sock->recv_queue->chunks); + packet.offset = 0; + + /* assume that we may get called twice: + * + * 1. for the initial packet + * 2. for the win-auth extra data + * + * this is detected by con->client->response being NULL + */ + + if (con->client->response == NULL) { + + if (con->client->challenge == NULL) { + return NETWORK_SOCKET_ERROR; + } + + guint32 capabilities = con->client->challenge->capabilities; + auth = network_mysqld_auth_response_new(capabilities); + + int err = network_mysqld_proto_get_and_change_auth_response(&packet, auth); + + if (err) { + network_mysqld_auth_response_free(auth); + return NETWORK_SOCKET_ERROR; + } + if (!(auth->client_capabilities & CLIENT_PROTOCOL_41)) { + /* should use packet-id 0 */ + network_mysqld_queue_append(con->client, con->client->send_queue, + C("\xff\xd7\x07" "4.0 protocol is not supported")); + network_mysqld_auth_response_free(auth); + return NETWORK_SOCKET_ERROR; + } else if (auth->client_capabilities & CLIENT_COMPRESS) { + con->is_client_compressed = 1; + g_message("%s: client compressed for con:%p", G_STRLOC, con); + } else if (auth->client_capabilities & CLIENT_MULTI_STATEMENTS) { + con->client->is_multi_stmt_set = 1; + } + + con->client->response = auth; + + g_string_assign_len(con->client->default_db, S(auth->database)); + g_debug("%s:1nd round auth and set default db:%s for con:%p", + G_STRLOC, con->client->default_db->str, con); + + } else { + GString *auth_data; + gsize auth_data_len; + + /* + * get all the data from the packet and append it + * to the auth_plugin_data + */ + auth_data_len = packet.data->len - 4; + auth_data = g_string_sized_new(auth_data_len); + network_mysqld_proto_get_gstr_len(&packet, auth_data_len, auth_data); + + g_string_append_len(con->client->response->auth_plugin_data, + S(auth_data)); + + g_string_free(auth_data, TRUE); + + auth = con->client->response; + g_debug("sock:%p, 2nd round auth", con); + } + + /* Check allow and deny IP */ + gboolean check_ip; + if (allow_ip_table || deny_ip_table) { + char **client_addr_arr = g_strsplit(con->client->src->name->str, ":", -1); + char *client_ip = client_addr_arr[0]; + char *client_username = con->client->response->username->str; + char *client_ip_with_username = g_strdup_printf("%s@%s", client_username, client_ip); + char *ip_err_msg = NULL; + if ((g_hash_table_size(allow_ip_table) != 0 && + (g_hash_table_lookup(allow_ip_table, client_ip) || g_hash_table_lookup(deny_ip_table, "*"))) || + g_hash_table_lookup(allow_ip_table, client_ip_with_username)) { + check_ip = FALSE; + } else if ((g_hash_table_size(deny_ip_table) != 0 && + (g_hash_table_lookup(deny_ip_table, client_ip) || g_hash_table_lookup(deny_ip_table, "*"))) || + g_hash_table_lookup(deny_ip_table, client_ip_with_username)) { + check_ip = TRUE; + ip_err_msg = g_strdup_printf( + "Access denied for user '%s'@'%s'", client_username, client_ip); + //g_debug("Allow IP check failed: '%s'@'%s'", client_username, client_ip); + } else { + check_ip = FALSE; + } + g_free(client_ip_with_username); + g_strfreev(client_addr_arr); + if (check_ip) { + network_mysqld_con_send_error_full(recv_sock, L(ip_err_msg), 1045, "28000"); + g_free(ip_err_msg); + con->state = ST_SEND_ERROR; + return NETWORK_SOCKET_SUCCESS; + } + } + + const char *client_charset = charset_get_name(auth->charset); + recv_sock->charset_code = auth->charset; + g_string_assign(recv_sock->charset, client_charset); + g_string_assign(recv_sock->charset_client, client_charset); + g_string_assign(recv_sock->charset_results, client_charset); + g_string_assign(recv_sock->charset_connection, client_charset); + + cetus_users_t *users = con->srv->priv->users; + network_mysqld_auth_challenge *challenge = con->client->challenge; + network_mysqld_auth_response *response = con->client->response; + if (cetus_users_authenticate_client(users, challenge, response)) { + con->state = ST_SEND_AUTH_RESULT; + network_mysqld_con_send_ok(recv_sock); + } else { + char msg[256] = {0}; + snprintf(msg, sizeof(msg), + "Access denied for user '%s'@'%s' (using password: YES)", + response->username->str, + con->client->src->name->str); + network_mysqld_con_send_error_full(con->client, L(msg), + ER_ACCESS_DENIED_ERROR, "28000"); + g_message("%s", msg); + con->login_failed = 1; + con->state = ST_SEND_ERROR; + } + + g_string_free(g_queue_pop_tail(recv_sock->recv_queue->chunks), TRUE); + if (recv_sock->recv_queue->chunks->length > 0) { + g_warning("%s: client-recv-queue-len = %d", + G_STRLOC, recv_sock->recv_queue->chunks->length); + } + + return NETWORK_SOCKET_SUCCESS; +} + +static int proxy_c_connect_server(network_mysqld_con *con, + network_backend_t **p_backend, int *p_backend_ndx) +{ + int i, num; + network_backends_t *bs = con->srv->priv->backends; + num = bs->backends->len; + + for (i = 0; i < num; i++) { + network_backend_t *backend = g_ptr_array_index(bs->backends, i); + + if (backend->state != BACKEND_STATE_UP && + backend->state != BACKEND_STATE_UNKNOWN) + { + continue; + } + + if (backend->config == NULL) { + g_warning("%s, config is null for back ndx:%d", + G_STRLOC, i); + continue; + } + + int total = network_backend_conns_count(backend); + int connected_clts = backend->connected_clients; + int cur_idle = total - connected_clts; + int max_idle_conns = backend->config->max_conn_pool; + + g_debug("%s, ndx:%d, total:%d, connected:%d, idle:%d, max:%d", + G_STRLOC, i, total, connected_clts, + cur_idle, max_idle_conns); + + if (cur_idle > 0 || total <= max_idle_conns) { + *p_backend_ndx = i; + *p_backend = backend; + break; + } + } + + if (i == num) { + g_message("%s, service unavailable for con:%p, back ndx:%d", + G_STRLOC, con, *p_backend_ndx); + /* no backend server */ + network_mysqld_con_send_error(con->client, + C("(proxy) service unavailable")); + return PROXY_SEND_RESULT; + } + + return PROXY_IGNORE_RESULT; +} + +network_socket_retval_t do_connect_cetus(network_mysqld_con *con, + network_backend_t **backend, int *backend_ndx) +{ + chassis_private *g = con->srv->priv; + guint i; + network_backend_t *cur; + + *backend = NULL; + *backend_ndx = -1; + + /* Disable backend check + * Conflict to backend state active check. + */ + guint disable_threads = con->srv->disable_threads; + if (disable_threads) { + network_backends_check(g->backends); + } + + network_mysqld_stmt_ret ret; + ret = proxy_c_connect_server(con, backend, backend_ndx); + + switch (ret) { + case PROXY_SEND_RESULT: + /* we answered directly ... like denial ... + * + * for sure we have something in the send-queue + * + */ + + return NETWORK_SOCKET_SUCCESS; + case PROXY_NO_DECISION: + /* just go on */ + + break; + case PROXY_IGNORE_RESULT: + break; + default: + g_error("%s: ... ", G_STRLOC); + break; + } + + /* protect the typecast below */ + g_assert_cmpint(g->backends->backends->len, <, G_MAXINT); + + /** + * if the current backend is down, ignore it + */ + cur = network_backends_get(g->backends, *backend_ndx); + + if (cur) { + if (cur->state == BACKEND_STATE_DOWN || + cur->state == BACKEND_STATE_MAINTAINING) { + *backend_ndx = -1; + } + } + + if (*backend_ndx < 0) { + /** + * we can choose between different back addresses + * + * prefer SQF (shorted queue first) to load all backends equally + */ + + int min_connected_clients = 0x7FFFFFFF; + for (i = 0; i < network_backends_count(g->backends); i++) { + cur = network_backends_get(g->backends, i); + + /** + * skip backends which are down or not writable + */ + if (cur->state == BACKEND_STATE_DOWN || + cur->state == BACKEND_STATE_MAINTAINING || + cur->type != BACKEND_TYPE_RW) continue; + + if (cur->connected_clients < min_connected_clients) { + *backend_ndx = i; + min_connected_clients = cur->connected_clients; + } + } + + if ((cur = network_backends_get(g->backends, *backend_ndx))) { + *backend = cur; + } + } else if (*backend == NULL) { + if ((cur = network_backends_get(g->backends, *backend_ndx))) { + *backend = cur; + } + } + + if (*backend == NULL) { + network_mysqld_con_send_error_pre41(con->client, + C("(proxy) all backends are down")); + g_critical("%s: Cannot connect, all backends are down.", G_STRLOC); + return NETWORK_SOCKET_ERROR; + } + + network_mysqld_auth_challenge *challenge = + network_backends_get_challenge(g->backends, *backend_ndx); + + if (challenge == NULL) { + network_connection_pool_create_conn(con); + network_mysqld_con_send_error(con->client, + C(" no server challenge for this client")); + con->state = ST_SEND_AUTH_RESULT; + return NETWORK_SOCKET_SUCCESS; + } + + GString *auth_packet = g_string_new(NULL); + network_mysqld_proto_append_auth_challenge(auth_packet, challenge); + + network_mysqld_queue_append( + con->client, + con->client->send_queue, + S(auth_packet)); + + g_string_free(auth_packet, TRUE); + + g_assert(con->client->challenge == NULL); + + con->client->challenge = network_mysqld_auth_challenge_copy(challenge); + + con->state = ST_SEND_HANDSHAKE; + + /** + * connect_clients is already incremented + */ + + return NETWORK_SOCKET_SUCCESS; +} + + +network_socket_retval_t plugin_add_backends(chassis *chas, + gchar **backend_addresses, gchar **read_only_backend_addresses) +{ + guint i; + chassis_private *g = chas->priv; + + GPtrArray *backends_arr = g->backends->backends; + for (i = 0; backend_addresses[i]; i++) { + if (-1 == network_backends_add(g->backends, backend_addresses[i], + BACKEND_TYPE_RW, BACKEND_STATE_DOWN, chas)) { + return -1; + } + network_backend_init_extra(backends_arr->pdata[backends_arr->len - 1], chas); + } + + for (i = 0; read_only_backend_addresses && read_only_backend_addresses[i]; i++) + { + if (-1 == network_backends_add(g->backends, + read_only_backend_addresses[i], + BACKEND_TYPE_RO, BACKEND_STATE_DOWN, chas)) + { + return -1; + } + /* set conn-pool config */ + network_backend_init_extra(backends_arr->pdata[backends_arr->len - 1], chas); + } + + g_message("%s, ro server num:%d", G_STRLOC, i); + + return 0; +} + +int do_check_qeury_cache(network_mysqld_con *con) +{ + if (con->is_client_compressed) { + return 0; + } + + con->query_cache_judged = 1; + gettimeofday(&(con->resp_recv_time), NULL); + int diff = (con->resp_recv_time.tv_sec - con->req_recv_time.tv_sec) * 1000; + diff += (con->resp_recv_time.tv_usec - con->req_recv_time.tv_usec) / 1000; + g_debug("%s:req time:%d, min:%d for cache", G_STRLOC, + diff, con->srv->min_req_time_for_cache); + if (diff >= con->srv->min_req_time_for_cache) { + if (g_hash_table_size(con->srv->query_cache_table) < MAX_CACHED_ITEMS) { + con->client->do_query_cache = 1; + con->client->cache_queue = network_queue_new(); + g_debug("%s: candidate for query cache", G_STRLOC); + return 1; + } else { + g_message("%s: too many cached items", G_STRLOC); + } + } else { + g_debug("%s: not cached for sql:%s", G_STRLOC, con->orig_sql->str); + } + + return 0; +} + +int try_to_get_resp_from_query_cache(network_mysqld_con *con) +{ + chassis *srv = con->srv; + GString *key = g_string_new(NULL); + g_string_append(key, con->orig_sql->str); + g_string_append(key, con->client->response->username->str); + g_string_append(key, con->client->default_db->str); + gchar *md5_key = g_compute_checksum_for_string(G_CHECKSUM_MD5, S(key)); + + g_debug("%s:visit try_to_get_resp_from_query_cache:%s", G_STRLOC, key->str); + g_string_free(key, TRUE); + + unsigned long long access_ms; + access_ms = con->req_recv_time.tv_sec * 1000 + con->req_recv_time.tv_usec / 1000; + /* purge first */ + if (srv->last_cache_purge_time != access_ms) { + int len = srv->cache_index->length; + while (len > 0) { + query_cache_index_item *index = g_queue_peek_head(srv->cache_index); + if (index->expire_ms > access_ms) { + break; + } else { + g_debug("%s:drop content from cache:%s", G_STRLOC, index->key); + g_hash_table_remove(con->srv->query_cache_table, index->key); + g_queue_pop_head(srv->cache_index); + len = srv->cache_index->length; + g_free(index->key); + g_free(index); + } + } + + srv->last_cache_purge_time = access_ms; + } + + query_cache_item *item = g_hash_table_lookup(con->srv->query_cache_table, md5_key); + g_free(md5_key); + + if (item != NULL) { + int i; + int len = item->queue->chunks->length; + for (i = 0; i < len; i++) { + GString *packet = g_queue_peek_nth(item->queue->chunks, i); + GString *dup_packet = g_string_new(NULL); + g_string_append_len(dup_packet, S(packet)); + network_queue_append(con->client->send_queue, dup_packet); + g_debug("%s:read packet len:%d from cache", G_STRLOC, (int) dup_packet->len); + g_debug_hexdump(G_STRLOC, S(dup_packet)); + } + con->state = ST_SEND_QUERY_RESULT; + con->client->do_query_cache = 0; + g_debug("%s:read content from cache:%s", G_STRLOC, con->orig_sql->str); + + network_queue_clear(con->client->recv_queue); + network_mysqld_queue_reset(con->client); + + return 1; + } else { + g_debug("%s:no cached item for con:%p", G_STRLOC, con); + return 0; + } +} + +gboolean proxy_put_shard_conn_to_pool(network_mysqld_con *con) +{ + int i; + int is_reduced = 0; + + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = (server_session_t *)g_ptr_array_index(con->servers, i); + if (pmd) { + network_connection_pool *pool = pmd->backend->pool; + network_socket *server = pmd->server; + int is_put_to_pool_allowed = 1; + + if (con->server_to_be_closed) { + is_put_to_pool_allowed = 0; + g_debug("%s: con server_to_be_closed is true", G_STRLOC); + } + if (is_put_to_pool_allowed && server->is_closed) { + is_put_to_pool_allowed = 0; + g_debug("%s: server is_closed is true", G_STRLOC); + } + if (is_put_to_pool_allowed && server->is_in_tran_context) { + is_put_to_pool_allowed = 0; + g_debug("%s: is_in_tran_context is true", G_STRLOC); + } + if (is_put_to_pool_allowed && pmd->is_in_xa && !pmd->is_xa_over) { + is_put_to_pool_allowed = 0; + g_warning("%s: xa is not over yet", G_STRLOC); + } + + if (is_put_to_pool_allowed && server->recv_queue->chunks->length > 0) { + g_message("%s: server recv queue not empty, sql:%s", + G_STRLOC, con->orig_sql->str); + is_put_to_pool_allowed = 0; + } + + if (con->resp_too_long) { + g_message("%s: resp too long for con:%p", G_STRLOC, con); + } + + is_reduced = 0; + if (con->srv->is_reduce_conns && is_put_to_pool_allowed) { + if (network_conn_pool_do_reduce_conns_verdict(pool, + pmd->backend->connected_clients)) + { + is_reduced = 1; + is_put_to_pool_allowed = 0; + } + } + + CHECK_PENDING_EVENT(&(server->event)); + + if (is_put_to_pool_allowed) { + g_debug("%s: is_put_to_pool_allowed true here, server:%p, con:%p, num:%d", + G_STRLOC, server, con, (int) con->servers->len); + network_pool_add_idle_conn(pool, con->srv, server); + } else { + g_debug("%s: is_put_to_pool_allowed false here, server:%p, con:%p, num:%d", + G_STRLOC, server, con, (int) con->servers->len); + network_socket_free(server); + if (!is_reduced) { + con->srv->complement_conn_cnt++; + } + } + + pmd->backend->connected_clients--; + g_debug("%s: conn clients sub, total len:%d, backend:%p, value:%d con:%p", + G_STRLOC, con->servers->len, pmd->backend, + pmd->backend->connected_clients, con); + + pmd->sql = NULL; + g_free(pmd); + } + } + + g_ptr_array_free(con->servers, TRUE); + con->servers = NULL; + con->client->is_server_conn_reserved = 0; + con->attr_adj_state = ATTR_START; + + return TRUE; +} + +void +remove_mul_server_recv_packets(network_mysqld_con *con) +{ + int iter; + + if (con->servers == NULL) { + g_critical("%s: con servers is NULL", G_STRLOC); + return; + } + + for (iter = 0; iter < con->servers->len; iter++) { + server_session_t *pmd = g_ptr_array_index(con->servers, iter); + g_debug("%s: remove packets for server:%p", G_STRLOC, pmd->server); + GQueue *out = pmd->server->recv_queue->chunks; + GString *packet = g_queue_pop_head(out); + while (packet) { + g_string_free(packet, TRUE); + packet = g_queue_pop_head(out); + } + network_mysqld_queue_reset(pmd->server); + } +} + diff --git a/src/plugin-common.h b/src/plugin-common.h new file mode 100644 index 0000000..c2507d7 --- /dev/null +++ b/src/plugin-common.h @@ -0,0 +1,33 @@ +#ifndef _PLUGIN_COMMON_H +#define _PLUGIN_COMMON_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_SYS_TIME_H +/** + * event.h needs struct timeval and doesn't include sys/time.h itself + */ +#include +#endif + +#include + +#include + +#include + +#include + +#include "network-exports.h" + +NETWORK_API network_socket_retval_t do_read_auth(network_mysqld_con *, GHashTable *, GHashTable *); +NETWORK_API network_socket_retval_t do_connect_cetus(network_mysqld_con *, network_backend_t **, int *); +NETWORK_API network_socket_retval_t plugin_add_backends(chassis *, gchar **, gchar **); +NETWORK_API int do_check_qeury_cache(network_mysqld_con *con); +NETWORK_API int try_to_get_resp_from_query_cache(network_mysqld_con *con); +NETWORK_API gboolean proxy_put_shard_conn_to_pool(network_mysqld_con *con); +NETWORK_API void remove_mul_server_recv_packets(network_mysqld_con *con); + +#endif diff --git a/src/resultset_merge.c b/src/resultset_merge.c new file mode 100644 index 0000000..884585a --- /dev/null +++ b/src/resultset_merge.c @@ -0,0 +1,3152 @@ +#include +#include +#include +#include +#include +#include + +#include +#include "glib-ext.h" +#include "network-mysqld-packet.h" +#include "sys-pedantic.h" +#include "resultset_merge.h" +#include "sql-context.h" +#include "shard-plugin-con.h" +#include "server-session.h" +#include "chassis-event.h" +#include "sharding-query-plan.h" + +const char EPOCH[] = "1970-01-01 00:00:00"; +const char *type_name[] = +{ + "FIELD_TYPE_DECIMAL", //0x00 + "FIELD_TYPE_TINY", //0x01 + "FIELD_TYPE_SHORT", //0x02 + "FIELD_TYPE_LONG", //0x03 + "FIELD_TYPE_FLOAT", //0x04 + "FIELD_TYPE_DOUBLE", //0x05 + "FIELD_TYPE_NULL", //0x06 + "FIELD_TYPE_TIMESTAMP", //0x07 + "FIELD_TYPE_LONGLONG", //0x08 + "FIELD_TYPE_INT24", //0x09 + "FIELD_TYPE_DATE", //0x0a + "FIELD_TYPE_TIME", //0x0b + "FIELD_TYPE_DATETIME", //0x0c + "FIELD_TYPE_YEAR", //0x0d + "FIELD_TYPE_NEWDATE", //0x0e + "FIELD_TYPE_VARCHAR", //0x0f + "FIELD_TYPE_BIT", //0x10 + "FIELD_TYPE_NEWDECIMAL", //0xf6 + "FIELD_TYPE_ENUM", //0xf7 + "FIELD_TYPE_SET", //0xf8 + "FIELD_TYPE_TINY_BLOB", //0xf9 + "FIELD_TYPE_MEDIUM_BLOB",//0xfa + "FIELD_TYPE_LONG_BLOB", //0xfb + "FIELD_TYPE_BLOB", //0xfc + "FIELD_TYPE_VAR_STRING", //0xfd + "FIELD_TYPE_STRING", //0xfe + "FIELD_TYPE_GEOMETRY" //0xff +}; + +#define MAX_PACK_LEN 2048 +#define MAX_COL_VALUE_LEN 512 + +#define PRIOR_TO 1 +#define NOR_REL 0 + +typedef struct cetus_result_t { + network_mysqld_proto_fielddefs_t *fielddefs; + + /** + * if fielddefs != NULL, field_count equals fielddefs->len + * sometime no need to parse fielddefs, we only want field_count + */ + int field_count; +} cetus_result_t; + +static void cetus_result_destroy(cetus_result_t *res) +{ + if (res->fielddefs) { + network_mysqld_proto_fielddefs_free(res->fielddefs); + res->fielddefs = NULL; + } +} + +static int compare_records_from_column(char *, char *, int, int, int *); + +static int check_str_num_supported(char *s, int len, char **p) +{ + if (len > 0) { + if (s[0] == '+' || s[0] == '-') { + s++; + len--; + } + } else { + return 1; + } + + int i, is_valid = 1; + int dot_cnt = 0, end = len - 1; + + for (i = 0; i < len; i++) { + if (s[i] == '.') { + dot_cnt++; + if (i == 0 || i == end || dot_cnt > 1) { + is_valid = 0; + } else { + if (p != NULL) { + *p = s + i; + } + } + } else { + if (s[i] < '0' || s[i] > '9') { + is_valid = 0; + } + } + } + + if (is_valid) { + return 1; + } else { + return 0; + } +} + +static int +cmp_str_pos_num(char *s1, int len1, char *s2, int len2, int *compare_failed) +{ + char *p = NULL, *q = NULL; + + if (!check_str_num_supported(s1, len1, &p)) { + g_warning("%s: str num is not supported:%s", G_STRLOC, s1); + *compare_failed = 1; + return -1; + } + + if (!check_str_num_supported(s2, len2, &q)) { + g_warning("%s: str num is not supported:%s", G_STRLOC, s2); + *compare_failed = 1; + return -1; + } + + int s1_int_len; + if (p == NULL) { + s1_int_len = len1; + } else { + s1_int_len = p - s1; + } + + int s2_int_len; + if (q == NULL) { + s2_int_len = len2; + } else { + s2_int_len = q - s2; + } + + char *end = s1 + len1 - 1; + while (s1[0] == '0' && s1[1] != '.' && s1 < end) { + s1_int_len--; + s1++; + len1--; + } + + end = s2 + len2 - 1; + while (s2[0] == '0' && s2[1] != '.' && s2 < end) { + s2_int_len--; + s2++; + len2--; + } + + if (s1_int_len > s2_int_len) { + return 1; + } else if (s1_int_len < s2_int_len) { + return -1; + } + + int i; + for (i = 0; i < s1_int_len; i++) { + if (s1[i] > s2[i]) { + return 1; + } else if (s1[i] < s2[i]) { + return -1; + } + } + + if (p == NULL && q == NULL) { + return 0; + } + + char *short1, *short2; + if (p == NULL) { + short2 = q + 1; + while (short2[0] != '\0') { + if (short2[0] != '0') { + return -1; + } + short2++; + } + return 0; + } else if (q == NULL) { + short1 = p + 1; + while (short1[0] != '\0') { + if (short1[0] != '0') { + return 1; + } + short1++; + } + return 0; + } else { + short1 = p + 1; + short2 = q + 1; + } + + int short_len = len1 - s1_int_len - 1; + + for (i = 0; i < short_len; i++) { + if (short1[i] > short2[i]) { + return 1; + } else if (short1[i] < short2[i]) { + return -1; + } + } + + return 0; +} + +static int +cmp_str_num(char *s1, int len1, char *s2, int len2, int *compare_failed) +{ + if (len1 == 0 || len2 == 0) { + g_critical("%s len in cmp_str_num is nil", G_STRLOC); + } + + char *pos_s1 = s1, *pos_s2 = s2; + int pos_len1 = len1, pos_len2 = len2; + int neg_s1 = 0, neg_s2 = 0; + + if (s1[0] == '-') { + pos_len1--; + pos_s1++; + neg_s1 = 1; + } else if (s1[0] == '+') { + pos_len1--; + pos_s1++; + } + + if (s2[0] == '-') { + pos_len2--; + pos_s2++; + neg_s2 = 1; + } else if (s2[0] == '+') { + pos_len2--; + pos_s2++; + } + + if (neg_s1 && neg_s2) { + return (-1) * cmp_str_pos_num(pos_s1, pos_len1, pos_s2, pos_len2, compare_failed); + } else if (neg_s1) { + return -1; + } else if (neg_s2) { + return 1; + } else { + return cmp_str_pos_num(pos_s1, pos_len1, pos_s2, pos_len2, compare_failed); + } +} + +static int +padding_zero(char *s1, int *p_len1, int size1, char *s2, int *p_len2, int size2) +{ + char *p = NULL, *q = NULL; + + int len1 = *p_len1; + int len2 = *p_len2; + + if (len1 == 0) { + s1[0] = '0'; + len1 = 1; + *p_len1 = len1; + } else { + p = strchr(s1, '.'); + } + + if (len2 == 0) { + s2[0] = '0'; + len2 = 1; + *p_len2 = len2; + } else { + q = strchr(s2, '.'); + } + + if (p == NULL && q == NULL) { + return 1; + } + + int s1_int_len, s2_int_len; + int short_len1 = 0, short_len2 = 0; + char *short1, *short2; + + if (p == NULL) { + s1_int_len = len1; + s1[len1] = '.'; + *p_len1 = (*p_len1) + 1; + short1 = s1 + len1 + 1; + } else { + s1_int_len = p - s1; + short1 = p + 1; + short_len1 = len1 - s1_int_len - 1; + } + + if (q == NULL) { + s2_int_len = len2; + s2[len2] = '.'; + *p_len2 = (*p_len2) + 1; + short2 = s2 + len2 + 1; + } else { + s2_int_len = q - s2; + short2 = q + 1; + short_len2 = len2 - s2_int_len - 1; + } + + char *padding; + int pad_len, new_str_len; + if (short_len1 == short_len2) { + return 1; + } else if (short_len1 < short_len2) { + padding = short1 + short_len1; + pad_len = short_len2 - short_len1; + new_str_len = (*p_len1) + pad_len; + if (new_str_len >= size1) { + return 0; + } + *p_len1 = new_str_len; + } else { + padding = short2 + short_len2; + pad_len = short_len1 - short_len2; + new_str_len = (*p_len2) + pad_len; + if (new_str_len >= size2) { + return 0; + } + *p_len2 = new_str_len; + } + + int i; + + for (i = 0; i < pad_len; i++) { + padding[i] = '0'; + } + + return 1; +} + +static int +compare_str_num_value(char *str1, char *str2, int com_type, int *dest, int desc, + int *compare_failed) +{ + int len1 = strlen(str1); + int len2 = strlen(str2); + + if (!padding_zero(str1, &len1, MAX_COL_VALUE_LEN, + str2, &len2, MAX_COL_VALUE_LEN)) + { + *compare_failed = 1; + return 0; + } + + int result = cmp_str_num(str1, len1, str2, len2, compare_failed); + + g_debug("%s:string str1:%s, str2:%s, cmp result:%d", + G_STRLOC, str1, str2, result); + + if (result == 0) { + return 1; + } + + if ((desc && result > 0) || (!desc && result < 0)) { + *dest = 1; + return 0; + } + + if ((desc && result < 0) || (!desc && result > 0)) { + if (com_type == PRIOR_TO) { + *dest = 0; + } else { + *dest = -1; + } + return 0; + } + + g_critical("%s: reach the unreachable place", G_STRLOC); + return 0; +} + + +static int compare_date(char *str1, char *str2, int com_type, int *result, int desc) +{ + if (str1[0] == '\0') { + strcpy(str1, EPOCH); + } + + if (str2[0] == '\0') { + strcpy(str2, EPOCH); + } + + struct tm tm1 = {0}; + struct tm tm2 = {0}; + g_debug("%s:data1:%s, data2:%s", G_STRLOC, str1, str2); + strptime(str1, "%Y-%m-%d", &tm1); + strptime(str2, "%Y-%m-%d", &tm2); + + int diff[6] ; + diff[0] = tm1.tm_year - tm2.tm_year; + diff[1] = tm1.tm_mon - tm2.tm_mon; + diff[2] = tm1.tm_mday - tm2.tm_mday; + + g_debug("%s:diff0:%d, diff1:%d, diff2:%d", G_STRLOC, diff[0], diff[1], diff[2]); + int p; + for (p = 0; p < 3; p++) { + if ((desc && diff[p] > 0) || (!desc && diff[p] < 0)) { + *result = 1; + return 0; + } + + if ((desc && diff[p] < 0) || (!desc && diff[p] > 0)) { + if (com_type == PRIOR_TO) { + *result = 0; + } else { + *result = -1; + } + return 0; + } + } + + return 1; +} + +static int compare_time(char *str1, char *str2, int com_type, int *result, int desc) +{ + if (str1[0] == '\0') { + strcpy(str1, EPOCH); + } + + if (str2[0] == '\0') { + strcpy(str2, EPOCH); + } + + struct tm tm1 = {0}; + struct tm tm2 = {0}; + + strptime(str1, "%H:%M:%S", &tm1); + strptime(str2, "%H:%M:%S", &tm2); + + int diff[3] ; + diff[0] = tm1.tm_hour - tm2.tm_hour; + diff[1] = tm1.tm_min - tm2.tm_min; + diff[2] = tm1.tm_sec - tm2.tm_sec; + + int p; + for (p = 0; p < 3; p++) { + if ((desc && diff[p] > 0) || (!desc && diff[p] < 0)) { + *result = 1; + return 0; + } + + if ((desc && diff[p] < 0) || (!desc && diff[p] > 0)) { + if (com_type == PRIOR_TO) { + *result = 0; + } else { + *result = -1; + } + + return 0; + } + } + + return 1; +} + + +static int compare_year(char *str1, char *str2, int com_type, int *result, int desc) +{ + if (str1[0] == '\0') { + strcpy(str1, EPOCH); + } + + if (str2[0] == '\0') { + strcpy(str2, EPOCH); + } + + long y1 = atol(str1); + long y2 = atol(str2); + + if (y1 == y2) { + return 1; + } + + if (desc) { + *result = y1 > y2; + } else { + *result = y1 < y2; + } + + if (com_type != PRIOR_TO) { + if (*result == 0) { + *result = -1; + } + } + + return 0; +} + +/* skip some column (lenenc_str or NULL) */ +static inline gint skip_field(network_packet *packet, guint skip) { + guint iter; + + for (iter = 0; iter < skip; iter++) { + guint8 first = 0; + if (network_mysqld_proto_peek_int8(packet, &first) == -1) { + return -1; + } + + if (first == MYSQLD_PACKET_NULL) { + network_mysqld_proto_skip(packet, 1); + } else { + if (network_mysqld_proto_skip_lenenc_str(packet) == -1) { + return -1; + } + } + } + return 0; +} + +#define MAX_ORDER_BY_ITEMS 16 + +static guint64 set_rec_fields_off(network_packet *packet, + ORDER_BY order_array[], int order_array_size) +{ + int i, max_pos = 0; + int orderby_count = MIN(order_array_size, 4); + + for (i = 0; i < orderby_count; i++) { + if (order_array[i].pos > max_pos) { + max_pos = order_array[i].pos; + } + } + + if (max_pos > MAX_ORDER_BY_ITEMS) { + return 0; + } + + unsigned char map[MAX_ORDER_BY_ITEMS] = {0}; /* field pos ==> order by pos */ + for (i = 0; i < orderby_count; i++) { + /* hack: add 1 for existence probing */ + map[order_array[i].pos] = i + 1; + } + + guint64 value = 0; + if (max_pos == 0) { + value = NET_HEADER_SIZE; + } + + guint iter; + for (iter = 0; iter < max_pos; iter++) { + if (packet->data->str[packet->offset] == MYSQLD_PACKET_NULL) { + network_mysqld_proto_skip(packet, 1); + } else { + if (network_mysqld_proto_skip_lenenc_str(packet) == -1) { + return 0; + } + } + size_t next_iter = iter + 1; + if (next_iter < MAX_ORDER_BY_ITEMS && map[next_iter]) { + int seq = map[next_iter] - 1; + guint64 new_value = packet->offset; + g_debug("offset value:%ld, seq=%d, iter=%d %p", + new_value, seq, iter, packet->data); + if (new_value <= 0xFFFF) { + int j; + for (j = 0; j < seq; j++) { + new_value = new_value << 16; + } + + value += new_value; + } + g_debug("part all value set:%ld for %p", value, packet->data); + } + } + + return value; +} + +/** + * OK Packet 0x00 + * Error Packet 0xff 255 + * Result Set Packet 1-250 (first byte of Length-Coded Binary) + * Field Packet 1-250 ("") + * Row Data Packet 1-250 ("") + * EOF Packet 0xfe 254 + */ +static inline guchar get_pkt_type(GString *pkt) +{ + return (unsigned char) pkt->str[NET_HEADER_SIZE]; +} + +static char *retrieve_aggr_value(GString *data, GROUP_AGGR *aggr, char *str) +{ + network_packet packet; + packet.data = data; + packet.offset = NET_HEADER_SIZE; + skip_field(&packet, aggr->pos); + network_mysqld_proto_get_column(&packet, str, MAX_COL_VALUE_LEN); + + return str; +} + +static char * +str_int_add(char *merged_int, char *s1, int len1, char *s2, int len2, int carry) +{ + char *large = s2; + int len = len1; + int max_len = len2; + + if (len1 > len2) { + len = len2; + max_len = len1; + large = s1; + } + + char *p = s1 + len1 - 1; + char *q = s2 + len2 - 1; + char *dest = merged_int + max_len; + int tmp = carry; + int i; + for (i = len - 1; i >= 0; i--) { + tmp += *p - '0' + *q - '0'; + *dest = tmp % 10 + '0'; + tmp = tmp / 10; + p--; + q--; + dest--; + } + + len = max_len - len; + if (len == 0) { + *dest = tmp + '0'; + } else { + large = large + len - 1; + for (i = len - 1; i >= 0; i--) { + tmp += *large - '0'; + *dest = tmp % 10 + '0'; + tmp = tmp / 10; + large--; + dest--; + } + + *dest = tmp + '0'; + } + + return merged_int; +} + +static char * +str_int_sub(char *merged_int, char *s1, int len1, char *s2, int len2, int borrow) +{ + char *p = s1 + len1 - 1; + char *q = s2 + len2 - 1; + char *dest = merged_int + len1 - 1; + int tmp = 0; + int i; + for (i = len2 - 1; i >= 0; i--) { + tmp = (*p - '0') - (*q - '0') - borrow; + if (tmp < 0) { + borrow = 1; + tmp = tmp + 10; + } else { + borrow = 0; + } + + *dest = tmp + '0'; + p--; + q--; + dest--; + } + + int len = len1 - len2; + if (len != 0) { + char *large = s1 + len - 1; + for (i = len - 1; i >= 0; i--) { + tmp = (*large - '0') - borrow; + if (tmp < 0) { + borrow = 1; + tmp = tmp + 10; + } else { + borrow = 0; + } + *dest = tmp + '0'; + large--; + dest--; + } + } + + return merged_int; +} + +static char * +str_decimal_add(char *merged_value, char *s1, int len1, char *s2, int len2) +{ + char *p, *q; + + p = strchr(s1, '.'); + q = strchr(s2, '.'); + + if (p != NULL && q != NULL) { + p[0] = '\0'; + q[0] = '\0'; + + int s1_int_len = p - s1; + int s2_int_len = q - s2; + + int int_len; + if (s1_int_len < s2_int_len) { + int_len = s2_int_len; + } else { + int_len = s1_int_len; + } + + char *short1 = p + 1; + char *short2 = q + 1; + char *merged_short = merged_value + int_len + 1; + + merged_short[0] = '.'; + merged_short++; + + int short_len = len1 - s1_int_len - 1; + + int i; + + int tmp = 0; + for (i = short_len - 1; i >= 0; i--) { + tmp += short1[i] - '0' + short2[i] - '0'; + merged_short[i] = tmp % 10 + '0'; + tmp = tmp / 10; + } + + str_int_add(merged_value, s1, s1_int_len, s2, s2_int_len, tmp); + + p[0] = '.'; + q[0] = '.'; + + } else { + str_int_add(merged_value, s1, len1, s2, len2, 0); + } + + return merged_value; +} + +static char * +str_decimal_sub(char *merged_value, char *s1, int len1, char *s2, int len2) +{ + char *p, *q; + + p = strchr(s1, '.'); + q = strchr(s2, '.'); + + if (p != NULL && q != NULL) { + p[0] = '\0'; + q[0] = '\0'; + + int s1_int_len = p - s1; + int s2_int_len = q - s2; + + int int_len; + if (s1_int_len < s2_int_len) { + int_len = s2_int_len; + } else { + int_len = s1_int_len; + } + + char *short1 = p + 1; + char *short2 = q + 1; + char *merged_short = merged_value + int_len; + + merged_short[0] = '.'; + merged_short++; + + int short_len = len1 - s1_int_len - 1; + + int i; + + int borrow = 0; + for (i = short_len - 1; i >= 0; i--) { + int tmp = (short1[i] - '0') - (short2[i] - '0') - borrow; + if (tmp < 0) { + borrow = 1; + tmp = tmp + 10; + } else { + borrow = 0; + } + merged_short[i] = tmp + '0'; + } + + str_int_sub(merged_value, s1, s1_int_len, s2, s2_int_len, borrow); + + p[0] = '.'; + q[0] = '.'; + + } else { + str_int_sub(merged_value, s1, len1, s2, len2, 0); + } + + return merged_value; +} + +static void trim_zero(char *s) +{ + char *p = s; + char *q = NULL; + int depth = 0; + while (p[0] == '0') { + if (p[1] >= '0' && p[1] <= '9') { + depth++; + p++; + if (p[0] == '.') { + q = p - 1; + } else if (p[0] != '0') { + q = p; + } + } else { + break; + } + } + + if (depth > 0) { + if (q == NULL) { + q = p; + } + } + + if (q != NULL) { + p = s; + while (q[0] != '\0') { + *p++ = *q++; + } + *p = '\0'; + } +} + +static int +dispose_sign_add(int is_integer, char *merged_value, + char *s1, int len1, char *s2, int len2, int *supported) +{ + if (s1[0] != '-' && s2[0] != '-') { + if (is_integer) { + str_int_add(merged_value, s1, len1, s2, len2, 0); + } else { + str_decimal_add(merged_value, s1, len1, s2, len2); + } + trim_zero(merged_value); + } else if (s1[0] == '-' && s2[0] == '-') { + if (is_integer) { + str_int_add(merged_value + 1, s1 + 1, len1 - 1, s2 + 1, len2 - 1, 0); + } else { + str_decimal_add(merged_value + 1, s1 + 1, len1 - 1, s2 + 1, len2 - 1); + } + trim_zero(merged_value + 1); + merged_value[0] = '-'; + } else if (s1[0] == '-') { + int result = cmp_str_num(s1 + 1, len1 - 1, s2, len2, supported); + if (result > 0) { + if (is_integer) { + str_int_sub(merged_value + 1, s1 + 1, len1 - 1, s2, len2, 0); + } else { + str_decimal_sub(merged_value + 1, s1 + 1, len1 - 1, s2, len2); + } + trim_zero(merged_value + 1); + merged_value[0] = '-'; + } else if (result < 0) { + if (is_integer) { + str_int_sub(merged_value, s2, len2, s1 + 1, len1 -1, 0); + } else { + str_decimal_sub(merged_value, s2, len2, s1 + 1, len1 -1); + } + trim_zero(merged_value); + } else { + merged_value[0] = '0'; + return 0; + } + } else if (s2[0] == '-') { + int result = cmp_str_num(s1, len1, s2 + 1, len2 - 1, supported); + if (result > 0) { + if (is_integer) { + str_int_sub(merged_value, s1 + 1, len1 - 1, s2, len2, 0); + } else { + str_decimal_sub(merged_value, s1, len1, s2 + 1, len2 - 1); + } + trim_zero(merged_value); + } else if (result < 0) { + if (is_integer) { + str_int_sub(merged_value + 1, s2 + 1, len2 - 1, s1, len1, 0); + } else { + str_decimal_sub(merged_value + 1, s2 + 1, len2 - 1, s1, len1); + } + trim_zero(merged_value + 1); + merged_value[0] = '-'; + } else { + merged_value[0] = '0'; + return 0; + } + } + + return 1; +} + +static char * +str_add(int type, char *merged_value, char *s1, int len1, char *s2, + int len2, int *merge_failed) +{ + int is_result_padding = 0, is_integer = 0, is_need_check = 0; + + switch (type) { + case FIELD_TYPE_TINY: + case FIELD_TYPE_SHORT: + case FIELD_TYPE_LONG: + case FIELD_TYPE_LONGLONG: + case FIELD_TYPE_INT24: + is_integer = 1; + break; + case FIELD_TYPE_DOUBLE: + case FIELD_TYPE_FLOAT: + is_need_check = 1; + break; + case FIELD_TYPE_NEWDECIMAL: + case FIELD_TYPE_DECIMAL: + is_result_padding = 1; + break; + default: + g_warning("%s: unknown type for add:%d", G_STRLOC, type); + *merge_failed = 1; + return NULL; + } + + if (!is_integer) { + if (is_need_check) { + if (!check_str_num_supported(s1, len1, NULL)) { + g_warning("%s: str num is not supported:%s", G_STRLOC, s1); + *merge_failed = 1; + return NULL; + } + if (!check_str_num_supported(s2, len2, NULL)) { + g_warning("%s: str num is not supported:%s", G_STRLOC, s2); + *merge_failed = 1; + return NULL; + } + } + + if (!padding_zero(s1, &len1, MAX_COL_VALUE_LEN, + s2, &len2, MAX_COL_VALUE_LEN)) + { + *merge_failed = 1; + return NULL; + } + } + + int result = dispose_sign_add(is_integer, merged_value, s1, + len1, s2, len2, merge_failed); + if (*merge_failed) { + return NULL; + } + + if ((!result) && is_result_padding) { + int merged_len = strlen(merged_value); + if (!padding_zero(merged_value, &merged_len, 2 * MAX_COL_VALUE_LEN, + s2, &len2, MAX_COL_VALUE_LEN)) + { + *merge_failed = 1; + return NULL; + } + } + + return merged_value; +} + + +static int merge_aggr_value(int fun_type, int type, char *merged_value, + char *str1, char *str2, int len1, int len2, int *merge_failed) +{ + if (len1 == 0 || len2 == 0) { + if (len1 == 0 && len2 == 0) { + return 0; + } else if (len1 == 0) { + strncpy(merged_value, str2, len2); + merged_value[len2] = '\0'; + return 1; + } else { + strncpy(merged_value, str1, len1); + merged_value[len1] = '\0'; + return 1; + } + } + + int is_str_type = 0; + switch (type) { + case FIELD_TYPE_TIME: + case FIELD_TYPE_TIMESTAMP: + case FIELD_TYPE_DATETIME: + case FIELD_TYPE_YEAR: + case FIELD_TYPE_NEWDATE: + case FIELD_TYPE_DATE: + case FIELD_TYPE_VAR_STRING: + case FIELD_TYPE_STRING: + is_str_type = 1; + break; + default: + break; + } + + switch (fun_type) { + case FT_SUM: + if (!str_add(type, merged_value, str1, len1, str2, len2, merge_failed)) { + return 0; + } + break; + case FT_MAX: + if (is_str_type) { + if (strcmp(str1, str2) >= 0) { + return 0; + } else { + strncpy(merged_value, str2, len2); + merged_value[len2] = '\0'; + } + } else { + if (cmp_str_num(str1, len1, str2, len2, merge_failed) > 0) { + return 0; + } else { + strncpy(merged_value, str2, len2); + merged_value[len2] = '\0'; + } + } + break; + case FT_MIN: + if (is_str_type) { + if (strcmp(str1, str2) <= 0) { + return 0; + } else { + strncpy(merged_value, str2, len2); + merged_value[len2] = '\0'; + } + } else { + if (cmp_str_num(str1, len1, str2, len2, merge_failed) <= 0) { + return 0; + } else { + strncpy(merged_value, str2, len2); + merged_value[len2] = '\0'; + } + } + break; + case FT_COUNT: + if (!str_add(type, merged_value, str1, len1, str2, + len2, merge_failed)) + { + return 0; + } + break; + } + + return 1; +} + + +static int modify_record(GList *cand1, GROUP_AGGR *aggr, + network_packet *packet1, network_packet *packet2, + int *orig_packet_len, int *merge_failed) +{ + char str1[MAX_COL_VALUE_LEN] = {0}; + char str2[MAX_COL_VALUE_LEN] = {0}; + char merged_value[2 * MAX_COL_VALUE_LEN] = {0}; + + char *before, *hit, *after; + + GString *pkt1 = packet1->data; + + before = (char *) (pkt1->str + packet1->offset); + skip_field(packet1, aggr->pos); + skip_field(packet2, aggr->pos); + + hit = (char *) (pkt1->str + packet1->offset); + network_mysqld_proto_get_column(packet1, str1, MAX_COL_VALUE_LEN); + network_mysqld_proto_get_column(packet2, str2, MAX_COL_VALUE_LEN); + + after = (char *) (pkt1->str + packet1->offset); + + char buffer[MAX_PACK_LEN] = {0}; + char *buf_pos = buffer; + int len = hit - before; + memcpy(buf_pos, before, len); + buf_pos = buf_pos + len + 1; + + int len1 = strlen(str1); + int len2 = strlen(str2); + if (!merge_aggr_value(aggr->fun_type, aggr->type, merged_value, + str1, str2, len1, len2, merge_failed)) + { + return 0; + } + + len = strlen(merged_value); + if (aggr->type == FIELD_TYPE_DOUBLE || aggr->type == FIELD_TYPE_FLOAT) { + char *p = strchr(merged_value, '.'); + if (p != NULL) { + int i = len - 1; + while (i >= 1 && merged_value[i] == '0') { + len--; + i--; + } + + if (merged_value[i] == '.') { + len--; + } + } + } + + + if (len == len1) { + memcpy(hit + 1, merged_value, len); + } else { + int packet_len = (*orig_packet_len) - len1 + len; + + if (packet_len > MAX_PACK_LEN) { + *merge_failed = 1; + return 0; + } + + strncpy(buf_pos, merged_value, len); + *(buf_pos - 1) = (char) len; + buf_pos = buf_pos + len; + memcpy(buf_pos, after, (*orig_packet_len) - (after - before)); + + *orig_packet_len = packet_len; + GString *packet = g_string_sized_new(NET_HEADER_SIZE + packet_len); + packet->len = NET_HEADER_SIZE; + g_string_append_len(packet, buffer, packet_len); + network_mysqld_proto_set_packet_len(packet, packet_len); + + g_string_free(cand1->data, TRUE); + cand1->data = packet; + packet1->data = packet; + } + + return 1; +} + +static gint combine_aggr_record(GList *cand1, GList *cand2, + aggr_by_group_para_t *para, int *merge_failed) +{ + int i; + network_packet packet1; + network_packet packet2; + packet1.data = cand1->data; + packet2.data = cand2->data; + + int orig_packet_len = network_mysqld_proto_get_packet_len(packet1.data); + + if (orig_packet_len >= MAX_PACK_LEN) { + g_warning("%s record too long for group by", G_STRLOC); + *merge_failed = 1; + return 0; + } + + short aggr_num = para->aggr_num; + + for (i = 0; i < aggr_num; i++) { + GROUP_AGGR *aggr = para->aggr_array + (aggr_num - 1 - i); + + packet1.offset = NET_HEADER_SIZE; + packet2.offset = NET_HEADER_SIZE; + + /* if equal, combined */ + switch (aggr->type) { + case FIELD_TYPE_TINY: + case FIELD_TYPE_SHORT: + case FIELD_TYPE_LONG: + case FIELD_TYPE_LONGLONG: + case FIELD_TYPE_INT24: + modify_record(cand1, aggr, &packet1, &packet2, + &orig_packet_len, merge_failed); + break; + case FIELD_TYPE_NEWDECIMAL: + case FIELD_TYPE_DECIMAL: + case FIELD_TYPE_FLOAT: + case FIELD_TYPE_DOUBLE: + modify_record(cand1, aggr, &packet1, &packet2, + &orig_packet_len, merge_failed); + break; + case FIELD_TYPE_TIME: + case FIELD_TYPE_TIMESTAMP: + case FIELD_TYPE_DATETIME: + case FIELD_TYPE_YEAR: + case FIELD_TYPE_NEWDATE: + case FIELD_TYPE_DATE: + case FIELD_TYPE_VAR_STRING: + case FIELD_TYPE_STRING: + switch (aggr->fun_type) { + case FT_MAX: + case FT_MIN: + break; + default: + g_warning("%s: string is not valid for aggr fun:%d", + G_STRLOC, aggr->fun_type); + *merge_failed = 1; + return 0; + } + + modify_record(cand1, aggr, &packet1, &packet2, + &orig_packet_len, merge_failed); + break; + + default: + *merge_failed = 1; + g_warning("%s:unknown Field Type: %d", G_STRLOC, + para->group_array[i].type); + return 1; + } + } + return 0; +} + + +static gint +cal_aggr_rec_rel(GList *cand1, GList *cand2, + aggr_by_group_para_t *para, int *merge_failed) +{ + int i, disp_flag; + char str1[MAX_COL_VALUE_LEN] = {0}; + char str2[MAX_COL_VALUE_LEN] = {0}; + network_packet packet1; + network_packet packet2; + packet1.data = cand1->data; + packet2.data = cand2->data; + + GROUP_BY *group_array = para->group_array; + + for (i = 0; i < para->group_array_size; i++) { + disp_flag = 0; + + packet1.offset = NET_HEADER_SIZE; + packet2.offset = NET_HEADER_SIZE; + + skip_field(&packet1, group_array[i].pos); + skip_field(&packet2, group_array[i].pos); + + network_mysqld_proto_get_column(&packet1, str1, MAX_COL_VALUE_LEN); + network_mysqld_proto_get_column(&packet2, str2, MAX_COL_VALUE_LEN); + + switch (group_array[i].type) { + case FIELD_TYPE_DATE: + if (!compare_date(str1, str2, NOR_REL, &disp_flag, + group_array[i].desc)) + { + return disp_flag; + } + break; + case FIELD_TYPE_TIME: + if (!compare_time(str1, str2, NOR_REL, &disp_flag, + group_array[i].desc)) + { + return disp_flag; + } + break; + case FIELD_TYPE_YEAR: + if (!compare_year(str1, str2, NOR_REL, &disp_flag, + group_array[i].desc)) + { + return disp_flag; + } + break; + case FIELD_TYPE_NEWDATE: + return 1; + /* case FIELD_TYPE_VARCHAR: */ + case FIELD_TYPE_VAR_STRING: + case FIELD_TYPE_TIMESTAMP: + case FIELD_TYPE_DATETIME: + case FIELD_TYPE_STRING: + if (!compare_records_from_column(str1, str2, + NOR_REL, group_array[i].desc, &disp_flag)) + { + return disp_flag; + } + + break; + case FIELD_TYPE_TINY: + case FIELD_TYPE_SHORT: + case FIELD_TYPE_LONG: + case FIELD_TYPE_LONGLONG: + case FIELD_TYPE_INT24: + case FIELD_TYPE_DOUBLE: + case FIELD_TYPE_NEWDECIMAL: + case FIELD_TYPE_DECIMAL: + case FIELD_TYPE_FLOAT: + if (!compare_str_num_value(str1, str2, NOR_REL, &disp_flag, + group_array[i].desc, merge_failed)) + { + return disp_flag; + } + break; + + default: + g_warning("%s:unknown Field Type: %d", G_STRLOC, group_array[i].type); + *merge_failed = 1; + return 1; + } + } + return combine_aggr_record(cand1, cand2, para, merge_failed); +} + + +static int heap_count = 0; + + +static int +compare_value_from_records(network_packet *packet1, network_packet *packet2, + ORDER_BY *ob, int type, int *result, int *compare_failed) +{ + char str1[MAX_COL_VALUE_LEN] = {0}; + char str2[MAX_COL_VALUE_LEN] = {0}; + + skip_field(packet1, ob->pos); + skip_field(packet2, ob->pos); + network_mysqld_proto_get_column(packet1, str1, MAX_COL_VALUE_LEN); + network_mysqld_proto_get_column(packet2, str2, MAX_COL_VALUE_LEN); + + switch(type) { + case FIELD_TYPE_LONG: + case FIELD_TYPE_DOUBLE: + return compare_str_num_value(str1, str2, PRIOR_TO, result, + ob->desc, compare_failed); + case FIELD_TYPE_DATE: + return compare_date(str1, str2, PRIOR_TO, result, ob->desc); + case FIELD_TYPE_TIME: + return compare_time(str1, str2, PRIOR_TO, result, ob->desc); + case FIELD_TYPE_YEAR: + return compare_year(str1, str2, PRIOR_TO, result, ob->desc); + } + return 1; +} + + +static int +compare_records_from_column(char *str1, char *str2, + int com_type, int desc, int *result) +{ + int ret = 0; + if (str1[0] == '\0' && str2[0] != '\0') { + ret = -1; + } else if (str1[0] != '\0' && str2[0] == '\0') { + ret = 1; + } else if (str1[0] != '\0' && str2[0] != '\0') { + ret = strcasecmp(str1, str2); + } else { + ret = 0; + } + + if (ret == 0) { + return 1; + } + + if ((desc && ret > 0) || (!desc && ret < 0)) { + *result = 1; + return 0; + } + + if ((desc && ret < 0) || (!desc && ret > 0)) { + if (com_type == PRIOR_TO) { + *result = 0; + } else { + *result = -1; + } + return 0; + } + + g_critical("%s: reach the unreachable place", G_STRLOC); + + return 0; +} + + +static int +compare_records_by_str(network_packet *packet1, network_packet *packet2, + order_by_para_t *para, int pkt1_index, int pkt2_index, int i, int *result) +{ + int j; + char str1[MAX_COL_VALUE_LEN] = {0}; + char str2[MAX_COL_VALUE_LEN] = {0}; + ORDER_BY *ob = &(para->order_array[i]); + + guint64 mask = 0xFFFF; + for (j = 0; j < i; j++) { + mask = mask << 16; + } + + uint64_t value1, value2; + if (para->field_index[pkt1_index]) { + value1 = para->field_index[pkt1_index]; + } else { + value1 = set_rec_fields_off(packet1, para->order_array, + para->order_array_size); + para->field_index[pkt1_index] = value1; + } + + guint64 expect_value = mask & value1; + + for (j = 0; j < i; j++) { + expect_value = expect_value >> 16; + } + + if (expect_value == 0) { + expect_value = NET_HEADER_SIZE; + } + packet1->offset = (guint) (intptr_t) expect_value; + + if (para->field_index[pkt2_index]) { + value2 = para->field_index[pkt2_index]; + } else { + value2 = set_rec_fields_off(packet2, para->order_array, + para->order_array_size); + para->field_index[pkt2_index] = value2; + } + + expect_value = mask & value2; + + for (j = 0; j < i; j++) { + expect_value = expect_value >> 16; + } + + if (expect_value == 0) { + expect_value = NET_HEADER_SIZE; + } + packet2->offset = (guint) (intptr_t) expect_value; + + network_mysqld_proto_get_column(packet1, str1, MAX_COL_VALUE_LEN); + network_mysqld_proto_get_column(packet2, str2, MAX_COL_VALUE_LEN); + + return compare_records_from_column(str1, str2, PRIOR_TO, ob->desc, result); +} + +/** + * is_prior_to Relation(record_A *record_B) defined ORDER BY + * return 1 if record A is prior to record B else 0 + */ +static gint is_prior_to(GString *pkt1, GString *pkt2, order_by_para_t *para, + int pkt1_index, int pkt2_index, int *is_record_equal, int *compare_failed) +{ + int i, equal_field_cnt, result; + network_packet packet1; + network_packet packet2; + + packet1.data = pkt1; + packet2.data = pkt2; + + equal_field_cnt = 0; + + g_debug("%s: call is_prior_to, index1:%d, index2:%d, count:%d, pkt1:%p, pkt2:%p", + G_STRLOC, pkt1_index, pkt2_index, ++heap_count, pkt1, pkt2); + + for (i = 0; i < para->order_array_size; i++) { + result = 0; + packet1.offset = NET_HEADER_SIZE; + packet2.offset = NET_HEADER_SIZE; + + ORDER_BY *order = &(para->order_array[i]); + + switch (order->type) { + case FIELD_TYPE_TINY: + case FIELD_TYPE_SHORT: + case FIELD_TYPE_LONG: + case FIELD_TYPE_LONGLONG: + case FIELD_TYPE_INT24: + if (!compare_value_from_records(&packet1, &packet2, order, + FIELD_TYPE_LONG, &result, compare_failed)) + { + return result; + } + equal_field_cnt++; + break; + case FIELD_TYPE_NEWDECIMAL: + case FIELD_TYPE_DECIMAL: + case FIELD_TYPE_FLOAT: + case FIELD_TYPE_DOUBLE: + if (!compare_value_from_records(&packet1, &packet2, order, + FIELD_TYPE_DOUBLE, &result, compare_failed)) + { + return result; + } + equal_field_cnt++; + break; + case FIELD_TYPE_DATE: + if (!compare_value_from_records(&packet1, &packet2, order, + FIELD_TYPE_DATE, &result, compare_failed)) + { + return result; + } + + equal_field_cnt++; + break; + case FIELD_TYPE_TIME: + if (!compare_value_from_records(&packet1, &packet2, order, + FIELD_TYPE_DATE, &result, compare_failed)) + { + return result; + } + equal_field_cnt++; + break; + case FIELD_TYPE_YEAR: + if (!compare_value_from_records(&packet1, &packet2, order, + FIELD_TYPE_DATE, &result, compare_failed)) + { + return result; + } + equal_field_cnt++; + break; + case FIELD_TYPE_NEWDATE: + return 1; + /* case FIELD_TYPE_VARCHAR: */ + case FIELD_TYPE_TIMESTAMP: + case FIELD_TYPE_DATETIME: + case FIELD_TYPE_VAR_STRING: + case FIELD_TYPE_STRING: + if (!compare_records_by_str(&packet1, &packet2, para, + pkt1_index, pkt2_index, i, &result)) + { + return result; + } + equal_field_cnt++; + break; + case FIELD_TYPE_NULL: + case FIELD_TYPE_BIT: + case FIELD_TYPE_ENUM: + case FIELD_TYPE_SET: + case FIELD_TYPE_TINY_BLOB: + case FIELD_TYPE_MEDIUM_BLOB: + case FIELD_TYPE_LONG_BLOB: + case FIELD_TYPE_BLOB: + case FIELD_TYPE_GEOMETRY: + return 1; + default: + *compare_failed = 1; + g_warning("%s:unknown Field Type: %d", G_STRLOC, order->type); + return 1; + } + } + + if (equal_field_cnt == para->order_array_size) { + if (is_record_equal) { + *is_record_equal = 1; + } + } + + return 1; +} + + +/* find index of field by name, the name might be an alias */ +static int cetus_result_find_fielddef(cetus_result_t *res, + const char *table, const char *field) +{ + int i; + for (i = 0; i < res->fielddefs->len; ++i) { + network_mysqld_proto_fielddef_t *fdef = g_ptr_array_index(res->fielddefs, i); + if (table && table[0] != '\0') { + if ((fdef->table && strcmp(table, fdef->table) == 0) + || (fdef->org_table && strcmp(table, fdef->org_table) == 0)) { + if ((fdef->name && strcmp(field, fdef->name) == 0) + || (fdef->org_name && strcmp(field, fdef->org_name) == 0)) + return i; + } + } else { + if ((fdef->name && strcmp(field, fdef->name) == 0) + || (fdef->org_name && strcmp(field, fdef->org_name) == 0)) + return i; + } + } + return -1; +} + +static gboolean cetus_result_parse_fielddefs(cetus_result_t *res_merge, GQueue *input) +{ + network_packet packet = {0}; + + res_merge->fielddefs = network_mysqld_proto_fielddefs_new(); + int i; + for (i = 0; i < res_merge->field_count; ++i) { + packet.data = g_queue_peek_nth(input, i + 1); + packet.offset = 0; + network_mysqld_proto_skip_network_header(&packet); + network_mysqld_proto_fielddef_t *fdef = network_mysqld_proto_fielddef_new(); + int err = network_mysqld_proto_get_fielddef(&packet, fdef, CLIENT_PROTOCOL_41); + if (err) { + network_mysqld_proto_fielddef_free(fdef); + network_mysqld_proto_fielddefs_free(res_merge->fielddefs); + res_merge->fielddefs = NULL; + return FALSE; + } + g_ptr_array_add(res_merge->fielddefs, fdef); + } + + return TRUE; +} + + +static gboolean cetus_result_retrieve_field_count(GQueue *input, guint64 *p_field_count) +{ + int packet_count = g_queue_get_length(input); + network_packet packet = {0}; + packet.data = g_queue_peek_head(input); /* Number-of-Field packet */ + int err = network_mysqld_proto_skip_network_header(&packet); + guint64 field_count; + err |= network_mysqld_proto_get_lenenc_int(&packet, &field_count); + if (err || field_count >= packet_count) { + return FALSE; + } + + *p_field_count = field_count; + + return TRUE; +} + +static gboolean cetus_result_parse_field_count(cetus_result_t *res_merge, GQueue *input) +{ + guint64 field_count = 0; + if (cetus_result_retrieve_field_count(input, &field_count) == TRUE) { + res_merge->field_count = field_count; + return TRUE; + } else { + return FALSE; + } +} + +/** + * Get order_array.pos, order_array.type + */ +static gboolean +get_order_by_fields(cetus_result_t *res_merge, ORDER_BY *order_array, + guint order_array_size, result_merge_t *merged_result) +{ + int i; + for (i = 0; i < order_array_size; ++i) { + ORDER_BY *orderby = &(order_array[i]); + + if (orderby->pos == -1) { + int index = cetus_result_find_fielddef(res_merge, + orderby->table_name, orderby->name); + if (index == -1) { + merged_result->status = RM_FAIL; + char msg[128] = {0}; + snprintf(msg, sizeof(msg), "order by:no %s in field list", orderby->name); + merged_result->detail = g_string_new(msg); + return FALSE; + } + orderby->pos = index; + } + network_mysqld_proto_fielddef_t *fdef = + g_ptr_array_index(res_merge->fielddefs, orderby->pos); + orderby->type = fdef->type; + } + return TRUE; +} + +static gboolean +get_group_by_fields(cetus_result_t *res_merge, GROUP_BY *group_array, guint group_array_size, + result_merge_t *merged_result) +{ + int i; + for (i = 0; i < group_array_size; ++i) { + GROUP_BY *groupby = &(group_array[i]); + if (groupby->pos == -1) { + int index = cetus_result_find_fielddef(res_merge, + groupby->table_name, groupby->name); + if (index == -1) { + merged_result->status = RM_FAIL; + char msg[128] = {0}; + snprintf(msg, sizeof(msg), "group by: no %s in field list", groupby->name); + merged_result->detail = g_string_new(msg); + return FALSE; + } + groupby->pos = index; + } + network_mysqld_proto_fielddef_t *fdef = + g_ptr_array_index(res_merge->fielddefs, groupby->pos); + groupby->type = fdef->type; + } + return TRUE; +} + +static gboolean +fulfill_condi(char *aggr_value, having_condition_t *hav_condi, result_merge_t *merged_result) +{ + int is_num = 0; + switch(hav_condi->data_type) { + case TK_INTEGER: + is_num = 1; + break; + case TK_FLOAT: + is_num = 1; + break; + default: + break; + } + + int len1 = strlen(aggr_value); + int len2 = strlen(hav_condi->condition_value); + int result; + + if (is_num) { + if (len2 == 0) { + return TRUE; + } + + if (len1 == 0) { + return FALSE; + } + + int num_unsupported = 0; + result = cmp_str_num(aggr_value, len1, hav_condi->condition_value, + len2, &num_unsupported); + if (num_unsupported) { + merged_result->status = RM_FAIL; + return FALSE; + } + + } else { + result = strcmp(aggr_value, hav_condi->condition_value); + } + + switch(hav_condi->rel_type) { + case TK_LE: + if (result <= 0) { + return TRUE; + } + break; + case TK_GE: + if (result >= 0) { + return TRUE; + } + break; + case TK_LT: + if (result < 0) { + return TRUE; + } + break; + case TK_GT: + if (result > 0) { + return TRUE; + } + break; + case TK_EQ: + if (result == 0) { + return TRUE; + } + break; + case TK_NE: + if (result != 0) { + return TRUE; + } + break; + } + + return FALSE; +} + + +static int aggr_by_group(aggr_by_group_para_t *para, + GList **candidates, guint *pkt_count, result_merge_t *merged_result) +{ + guint cand_index = 0; + GList *candidate = NULL; + size_t row_cnter = 0; + size_t off_pos = 0; + size_t iter; + + GPtrArray *recv_queues = para->recv_queues; + + while (row_cnter < para->limit->row_count) + { + if (!candidate || get_pkt_type((GString *) candidate->data) == MYSQLD_PACKET_EOF) + { + for (iter = 0; iter < recv_queues->len; iter++) { + GString *item = candidates[iter]->data; + if (item != NULL && + get_pkt_type(item) != MYSQLD_PACKET_EOF) + { + cand_index = iter; + candidate = candidates[iter]; + break; + } + } + } + /* if candidate is still NULL, all possible candidates have been exhausted */ + if (candidate == NULL || + get_pkt_type((GString *) candidate->data) == MYSQLD_PACKET_EOF) + { + break; + } + + /* to obtain candidate ptr and its index in recv_queues by scanning candidates once */ + for (iter = 0; iter < recv_queues->len; iter++) + { + /* don't compare with itself */ + if (iter == cand_index) { + continue; + } else { + GList *tmp_list = candidates[iter]; + /* some recv_queue may be shorter than others */ + if (tmp_list == NULL || + get_pkt_type((GString *) tmp_list->data) == MYSQLD_PACKET_EOF) + { + continue; + } + + /* 1, output; 0, wait; -1, change index */ + + g_debug("record1:%p, record2:%p", candidate, tmp_list); + int merge_failed = 0; + int result = cal_aggr_rec_rel(candidate, tmp_list, para, &merge_failed); + + if (merge_failed) { + merged_result->status = RM_FAIL; + return 0; + } + + if (result == -1) { + candidate = tmp_list; + cand_index = iter; + } else if (result == 0) { + candidates[iter] = tmp_list->next; + continue; + } + } + } + + g_debug("candidate:%p", candidate); + if (off_pos < para->limit->offset) + { + off_pos++; + candidates[cand_index] = candidate->next; + candidate = candidate->next; + continue; + } else { + char aggr_value[MAX_COL_VALUE_LEN] = {0}; + retrieve_aggr_value(candidate->data, para->aggr_array, aggr_value); + + if (!para->hav_condi->rel_type || + fulfill_condi(aggr_value, para->hav_condi, merged_result)) + { + ((GString *) candidate->data)->str[3] = (*pkt_count) + 1; + ++(*pkt_count); + row_cnter++; + network_queue_append(para->send_queue, (GString *) candidate->data); + } else { + g_string_free((GString *) candidate->data, TRUE); + } + + candidates[cand_index] = candidate->next; + GList *ptr_to_unlink = candidate; + candidate = candidate->next; + network_queue *recv_queue = recv_queues->pdata[cand_index]; + g_queue_delete_link(recv_queue->chunks, ptr_to_unlink); + } + } + + return 1; +} + +void heap_adjust(heap_type *heap, int s, int m, int *compare_failed) +{ + g_debug("%s: call heap_adjust, s:%d, m:%d", G_STRLOC, s, m); + + s = s - 1; + m = m - 1; + + heap_element *rc = heap->element[s]; + + int j, k, is_dup; + + for (j = (s << 1) + 1; j <= m; j = (j << 1) + 1) { + if (j < m) { + if (heap->element[j]->is_over && heap->element[j + 1]->is_over) { + g_debug("%s: in adjust, break here:%d", G_STRLOC, j); + break; + } + + if (heap->element[j]->is_over) { + j++; + } else if (!heap->element[j + 1]->is_over) { + if (!heap->element[j]->refreshed && !heap->element[j + 1]->refreshed) { + if (!heap->element[j]->is_prior_to) { + j++; + } else { + } + } else { + is_dup = 0; + k = j; + if (!is_prior_to(heap->element[j]->record->data, + heap->element[j + 1]->record->data, &(heap->order_para), + heap->element[j]->index, heap->element[j + 1]->index, + &is_dup, compare_failed)) + { + j++; + heap->element[j]->is_prior_to = 0; + + } else { + if (is_dup) { + heap->element[j + 1]->is_dup = 1; + } else { + } + + heap->element[j]->is_prior_to = 1; + } + + heap->element[k]->refreshed = 0; + heap->element[k + 1]->refreshed = 0; + } + } + } else { + if (heap->element[j]->is_over) { + g_debug("%s: call over here, break j:%d", G_STRLOC, j); + break; + } + } + + if (!rc->is_over) { + is_dup = 0; + if (is_prior_to(rc->record->data, heap->element[j]->record->data, + &(heap->order_para), rc->index, heap->element[j]->index, + &is_dup, compare_failed)) + { + if (is_dup) { + if (heap->element[j]->is_dup) { + heap->element[s]->is_dup = 1; + } else { + heap->element[j]->is_dup = 1; + } + } + + break; + } + } + + heap->element[s] = heap->element[j]; + heap->element[s]->refreshed = 1; + + s = j; + } + + heap->element[s] = rc; + heap->element[s]->refreshed = 1; +} + +static void check_server_sess_wait_for_event(network_mysqld_con *con, + int pmd_index, short ev_type, struct timeval *timeout) +{ + size_t i; + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + if (pmd_index >= 0) { + if (pmd_index != i) { + continue; + } + } + + if (!pmd->server->is_read_finished) { + if (pmd->server->is_waiting) { + g_debug("%s: pmd %d is waiting", G_STRLOC, (int) i); + continue; + } + con->num_read_pending++; + pmd->read_cal_flag = 0; + g_debug("%s: pmd %d is not read finished, read pending:%d, fd:%d, pmd index:%d", + G_STRLOC, (int) i, con->num_read_pending, pmd->server->fd, pmd->index); + event_set(&(pmd->server->event), pmd->server->fd, ev_type, + server_session_con_handler, pmd); + chassis_event_add_with_timeout(con->srv, &(pmd->server->event), timeout); + g_debug("%s: call chassis_event_add_with_timeout", G_STRLOC); + pmd->server->is_waiting = 1; + } else { + g_debug("%s: pmd %d is read finished", G_STRLOC, (int) i); + } + } +} + +static int check_after_limit(network_mysqld_con *con, merge_parameters_t *data, + int is_finished) +{ + GPtrArray *recv_queues = data->recv_queues; + GList **candidates = data->candidates; + gboolean is_more_to_read = FALSE; + GList *candidate = NULL; + size_t iter; + + g_debug("%s: call check_after_limit", G_STRLOC); + + for (iter = 0; iter < recv_queues->len; iter++) { + gboolean is_over = FALSE; + do { + candidate = candidates[iter]; + if (candidate == NULL || candidate->data == NULL) { + con->partially_merged = 1; + is_more_to_read = TRUE; + g_debug("%s: item is nil, index:%d", G_STRLOC, (int) iter); + break; + } + + GString *item = candidate->data; + guchar pkt_type = get_pkt_type(item); + if (pkt_type == MYSQLD_PACKET_EOF || pkt_type == MYSQLD_PACKET_ERR) { + g_debug("%s: is over true:%d, pkt type:%d", G_STRLOC, + (int) iter, (int) pkt_type); + is_over = TRUE; + break; + } + + candidates[iter] = candidate->next; + g_debug("%s: free packet addr:%p, iter:%d, pkt_type:%d", G_STRLOC, + candidate->data, (int) iter, (int) pkt_type); + g_string_free((GString *) candidate->data, TRUE); + network_queue *recv_queue = recv_queues->pdata[iter]; + g_queue_delete_link(recv_queue->chunks, candidate); + } while(!is_over); + } + + if (is_finished && is_more_to_read) { + g_warning("%s: finished reading, but needs more read:%p", + G_STRLOC, con); + } else if ((!is_finished) && (!is_more_to_read)) { + g_warning("%s: not finished reading, but is_more_to_read false:%p", + G_STRLOC, con); + } + + if (is_more_to_read) { + g_debug("%s: need more reading for:%p", G_STRLOC, con); + check_server_sess_wait_for_event(con, -1, EV_READ, &con->read_timeout); + return 0; + } + + return 1; +} + + +static int do_simple_merge(network_mysqld_con *con, merge_parameters_t *data, int is_finished) +{ + network_queue *send_queue = data->send_queue; + GPtrArray *recv_queues = data->recv_queues; + GList **candidates = data->candidates; + LIMIT *limit = &(data->limit); + int *row_cnter = &(data->row_cnter); + int *off_pos = &(data->off_pos); + + GList *candidate = NULL; + size_t iter; + + heap_count = 0; + + int merged_output_size = con->srv->merged_output_size; + if (con->is_client_compressed) { + merged_output_size = con->srv->compressed_merged_output_size; + } + + g_debug("%s: call do_simple_merge", G_STRLOC); + + gboolean shortaged = FALSE; + for (iter = 0; iter < recv_queues->len; iter++) { + candidate = candidates[iter]; + network_queue *recv_queue = recv_queues->pdata[iter]; + + g_debug("%s: analysis packets:%d", G_STRLOC, (int) iter); + + while (candidate != NULL) + { + if (candidate->data == NULL) { + g_debug("%s: candidate data is nil:%d", G_STRLOC, (int) iter); + break; + } + + if ((*row_cnter) == limit->row_count) { + g_debug("%s: reach limit:%d", G_STRLOC, (int) iter); + break; + } + + guchar pkt_type = get_pkt_type((GString *) candidate->data); + if (pkt_type == MYSQLD_PACKET_EOF) { + g_debug("%s: MYSQLD_PACKET_EOF here:%d", G_STRLOC, (int) iter); + break; + } + + if (pkt_type == MYSQLD_PACKET_ERR) { + data->is_pack_err = 1; + data->err_pack = candidate->data; + g_debug("%s: MYSQLD_PACKET_ERR here:%d", G_STRLOC, (int) iter); + break; + } + + if ((*off_pos) < limit->offset) { + (*off_pos) ++; + g_string_free((GString *) candidate->data, TRUE); + } else { + + int packet_len = network_mysqld_proto_get_packet_len(candidate->data); + data->aggr_output_len += packet_len; + + ((GString *) candidate->data)->str[3] = data->pkt_count + 1; + ++(data->pkt_count); + network_queue_append(send_queue, (GString *) candidate->data); + if (data->aggr_output_len >= merged_output_size) { + g_debug("%s: send_part_content_to_client:%d, iter:%d", + G_STRLOC, data->aggr_output_len, (int) iter); + send_part_content_to_client(con); + data->aggr_output_len = 0; + } + (*row_cnter)++; + } + + candidates[iter] = candidate->next; + g_queue_delete_link(recv_queue->chunks, candidate); + candidate = candidates[iter]; + } + + if (candidate == NULL || candidate->data == NULL) { + server_session_t *pmd = g_ptr_array_index(con->servers, iter); + if (pmd->server->is_waiting) { + g_debug("%s: is_waiting true:%d", G_STRLOC, (int) iter); + continue; + } + candidates[iter] = NULL; + g_debug("%s: candidate is nil for i:%d, recv_queues:%p", + G_STRLOC, (int) iter, recv_queues); + shortaged = TRUE; + } + } + + if (shortaged) { + con->partially_merged = 1; + g_debug("%s: need more reading for:%p", G_STRLOC, con); + if (data->aggr_output_len >= merged_output_size) { + send_part_content_to_client(con); + data->aggr_output_len = 0; + } + check_server_sess_wait_for_event(con, -1, EV_READ, &con->read_timeout); + return 0; + } else { + if (!is_finished) { + g_debug("%s: check limit for:%p", G_STRLOC, con); + if (limit->row_count > 0 && (*row_cnter) >= limit->row_count) { + g_debug("%s: do call check_after_limit for:%p", G_STRLOC, con); + if (check_after_limit(con, data, is_finished) == FALSE) { + g_debug("%s: call check_after_limit over for:%p", G_STRLOC, con); + return 0; + } + } + } + } + + return 1; +} + + + +static int do_sort_merge(network_mysqld_con *con, merge_parameters_t *data, + int is_finished, int *compare_failed) +{ + network_queue *send_queue = data->send_queue; + GPtrArray *recv_queues = data->recv_queues; + GList **candidates = data->candidates; + LIMIT *limit = &(data->limit); + heap_type *heap = data->heap; + int *row_cnter = &(data->row_cnter); + int *off_pos = &(data->off_pos); + uint64_t *field_index = heap->order_para.field_index; + + GList *candidate = NULL; + + int merged_output_size = con->srv->merged_output_size; + if (con->is_client_compressed) { + merged_output_size = con->srv->compressed_merged_output_size; + } + int last_output_index = -1; + while ((*row_cnter) < limit->row_count) + { + if (heap->element[0]->is_over) { + if (heap->is_err) { + data->is_pack_err = 1; + } + break; + } + + int cand_index = heap->element[0]->index; + + if (heap->element[0]->record == NULL) { + if (candidates[cand_index] == NULL) { + g_debug("%s: cand_index:%d is nil", G_STRLOC, cand_index); + return 0; + } else { + heap->element[0]->record = candidates[cand_index]; + g_debug("%s: record:%p for index:%d", G_STRLOC, heap->element[0]->record, cand_index); + heap_adjust(heap, 1, recv_queues->len, compare_failed); + if (*compare_failed) { + return 0; + } + cand_index = heap->element[0]->index; + } + } + + candidate = heap->element[0]->record; + field_index[cand_index] = 0; + + g_debug("%s: row counter:%d", G_STRLOC, (int) (*row_cnter)); + + if (data->is_distinct && heap->element[0]->is_dup && + cand_index != last_output_index) + { + g_debug("%s: dup element at:%d", G_STRLOC, cand_index); + g_string_free((GString *) candidate->data, TRUE); + } else if ((*off_pos) < limit->offset) { + (*off_pos)++; + g_string_free((GString *) candidate->data, TRUE); + g_debug("%s: off pos here:%d", G_STRLOC, (int) (*off_pos)); + } else { + + int packet_len = network_mysqld_proto_get_packet_len(candidate->data); + data->aggr_output_len += packet_len; + ((GString *) candidate->data)->str[3] = data->pkt_count + 1; + ++(data->pkt_count); + network_queue_append(send_queue, (GString *) candidate->data); + (*row_cnter)++; + last_output_index = cand_index; + + if (data->aggr_output_len >= merged_output_size) { + g_debug("%s: send_part_content_to_client:%d", + G_STRLOC, data->aggr_output_len); + send_part_content_to_client(con); + data->aggr_output_len = 0; + } + } + + heap->element[0]->is_dup = 0; + heap->element[0]->is_err = 0; + heap->element[0]->refreshed = 0; + heap->element[0]->is_prior_to = 0; + heap->element[0]->record = heap->element[0]->record->next; + + candidates[cand_index] = candidate->next; + network_queue *recv_queue = recv_queues->pdata[cand_index]; + g_debug("%s: remove candidate:%p for queue:%p, pmd:%d", + G_STRLOC, candidate, recv_queue, cand_index); + g_queue_delete_link(recv_queue->chunks, candidate); + + if (candidates[cand_index] == NULL || candidates[cand_index]->data == NULL) { + con->partially_merged = 1; + g_debug("%s: item is nil, index:%d", G_STRLOC, cand_index); + if (data->aggr_output_len >= merged_output_size) { + send_part_content_to_client(con); + g_debug("%s: send_part_content_to_client:%d", + G_STRLOC, data->aggr_output_len); + data->aggr_output_len = 0; + } + check_server_sess_wait_for_event(con, cand_index, EV_READ, &con->read_timeout); + return 0; + } + + GString *item = candidates[cand_index]->data; + guchar pkt_type = get_pkt_type(item); + + if (pkt_type == MYSQLD_PACKET_EOF || pkt_type == MYSQLD_PACKET_ERR) { + g_debug("%s: index is over:%d", G_STRLOC, cand_index); + heap->element[0]->is_over = 1; + if (pkt_type == MYSQLD_PACKET_ERR) { + heap->element[0]->is_err = 1; + heap->is_err = 1; + data->err_pack = item; + } + } else { + heap->element[0]->is_over = 0; + } + + g_debug("%s: candidate:%p for queue:%p, pmd:%d", + G_STRLOC, candidates[cand_index], recv_queue, cand_index); + + heap_adjust(heap, 1, recv_queues->len, compare_failed); + if (*compare_failed) { + return 0; + } + } + + if (limit->row_count > 0 && (*row_cnter) >= limit->row_count) { + if (check_after_limit(con, data, is_finished) == FALSE) { + return 0; + } + } + + return 1; +} + + +static int +create_heap_for_merge_sort(network_mysqld_con *con, merge_parameters_t *data, int *compare_failed) +{ + GList **candidates = data->candidates; + heap_type *heap = data->heap; + size_t iter; + + for (iter = 0; iter < heap->len; iter++) { + heap->element[iter]->is_dup = 0; + heap->element[iter]->is_err = 0; + heap->element[iter]->refreshed = 0; + heap->element[iter]->is_prior_to = 0; + + if (candidates[iter] != NULL) { + GString *item = candidates[iter]->data; + if (item != NULL) { + guchar pkt_type = get_pkt_type(item); + if (pkt_type == MYSQLD_PACKET_EOF || pkt_type == MYSQLD_PACKET_ERR) { + heap->element[iter]->is_over = 1; + if (pkt_type == MYSQLD_PACKET_ERR) { + heap->element[iter]->is_err = 1; + heap->is_err = 1; + data->err_pack = item; + } + } else { + heap->element[iter]->record = candidates[iter]; + heap->element[iter]->index = iter; + heap->element[iter]->is_over = 0; + } + } else { + con->partially_merged = 1; + check_server_sess_wait_for_event(con, iter, EV_READ, &con->read_timeout); + return 0; + } + } + } + + for (iter = heap->len / 2; iter > 0; --iter) { + heap_adjust(heap, iter, heap->len, compare_failed); + if (*compare_failed) { + return 0; + } + } + + g_debug("%s: create heap over", G_STRLOC); + + return 1; +} + +int callback_merge(network_mysqld_con *con, merge_parameters_t *data, int is_finished) +{ + int merge_failed = 0; + network_queue *send_queue = data->send_queue; + heap_type *heap = data->heap; + int order_array_size = heap->order_para.order_array_size; + + if (order_array_size > 0) { + int result = do_sort_merge(con, data, is_finished, &merge_failed); + if (merge_failed) { + return RM_FAIL; + } + if (!result) { + return RM_SUCCESS; + } + } else { + if (!do_simple_merge(con, data, is_finished)) { + return RM_SUCCESS; + } + } + + if (is_finished) { + g_debug("%s: finished is true", G_STRLOC); + if (data->is_pack_err) { + if (data->err_pack == NULL) { + g_warning("%s: packet err while p is nil", G_STRLOC); + return RM_FAIL; + } + data->pack_err_met = 1; + data->err_pack->str[3] = data->pkt_count + 1; + ++(data->pkt_count); + guchar pkt_type = get_pkt_type(data->err_pack); + if (pkt_type == MYSQLD_PACKET_ERR) { + g_message("%s: Add err packet to send queue", G_STRLOC); + } else { + g_warning("%s: Add non err packet info to send queue when err pack is met", G_STRLOC); + } + GString *new_err_pack = g_string_new_len(data->err_pack->str, data->err_pack->len); + network_queue_append(send_queue, new_err_pack); + } else { + /* TODO if in trans, then needs to set 'in transaction' flag ? */ + GString *eof_pkt = g_string_new_len("\x05\x00\x00\x07\xfe\x00\x00\x02\x00", 9); + eof_pkt->str[3] = (data->pkt_count) + 1; + network_queue_append(send_queue, eof_pkt); + } + } + g_debug("%s: send queue len:%d", G_STRLOC, send_queue->chunks->length); + + return RM_SUCCESS; +} + + +static int +do_merge(network_mysqld_con *con, merge_parameters_t *data, int *merge_failed) { + heap_type *heap = data->heap; + int is_finished = 0; + + if (con->num_pending_servers == 0) { + is_finished = 1; + } + + if (heap->order_para.order_array_size > 0) { + if (!create_heap_for_merge_sort(con, data, merge_failed)) { + return 1; + } + + if (!do_sort_merge(con, data, is_finished, merge_failed)) { + return 1; + } + } else { + if (!do_simple_merge(con, data, is_finished)) { + return 1; + } + } + + if (data->is_pack_err) { + if (data->err_pack == NULL) { + g_warning("%s: packet err while p is nil", G_STRLOC); + return 0; + } + data->pack_err_met = 1; + data->err_pack->str[3] = data->pkt_count + 1; + ++(data->pkt_count); + guchar pkt_type = get_pkt_type(data->err_pack); + if (pkt_type == MYSQLD_PACKET_ERR) { + g_message("%s: Add err packet to send queue", G_STRLOC); + } else { + g_warning("%s: Add non err packet info to send queue when err pack is met", G_STRLOC); + } + GString *new_err_pack = g_string_new_len(data->err_pack->str, data->err_pack->len); + network_queue_append(data->send_queue, new_err_pack); + } + + return 1; +} + + +gint check_dist_tran_resultset(network_queue *recv_queue, network_mysqld_con *con) +{ + int fail = 0; + GList *pkt = recv_queue->chunks->head; + /* only check the first packet in each recv_queue */ + if (pkt != NULL && pkt->data != NULL && ((GString *) pkt->data)->len > NET_HEADER_SIZE) + { + guchar pkt_type = get_pkt_type((GString *) pkt->data); + g_debug("%s: pkt type:%d", G_STRLOC, pkt_type); + if (pkt_type == MYSQLD_PACKET_ERR) { + network_packet packet; + packet.offset = NET_HEADER_SIZE; + packet.data = pkt->data; + network_mysqld_err_packet_t *err_packet; + err_packet = network_mysqld_err_packet_new(); + if (!network_mysqld_proto_get_err_packet(&packet, err_packet)) { + int checked = 0; + switch (err_packet->errcode) { + case ER_XA_RBROLLBACK: + g_message("%s: ER_XA_RBROLLBACK for con:%p", G_STRLOC, con); + fail = 1; + checked = 1; + break; + case ER_XA_RBDEADLOCK: + g_message("%s: ER_XA_RBDEADLOCK for con:%p", G_STRLOC, con); + fail = 1; + checked = 1; + break; + case ER_XA_RBTIMEOUT: + g_message("%s: ER_XA_RBTIMEOUT for con:%p", G_STRLOC, con); + fail = 1; + checked = 1; + break; + case ER_LOCK_DEADLOCK: + case ER_LOCK_WAIT_TIMEOUT: + fail = 1; + checked = 1; + break; + case ER_DUP_ENTRY: + g_message("%s: ER_DUP_ENTRY here:%d", G_STRLOC, + con->last_resp_num); + if (con->last_resp_num > 1) { + fail = 1; + } else { + fail = 0; + } + checked = 1; + break; + default: + fail = 0; + break; + } + if (!checked) { + if (strncasecmp(err_packet->sqlstate->str, "XA", 2) == 0) { + fail = 1; + } + } + } else { + g_warning("%s: network_mysqld_proto_get_err_packet err", G_STRLOC); + } + network_mysqld_err_packet_free(err_packet); + } else if (pkt_type == MYSQLD_PACKET_EOF) { + fail = 1; + } + } else { + fail = 1; + } + + if (fail) { + return -1; + } + + return 0; +} + +static int log_packet_error_info(network_socket *client, network_socket *server, char *orig_sql, + GString *packet_str, uint64_t uniq_id) +{ + network_mysqld_err_packet_t *err_packet; + network_packet packet; + + packet.data = packet_str; + packet.offset = NET_HEADER_SIZE; + + err_packet = network_mysqld_err_packet_new(); + + if (network_mysqld_proto_get_err_packet(&packet, err_packet)) { + g_message("%s:clt:%s,src:%s,dst:%s,db:%s,%s", + G_STRLOC, client->src->name->str, server->src->name->str, + server->dst->name->str, server->default_db->str, orig_sql); + network_mysqld_err_packet_free(err_packet); + return -1; + } + + g_message("%s: id:%llu,clt:%s,src:%s,dst:%s,db:%s,%s, error code:%d, errmsg:%s, sqlstate:%s", + G_STRLOC, (unsigned long long) uniq_id, client->src->name->str, server->src->name->str, + server->dst->name->str, server->default_db->str, orig_sql, (int) err_packet->errcode, + err_packet->errmsg->str, err_packet->sqlstate->str); + network_mysqld_err_packet_free(err_packet); + return 0; +} + +static int +merge_for_modify(sql_context_t *context, network_queue *send_queue, GPtrArray *recv_queues, + network_mysqld_con *con, cetus_result_t *res_merge, result_merge_t *merged_result) +{ + /* INSERT/UPDATE/DELETE expecting OK packet */ + int total_affected_rows = 0; + int total_warnings = 0; + int i; + + for (i = 0; i < recv_queues->len; i++) { + network_queue *recv_q = g_ptr_array_index(recv_queues, i); + GString *pkt = g_queue_peek_head(recv_q->chunks); + /* only check the first packet in each recv_queue */ + if (!pkt || pkt->len <= NET_HEADER_SIZE) { + cetus_result_destroy(res_merge); + merged_result->status = RM_FAIL; + return 0; + } + + guchar pkt_type = get_pkt_type(pkt); + switch (pkt_type) { + case MYSQLD_PACKET_OK: { + network_packet packet = {pkt, 0}; + network_mysqld_ok_packet_t one_ok; + network_mysqld_proto_skip_network_header(&packet); + if (!network_mysqld_proto_get_ok_packet(&packet, &one_ok)) { + total_affected_rows += one_ok.affected_rows; + total_warnings += one_ok.warnings; + } + break; + } + case MYSQLD_PACKET_EOF: + break; + case MYSQLD_PACKET_ERR: + network_queue_append(send_queue, pkt); + g_queue_remove(recv_q->chunks, pkt); + if (context->stmt_type == STMT_CALL) { + g_message("%s: stored procedure failed", G_STRLOC); + cetus_result_destroy(res_merge); + merged_result->status = RM_CALL_FAIL; + return 0; + } + cetus_result_destroy(res_merge); + merged_result->status = RM_FAIL; + return 0; + default: + break; + } + } + if (con->sharding_plan->table_type == GLOBAL_TABLE && recv_queues->len) { + total_affected_rows /= recv_queues->len; + } + + network_mysqld_con_send_ok_full(con->client, total_affected_rows, + 0, 0x02, total_warnings); + + return 1; +} + + +static gboolean +disp_orderby_info(sql_column_list_t *sel_orderby, cetus_result_t *res_merge, + ORDER_BY *order_array, int order_array_size, result_merge_t *merged_result) +{ + int i; + + for (i = 0; i < sel_orderby->len; ++i) { + sql_column_t *col = g_ptr_array_index(sel_orderby, i); + sql_expr_t *expr = col->expr; + ORDER_BY *ord_col = &(order_array[i]); + ord_col->pos = -1; /* initial invalid value: -1 */ + if (expr->op == TK_ID) { + strncpy(ord_col->name, expr->token_text, MAX_NAME_LEN - 1); + } else if (expr->op == TK_INTEGER) { + gint64 v; + sql_expr_get_int(expr, &v); + ord_col->pos = (v > 0 && v <= res_merge->field_count) ? (int)v-1 : -1; + } else if (expr->op == TK_FUNCTION) { + strncpy(ord_col->name, expr->start, expr->end - expr->start); + } else if (expr->op == TK_DOT) { + strncpy(ord_col->table_name, expr->left->token_text, MAX_NAME_LEN - 1); + strncpy(ord_col->name, expr->right->token_text, MAX_NAME_LEN - 1); + } else { + g_warning("order by name error"); + } + ord_col->desc = (col->sort_order == SQL_SO_DESC); + } + + g_debug("%s: merge reach here", G_STRLOC); + + if (!get_order_by_fields(res_merge, order_array, order_array_size, merged_result)) { + cetus_result_destroy(res_merge); + return FALSE; + } + + return TRUE; +} + +static gboolean +disp_groupby_info(sql_column_list_t *sel_groupby, cetus_result_t *res_merge, + GROUP_BY *group_array, int group_array_size, result_merge_t *merged_result) +{ + int i; + for (i = 0; i < sel_groupby->len; ++i) { + sql_expr_t *expr = g_ptr_array_index(sel_groupby, i); + GROUP_BY *group_col = &(group_array[i]); + group_col->pos = -1; /* initial invalid value: -1 */ + if (expr->op == TK_ID) {/* TODO: wrap sql_expr_t in list */ + strncpy(group_col->name, expr->token_text, MAX_NAME_LEN - 1); + } else if (expr->op == TK_DOT) { + strncpy(group_col->table_name, expr->left->token_text, MAX_NAME_LEN - 1); + strncpy(group_col->name, expr->right->token_text, MAX_NAME_LEN - 1); + } else if (expr->op == TK_INTEGER) { + gint64 v; + sql_expr_get_int(expr, &v); + group_col->pos = (v > 0 && v <= res_merge->field_count) ? (int)v-1 : -1; + } else if (expr->op == TK_FUNCTION) { + strncpy(group_col->name, expr->start, expr->end - expr->start); + } else { + g_warning("group by name error"); + } + } + + if (!get_group_by_fields(res_merge, group_array, group_array_size, merged_result)) { + cetus_result_destroy(res_merge); + return FALSE; + } + + return TRUE; +} + +static int +check_network_packet_err(GList **candidates, GPtrArray *recv_queues, + network_queue *send_queue, cetus_result_t *res_merge, result_merge_t *merged_result) +{ + int i; + for (i = 0; i < recv_queues->len; i++) { + GString *pkt = candidates[i]->data; + /* only check the first packet in incoming row packets */ + if (pkt != NULL && pkt->len > NET_HEADER_SIZE) { + guchar pkt_type = get_pkt_type(pkt); + if (pkt_type == MYSQLD_PACKET_ERR) { + network_queue_append(send_queue, pkt); + network_queue *recv_queue = g_ptr_array_index(recv_queues, i); + g_queue_remove(recv_queue->chunks, pkt); + cetus_result_destroy(res_merge); + merged_result->status = RM_SUCCESS; + return 0; + } + } else { + cetus_result_destroy(res_merge); + merged_result->status = RM_FAIL; + return 0; + } + } + + return 1; +} + +static int +check_field_count_consistant(GPtrArray *recv_queues, result_merge_t *merged_result, guint64 *field_count) +{ + int i; + guint64 last_field_count = 0; + for (i = 0; i < recv_queues->len; i++) { + network_queue *queue = g_ptr_array_index(recv_queues, i); + if (cetus_result_retrieve_field_count(queue->chunks, field_count) == FALSE) { + g_warning("%s:cetus_result_retrieve_field_count failure", G_STRLOC); + merged_result->status = RM_FAIL; + return 0; + } + + if (last_field_count) { + if (last_field_count != (*field_count)) { + g_warning("%s:field count different, field_count1:%d, field_count2:%d", + G_STRLOC, (int) last_field_count, (int) (*field_count)); + merged_result->status = RM_FAIL; + return 0; + } + } + + last_field_count = *field_count; + } + + return 1; +} + +static int +prepare_for_row_process(GList **candidates, GPtrArray *recv_queues, network_queue *send_queue, + guint pkt_count, result_merge_t *merged_result) +{ + int i, candidate_iter = 0; + + /* insert 'ROW packets' to candidates */ + for (i = 0; i < recv_queues->len; i++) { + network_queue *recv_q = g_ptr_array_index(recv_queues, i); + GList *rows_start = NULL; + + g_debug("%s: pkt_count:%d", G_STRLOC, pkt_count); + + int j; + /* sending result header, field-defs and EOF once */ + for (j = pkt_count; j > 0; --j) { + GString *packet = g_queue_pop_head(recv_q->chunks); + if (packet == NULL) { + g_warning("%s:packet null, enlarge max_header_size, pkt cnt:%d", + G_STRLOC, pkt_count); + merged_result->status = RM_FAIL; + return 0; + } + + if (i == 0) { + network_queue_append(send_queue, packet); + } else { + g_string_free(packet, TRUE); + } + + /* check if the last packet is field EOF packet */ + } + + rows_start = g_queue_peek_head_link(recv_q->chunks); + + if (rows_start) { + candidates[candidate_iter] = rows_start; + candidate_iter++; + } else { + g_warning("%s:rows start null, enlarge max_header_size, pkt cnt:%d", + G_STRLOC, pkt_count); + merged_result->status = RM_FAIL; + return 0; + } + } + + return 1; +} + + +static int +merge_for_show_warnings(network_queue *send_queue, GPtrArray *recv_queues, + network_mysqld_con *con, cetus_result_t *res_merge, result_merge_t *merged_result) +{ + guint64 field_count = 0; + if (!check_field_count_consistant(recv_queues, merged_result, &field_count)) { + return 0; + } + res_merge->field_count = field_count; + + GList **candidates = g_new0(GList *, recv_queues->len); + /* field-count-packet + eof-packet */ + guint pkt_count = res_merge->field_count + 2; + + if (!prepare_for_row_process(candidates, recv_queues, + send_queue, pkt_count, merged_result)) + { + g_warning("%s:prepare_for_row_process failed", G_STRLOC); + g_free(candidates); + return 0; + } + + if (!check_network_packet_err(candidates, recv_queues, + send_queue, res_merge, merged_result)) + { + g_warning("%s:packet err is met", G_STRLOC); + g_free(candidates); + return 0; + } + + merge_parameters_t *data = g_new0(merge_parameters_t, 1); + + data->send_queue = send_queue; + data->recv_queues = recv_queues; + data->candidates = candidates; + data->pkt_count = pkt_count; + data->limit.offset = 0; + data->limit.row_count = G_MAXINT32; + + data->pack_err_met = 0; + + con->data = data; + + if (!do_simple_merge(con, con->data, 1)) { + g_warning("%s:merge failed", G_STRLOC); + merged_result->status = RM_FAIL; + return 0; + } + + pkt_count = data->pkt_count; + + /* after adding all packets we don't need candidate list anymore */ + + /* need to append EOF after all Row Data Packets?? Yes + * update packet number in header + */ + + g_debug("%s: append here", G_STRLOC); + /* TODO if in trans, then needs to set 'in transaction' flag ? */ + GString *eof_pkt = g_string_new_len("\x05\x00\x00\x07\xfe\x00\x00\x02\x00", 9); + eof_pkt->str[3] = pkt_count + 1; + network_queue_append(send_queue, eof_pkt); + + return 1; +} + +static int +merge_for_select(sql_context_t *context, network_queue *send_queue, GPtrArray *recv_queues, + network_mysqld_con *con, cetus_result_t *res_merge, result_merge_t *merged_result) +{ + sql_select_t *select = (sql_select_t *)context->sql_statement; + + guint64 field_count = 0; + if (!check_field_count_consistant(recv_queues, merged_result, &field_count)) { + return 0; + } + res_merge->field_count = field_count; + + GROUP_AGGR aggr_array[MAX_AGGR_FUNS] = {{0}}; + int aggr_num = sql_expr_list_find_aggregates(select->columns, aggr_array); + sql_column_list_t *sel_orderby = select->orderby_clause; + sql_expr_list_t *sel_groupby = select->groupby_clause; + if (sel_orderby || sel_groupby || aggr_num > 0) { + network_queue *first_queue = g_ptr_array_index(recv_queues, 0); + gboolean ok = cetus_result_parse_fielddefs(res_merge, first_queue->chunks); + if (!ok) { + g_warning("%s:parse_fielddefs failed:%s", G_STRLOC, con->orig_sql->str); + merged_result->status = RM_FAIL; + return 0; + } + } + + ORDER_BY order_array[MAX_ORDER_COLS]; + int order_array_size = 0; /* number of ORDER_BY Columns */ + if (sel_orderby && sel_orderby->len > 0) { + memset(order_array, 0, sizeof(ORDER_BY) * MAX_ORDER_COLS); + order_array_size = MIN(MAX_ORDER_COLS, sel_orderby->len); + if (!disp_orderby_info(sel_orderby, res_merge, order_array, + order_array_size, merged_result)) + { + g_warning("%s:disp_orderby_info failed:%s", G_STRLOC, con->orig_sql->str); + return 0; + } + } + + GROUP_BY group_array[MAX_GROUP_COLS]; + int group_array_size = 0; /* number of GROUP_BY Columns */ + if (sel_groupby && sel_groupby->len > 0) { + memset(group_array, 0, sizeof(GROUP_BY) * MAX_GROUP_COLS); + group_array_size = MIN(MAX_GROUP_COLS, sel_groupby->len); + if (!disp_groupby_info(sel_groupby, res_merge, group_array, + group_array_size, merged_result)) + { + g_warning("%s:disp_groupby_info failed:%s", G_STRLOC, con->orig_sql->str); + return 0; + } + if (sel_orderby && sel_orderby->len > 0) { + /* if order by & group by both appears, it's guaranteed they have + only one same column */ + group_array[0].desc = order_array[0].desc; + } + } + + int i, index = 0; + for (i = 0; i < aggr_num; i++) { + network_mysqld_proto_fielddef_t *fdef = + g_ptr_array_index(res_merge->fielddefs, aggr_array[index].pos); + aggr_array[index].type = fdef->type; + index++; + } + + GList **candidates = g_new0(GList *, recv_queues->len); + /* field-count-packet + eof-packet */ + guint pkt_count = res_merge->field_count + 2; + + if (!prepare_for_row_process(candidates, recv_queues, + send_queue, pkt_count, merged_result)) + { + g_warning("%s:prepare_for_row_process failed", G_STRLOC); + g_free(candidates); + return 0; + } + + if (!check_network_packet_err(candidates, recv_queues, + send_queue, res_merge, merged_result)) + { + g_warning("%s:packet err is met", G_STRLOC); + g_free(candidates); + return 0; + } + + LIMIT limit; + limit.offset = 0; + limit.row_count = G_MAXINT32; + sql_expr_get_int(select->limit, &limit.row_count); + sql_expr_get_int(select->offset, &limit.offset); + int pack_err_met = 0; + having_condition_t *hav_condi = &(con->hav_condi); + + if (aggr_num > 0) { + aggr_by_group_para_t para; + + para.send_queue = send_queue; + para.recv_queues = recv_queues; + para.limit = &limit; + para.group_array = group_array; + para.aggr_array = aggr_array; + para.hav_condi = hav_condi; + para.group_array_size = group_array_size; + para.aggr_num = aggr_num; + + if (!aggr_by_group(¶, candidates, &pkt_count, merged_result)) { + g_free(candidates); + g_warning("%s:aggr_by_group error", G_STRLOC); + return 0; + } + g_free(candidates); + } else { + merge_parameters_t *data = g_new0(merge_parameters_t, 1); + heap_type *heap = g_new0(heap_type, 1); + data->heap = heap; + heap->len = recv_queues->len; + heap_element *elements = g_new0(heap_element, heap->len); + data->elements = elements; + + size_t iter; + for (iter = 0; iter < recv_queues->len; iter++) { + heap->element[iter] = elements + iter; + heap->element[iter]->is_dup = 0; + heap->element[iter]->is_err = 0; + heap->element[iter]->refreshed = 0; + heap->element[iter]->is_prior_to = 0; + } + + data->send_queue = send_queue; + data->recv_queues = recv_queues; + data->candidates = candidates; + data->pkt_count = pkt_count; + data->limit.offset = limit.offset; + data->limit.row_count = limit.row_count; + + for (iter = 0; iter < order_array_size; iter++) { + memcpy((heap->order_para).order_array + iter, order_array + iter, + sizeof(ORDER_BY)); + } + + data->pack_err_met = 0; + heap->order_para.order_array_size = order_array_size; + data->heap = heap; + + con->data = data; + + if (select->flags & SF_DISTINCT) { + data->is_distinct = 1; + } + + int merge_failed = 0; + int result = do_merge(con, con->data, &merge_failed); + + if (merge_failed) { + g_warning("%s:merge failed", G_STRLOC); + merged_result->status = RM_FAIL; + return 0; + } + + if (!result) { + g_warning("%s:result is zero", G_STRLOC); + return 0; + } + + pkt_count = data->pkt_count; + pack_err_met = data->pack_err_met; + } + + /* after adding all packets we don't need candidate list anymore */ + + /* need to append EOF after all Row Data Packets?? Yes + * update packet number in header + */ + + if (!con->partially_merged) { + if (pack_err_met == 0) { + g_debug("%s: append here", G_STRLOC); + /* TODO if in trans, then needs to set 'in transaction' flag ? */ + GString *eof_pkt = g_string_new_len("\x05\x00\x00\x07\xfe\x00\x00\x02\x00", 9); + eof_pkt->str[3] = pkt_count + 1; + network_queue_append(send_queue, eof_pkt); + } else { + g_debug("%s: err packet is met", G_STRLOC); + } + } + + return 1; +} + +static int check_fail_met(sql_context_t *context, network_queue *send_queue, GPtrArray *recv_queues, + network_mysqld_con *con, uint64_t *uniq_id, int *call_fail_met, result_merge_t *merged_result) +{ + int p; + char *orig_sql = con->orig_sql->str; + for (p = 0; p < recv_queues->len; p++) { + network_queue *recv_q = g_ptr_array_index(recv_queues, p); + GString *pkt = g_queue_peek_head(recv_q->chunks); + /* only check the first packet in each recv_queue */ + if (pkt != NULL && pkt->len > NET_HEADER_SIZE) { + guchar pkt_type = get_pkt_type(pkt); + if (pkt_type == MYSQLD_PACKET_ERR || pkt_type == MYSQLD_PACKET_EOF) { + server_session_t *pmd = g_ptr_array_index(con->servers, p); + if (pkt_type == MYSQLD_PACKET_ERR) { + g_warning("%s: failed query:%s, server:%s", + G_STRLOC, orig_sql, pmd->server->dst->name->str); + } + if (context->stmt_type == STMT_CALL) { + if (!(*call_fail_met)) { + *call_fail_met = 1; + chassis *srv = con->srv; + *uniq_id = incremental_guid_get_next(&(srv->guid_state)); + } + log_packet_error_info(con->client, pmd->server, orig_sql, pkt, *uniq_id); + continue; + } else { + network_queue_append(send_queue, pkt); + g_queue_pop_head(recv_q->chunks); + if (context->rw_flag & CF_DDL) { + g_warning("%s: failed ddl query:%s, server:%s", + G_STRLOC, orig_sql, pmd->server->dst->name->str); + } + return 0; + } + } + } else { + g_warning("%s: merge failed for con:%p", G_STRLOC, con); + merged_result->status = RM_FAIL; + return 0; + } + } + + return 1; +} + + +void resultset_merge(network_queue *send_queue, GPtrArray *recv_queues, + network_mysqld_con *con, uint64_t *uniq_id, + result_merge_t *merged_result) +{ + shard_plugin_con_t *st = con->plugin_con_state; + sql_context_t *context = st->sql_context; + + cetus_result_t res_merge = {0}; + + if (!send_queue || !recv_queues || recv_queues->len <= 0 || !context) { + g_warning("%s: packet->str[NET_HEADER_SIZE] != MYSQLD_PACKET_EOF", + G_STRLOC); + merged_result->status = RM_FAIL; + return; + } + + int call_fail_met = 0; + + if (!check_fail_met(context, send_queue, recv_queues, con, uniq_id, &call_fail_met, merged_result)) { + return; + } + + if (call_fail_met) { + g_warning("%s: call failed for con:%p", G_STRLOC, con); + merged_result->status = RM_CALL_FAIL; + return; + } + + if (context->explain) { + network_queue *first_q = g_ptr_array_index(recv_queues, 0); + if (first_q != NULL && first_q->chunks != NULL) { + GString *packet = NULL; + while ((packet = g_queue_pop_head(first_q->chunks)) != NULL) + network_queue_append(send_queue, packet); + } + merged_result->status = RM_SUCCESS; + return; + } + + switch (context->stmt_type) { + case STMT_SHOW_WARNINGS: + if (!merge_for_show_warnings(send_queue, recv_queues, con, + &res_merge, merged_result)) + { + cetus_result_destroy(&res_merge); + return; + } + break; + case STMT_SELECT: + if (!merge_for_select(context, send_queue, recv_queues, con, + &res_merge, merged_result)) + { + cetus_result_destroy(&res_merge); + return; + } + break; + case STMT_INSERT: + case STMT_UPDATE: + case STMT_DELETE: + case STMT_CALL: /* Response should have no records */ + case STMT_SET: + case STMT_START: + case STMT_COMMIT: + case STMT_ROLLBACK: + default: + if (!merge_for_modify(context, send_queue, recv_queues, con, + &res_merge, merged_result)) + { + cetus_result_destroy(&res_merge); + return; + } + break; + } + + cetus_result_destroy(&res_merge); + merged_result->status = RM_SUCCESS; +} + diff --git a/src/resultset_merge.h b/src/resultset_merge.h new file mode 100644 index 0000000..70914f2 --- /dev/null +++ b/src/resultset_merge.h @@ -0,0 +1,76 @@ +#ifndef __RESULTSET_MERGE_H__ +#include +#include +#include +#include +#include + +#include "sql-expression.h" +#include "network-mysqld.h" +#include "network-mysqld-proto.h" + + +#define MAX_NAME_LEN 64 +#define MAX_ORDER_COLS 16 +#define MAX_GROUP_COLS 16 +#define MAX_LIMIT G_MAXINT32 +#define MAX_SHARD_NUM MAX_SERVER_NUM + +typedef struct GROUP_BY { + char table_name[MAX_NAME_LEN]; + char name[MAX_NAME_LEN]; + unsigned int desc; + uint8_t type; + int pos; +} GROUP_BY; + +typedef struct ORDER_BY { + char table_name[MAX_NAME_LEN]; + char name[MAX_NAME_LEN]; + unsigned int desc; + unsigned int type; + int pos; +} ORDER_BY; + +typedef struct { + GList *record; + int index; + unsigned int is_over:1; + unsigned int is_err:1; + unsigned int refreshed:1; + unsigned int is_dup:1; + unsigned int is_prior_to:1; +} heap_element; + +typedef struct order_by_para_s { + ORDER_BY order_array[MAX_ORDER_COLS]; + uint64_t field_index[MAX_SHARD_NUM]; + int order_array_size; +} order_by_para_t; + +typedef struct { + heap_element *element[MAX_SHARD_NUM]; + order_by_para_t order_para; + unsigned int len:16; + unsigned int is_over:1; + unsigned int is_err:1; +} heap_type; + +typedef struct aggr_by_group_para_s { + network_queue *send_queue; + GPtrArray *recv_queues; + LIMIT *limit; + GROUP_BY *group_array; + GROUP_AGGR *aggr_array; + having_condition_t *hav_condi; + short aggr_num; + short group_array_size; +} aggr_by_group_para_t; + +NETWORK_API int callback_merge(network_mysqld_con *, merge_parameters_t *, int); +NETWORK_API void resultset_merge(network_queue *, GPtrArray *, network_mysqld_con *, + uint64_t *, result_merge_t *); + +NETWORK_API gint check_dist_tran_resultset(network_queue *recv_queue, network_mysqld_con *); + +#endif diff --git a/src/server-session.c b/src/server-session.c new file mode 100644 index 0000000..68fe472 --- /dev/null +++ b/src/server-session.c @@ -0,0 +1,439 @@ +#include "server-session.h" + +#include +#include +#include +#include + +#include "cetus-util.h" +#include "chassis-event.h" +#include "glib-ext.h" +#include "network-mysqld-proto.h" +#include "resultset_merge.h" +#include "plugin-common.h" + +void server_session_free(server_session_t *pmd) { + if (pmd) { + if (pmd->server != NULL) { + network_socket_free(pmd->server); + } + + pmd->sql = NULL; + + g_free(pmd); + } +} + +void server_sess_wait_for_event(server_session_t *pmd, short ev_type, + struct timeval *timeout) +{ + event_set(&(pmd->server->event), pmd->server->fd, ev_type, + server_session_con_handler, pmd); + chassis_event_add_with_timeout(pmd->con->srv, &(pmd->server->event), timeout); + pmd->server->is_waiting = 1; +} + +static void do_tcp_stream_after_recv_resp(network_mysqld_con *con, server_session_t *pmd) +{ + merge_parameters_t *data = con->data; + if (data->candidates[pmd->index] == NULL) { + data->candidates[pmd->index] = g_queue_peek_head_link(pmd->server->recv_queue->chunks); + } + + if (con->server_closed) { + if (con->servers) { + remove_mul_server_recv_packets(con); + proxy_put_shard_conn_to_pool(con); + } + g_message("%s: server conn is closed", G_STRLOC); + network_mysqld_con_send_error_full(con->client, + C("server closed prematurely"), ER_CETUS_RESULT_MERGE, "HY000"); + GList *err = con->client->send_queue->chunks->tail; + GString *err_packet = err->data; + merge_parameters_t *data = con->data; + + network_mysqld_proto_set_packet_id(err_packet, data->pkt_count + 1); + con->state = ST_SEND_QUERY_RESULT; + network_mysqld_con_handle(-1, 0, con); + return; + } + + if (!con->resp_too_long) { + if (con->num_pending_servers == 0) { + g_debug("%s: merge over", G_STRLOC); + if (callback_merge(con, con->data, 1) == RM_FAIL) { + network_queue_clear(con->client->send_queue); + network_mysqld_con_send_error_full(con->client, C("merge failed"), + ER_CETUS_RESULT_MERGE, "HY000"); + } + con->state = ST_SEND_QUERY_RESULT; + network_mysqld_con_handle(-1, 0, con); + } else { + if (callback_merge(con, con->data, 0) == RM_FAIL) { + network_queue_clear(con->client->send_queue); + network_mysqld_con_send_error_full(con->client, C("merge failed"), + ER_CETUS_RESULT_MERGE, "HY000"); + con->state = ST_SEND_QUERY_RESULT; + network_mysqld_con_handle(-1, 0, con); + } + } + } else { + g_message("%s: resp too long:%p, src port:%s, sql:%s", + G_STRLOC, con, con->client->src->name->str, + con->orig_sql->str); + network_mysqld_con_send_error_full(con->client, + C("response too long for proxy"),ER_CETUS_LONG_RESP, "HY000"); + GList *err = con->client->send_queue->chunks->tail; + GString *err_packet = err->data; + merge_parameters_t *data = con->data; + + network_mysqld_proto_set_packet_id(err_packet, data->pkt_count + 1); + con->state = ST_SEND_QUERY_RESULT; + network_mysqld_con_handle(-1, 0, con); + } +} + +static int process_read_event(network_mysqld_con *con, server_session_t *pmd) +{ + network_socket *sock = pmd->server; + + if (con->partially_merged) { + pmd->state = NET_RW_STATE_READ; + } + + int i, b = -1; + + if ((i = ioctl(sock->fd, FIONREAD, &b))) { + pmd->state = NET_RW_STATE_ERROR; + g_message("%s:NET_RW_STATE_ERROR is set i:%d,b:%d", + G_STRLOC, i, b); + } else if (b >= 0) { + sock->to_read = b; + sock->resp_len += b; + if (b == 0) { + if (con->dist_tran) { + record_xa_log_for_mending(con, sock); + sock->unavailable = 1; + pmd->state = NET_RW_STATE_FINISHED; + con->dist_tran_failed = 1; + g_message("%s: b is zero, xid:%s, con:%p, xa state:%d", + G_STRLOC, con->xid_str, con, con->dist_tran_state); + } + g_debug("%s:b is zero, socket fd:%d, con:%p", G_STRLOC, sock->fd, con); + } else { + con->resp_cnt &= (1 << pmd->index); + } + } else { + pmd->state = NET_RW_STATE_ERROR; + g_message("%s:NET_RW_STATE_ERROR is set i:%d,b:%d", + G_STRLOC, i, b); + } + + if (b > 0) { + return 1; + } else { + return 0; + } +} + +static void process_timeout_event(network_mysqld_con *con, server_session_t *pmd) +{ + network_socket *sock = pmd->server; + if (con->dist_tran) { + record_xa_log_for_mending(con, sock); + g_message("%s: EV_TIMEOUT for con xid:%s", G_STRLOC, con->xid_str); + } + con->is_timeout = 1; + pmd->state = NET_RW_STATE_FINISHED; + g_message("%s:EV_TIMEOUT, set server timeout, con:%p", G_STRLOC, con); +} + +static void +process_read_part_finished(network_mysqld_con *con, server_session_t *pmd) +{ + network_socket *sock = pmd->server; + + if (pmd->read_cal_flag == 0) { + con->num_read_pending--; + pmd->read_cal_flag = 1; + } + + g_debug("%s:part finished, num_read_pending:%d for con:%p, server fd:%d", + G_STRLOC, con->num_read_pending, con, pmd->server->fd); + if (!con->dist_tran_failed) { + if (con->num_read_pending == 0) { + if (!con->partially_merged) { + network_mysqld_con_handle(-1, 0, con); + } else { + do_tcp_stream_after_recv_resp(con, pmd); + } + } + } else { + if (con->num_pending_servers == 0) { + sock->is_closed = 1; + network_mysqld_con_handle(-1, 0, con); + } + } +} + +static void +process_read_finished(network_mysqld_con *con, server_session_t *pmd) +{ + network_socket *sock = pmd->server; + + if (pmd->read_cal_flag == 0) { + con->num_read_pending--; + pmd->read_cal_flag = 1; + } + + con->num_pending_servers--; + if (!con->dist_tran_failed) { + if (con->num_pending_servers == 0) { + network_mysqld_con_handle(-1, 0, con); + } + } else { + if (con->num_pending_servers == 0) { + sock->is_closed = 1; + network_mysqld_con_handle(-1, 0, con); + } + } +} + +static void +process_write_event(network_mysqld_con *con, server_session_t *pmd) +{ + network_socket *sock = pmd->server; + + switch (network_mysqld_write(con->srv, sock)) + { + case NETWORK_SOCKET_SUCCESS: + con->num_pending_servers++; + con->num_servers_visited++; + con->num_read_pending++; + pmd->read_cal_flag = 0; + con->num_write_pending--; + pmd->state = NET_RW_STATE_READ; + if (con->num_write_pending == 0) { + network_mysqld_con_handle(-1, 0, con); + } + break; + case NETWORK_SOCKET_WAIT_FOR_EVENT: + pmd->state = NET_RW_STATE_WRITE; + g_debug("%s:write wait for con:%p", G_STRLOC, con); + server_sess_wait_for_event(pmd, EV_WRITE, &con->write_timeout); + break; + default: + { + char *msg = "write error"; + con->state = ST_SEND_QUERY_RESULT; + con->server_to_be_closed = 1; + g_warning("%s:write error for con:%p", G_STRLOC, con); + network_mysqld_con_send_error_full(con->client, L(msg), + ER_NET_ERROR_ON_WRITE, "08S01"); + network_mysqld_con_handle(-1, 0, con); + break; + } + } + +} + +static int +process_read_server(network_mysqld_con *con, server_session_t *pmd) +{ + int is_finished = 0; + int ret = 0; + + network_socket *sock = pmd->server; + + if (sock->to_read == 0) { + sock->is_closed = 1; + is_finished = 1; + ret = NETWORK_SOCKET_SUCCESS; + con->server_to_be_closed = 1; + con->server_closed = 1; + } else { + ret = network_mysqld_read_mul_packets(con->srv, con, + sock, &is_finished); + } + + switch (ret) { + case NETWORK_SOCKET_SUCCESS: + g_debug("%s: resp len:%d, attr adj state:%d", + G_STRLOC, (int) sock->resp_len, con->attr_adj_state); + + if (is_finished) { + set_conn_attr(con, pmd->server); + pmd->state = NET_RW_STATE_FINISHED; + pmd->server->is_read_finished = 1; + pmd->server->is_waiting = 0; + } else { + if (con->candidate_tcp_streamed && con->num_servers_visited > 1) { + set_conn_attr(con, pmd->server); + pmd->state = NET_RW_STATE_PART_FINISHED; + g_debug("%s:tcp stream is true for:%p", G_STRLOC, con); + } else { + if (con->candidate_tcp_streamed) { + GString *packet; + while ((packet = g_queue_pop_head(pmd->server->recv_queue->chunks)) != NULL) { + network_mysqld_queue_append_raw(con->client, con->client->send_queue, packet); + } + + g_debug("%s: send_part_content_to_client", G_STRLOC); + + send_part_content_to_client(con); + } + server_sess_wait_for_event(pmd, EV_READ, &con->read_timeout); + } + } + break; + case NETWORK_SOCKET_WAIT_FOR_EVENT: + if (sock->resp_len > con->srv->max_resp_len) { + pmd->state = NET_RW_STATE_FINISHED; + con->server_to_be_closed = 1; + con->resp_too_long = 1; + g_debug("%s: resp len is too long for query:%s", + G_STRLOC, con->orig_sql->str); + if (!con->candidate_tcp_streamed || con->servers->len == 1) { + g_message("%s: resp too long:%p, src port:%s, sql:%s", + G_STRLOC, con, con->client->src->name->str, + con->orig_sql->str); + network_mysqld_con_send_error_full(con->client, + C("response too long for proxy"),ER_CETUS_LONG_RESP, "HY000"); + con->state = ST_SEND_QUERY_RESULT; + con->resultset_is_finished = TRUE; + network_mysqld_con_handle(-1, 0, con); + return 0; + } + + break; + } else { + + if (con->candidate_tcp_streamed && sock->max_header_size_reached && + con->num_servers_visited > 1) + { + g_debug("%s: set NET_RW_STATE_PART_FINISHED here", G_STRLOC); + pmd->state = NET_RW_STATE_PART_FINISHED; + if (con->partially_merged) { + do_tcp_stream_after_recv_resp(con, pmd); + } + break; + } else { + if (con->num_servers_visited == 1 && sock->max_header_size_reached + && con->candidate_tcp_streamed) + { + GString *packet; + while ((packet = g_queue_pop_head(pmd->server->recv_queue->chunks)) != NULL) { + network_mysqld_queue_append_raw(con->client, con->client->send_queue, packet); + } + g_debug("%s: send_part_content_to_client", G_STRLOC); + send_part_content_to_client(con); + } + server_sess_wait_for_event(pmd, EV_READ, &con->read_timeout); + return 0; + } + } + case NETWORK_SOCKET_ERROR: + pmd->state = NET_RW_STATE_ERROR; + g_debug("%s: NET_RW_STATE_ERROR is set here", G_STRLOC); + break; + } + + return 1; +} + + +static void +process_after_read(network_mysqld_con *con, server_session_t *pmd) +{ + con->num_pending_servers--; + if (pmd->read_cal_flag == 0) { + con->num_read_pending--; + pmd->read_cal_flag = 1; + } + if (!con->dist_tran_failed) { + if (con->partially_merged) { + do_tcp_stream_after_recv_resp(con, pmd); + } else { + if (con->num_pending_servers == 0 || (con->num_read_pending == 0)) + { + if (con->server_to_be_closed) { + g_message("%s:con:%p, state:%d", G_STRLOC, con, con->state); + } + + network_mysqld_con_handle(-1, 0, con); + } + } + } else { + if (con->num_pending_servers == 0) { + network_mysqld_con_handle(-1, 0, con); + } + } +} + +void server_session_con_handler(int event_fd, short events, void *user_data) +{ + server_session_t *pmd = (server_session_t *)user_data; + network_mysqld_con *con = pmd->con; + network_socket *sock = pmd->server; + + pmd->read_cal_flag = 0; + g_debug("%s:visit server_session_con_handler, state:%d for con:%p, fd:%d", + G_STRLOC, con->state, con, sock->fd); + + sock->is_waiting = 0; + + if (events == EV_READ) { + if (con->srv->query_cache_enabled) { + g_debug("%s: check if query can be cached, attr state:%d", G_STRLOC, con->attr_adj_state); + if (con->is_read_ro_server_allowed && con->attr_adj_state == ATTR_START) { + if (con->query_cache_judged == 0) { + do_check_qeury_cache(con); + } + } + } + + process_read_event(con, pmd); + } else if (events == EV_TIMEOUT) { + process_timeout_event(con, pmd); + } + + do { + + switch (pmd->state) { + case NET_RW_STATE_PART_FINISHED: + if (events == EV_READ || events == EV_TIMEOUT) { + process_read_part_finished(con, pmd); + } + return; + case NET_RW_STATE_ERROR: + case NET_RW_STATE_FINISHED: + if (events == EV_READ || events == EV_TIMEOUT) { + process_read_finished(con, pmd); + } + return; + case NET_RW_STATE_WRITE: + process_write_event(con, pmd); + return; + case NET_RW_STATE_READ: + if (!process_read_server(con, pmd)) { + return; + } + break; + default: + break; + } + + if (pmd->state == NET_RW_STATE_FINISHED) { + if (events == EV_READ) { + process_after_read(con, pmd); + } + return; + } + + } while (pmd->state != NET_RW_STATE_NONE); + + g_debug("%s:server_session_con_handler over for con:%p, pmd:%d", + G_STRLOC, con, pmd->index); +} + + diff --git a/src/server-session.h b/src/server-session.h new file mode 100644 index 0000000..3c7be37 --- /dev/null +++ b/src/server-session.h @@ -0,0 +1,14 @@ +#ifndef _SERVER_SESSION_ +#define _SERVER_SESSION_ + +#include "network-socket.h" +#include "network-mysqld.h" + +void server_session_free(server_session_t *pmd); + +void server_session_con_handler(int event_fd, short events, void *user_data); + +void server_sess_wait_for_event(server_session_t *ev_struct, short ev_type, + struct timeval *timeout); + +#endif /* _SERVER_SESSION_ */ diff --git a/src/shard-plugin-con.c b/src/shard-plugin-con.c new file mode 100644 index 0000000..e03ff3c --- /dev/null +++ b/src/shard-plugin-con.c @@ -0,0 +1,36 @@ +#include "shard-plugin-con.h" + +#include +#include +#include +#include + +#include "glib-ext.h" +#include "server-session.h" + +shard_plugin_con_t *shard_plugin_con_new() { + return g_new0(shard_plugin_con_t, 1); +} + +void +shard_plugin_con_free(network_mysqld_con *con, shard_plugin_con_t *st) +{ + if (!st) + return; + + if (con->servers) { + int i; + for (i = 0; i < con->servers->len; i++) { + server_session_t *pmd = g_ptr_array_index(con->servers, i); + pmd->sql = NULL; + g_free(pmd); + } + con->server = NULL; + } else { + if (con->server) { + st->backend->connected_clients--; + } + } + g_free(st); +} + diff --git a/src/shard-plugin-con.h b/src/shard-plugin-con.h new file mode 100644 index 0000000..1999fb2 --- /dev/null +++ b/src/shard-plugin-con.h @@ -0,0 +1,25 @@ +#ifndef _SHARD_PLUGIN_CON_ +#define _SHARD_PLUGIN_CON_ + +/* TODO: move to plugins/shard/ */ +#include "network-backend.h" +#include "network-mysqld.h" + +/** + * Contains extra connection state used for shard-plugin. + */ +typedef struct { + network_backend_t *backend; + /**< index into the backend-array, start from 0 */ + int backend_ndx; + + struct sql_context_t *sql_context; + int trx_read_write; /* default TF_READ_WRITE */ + int trx_isolation_level; /* default TF_REPEATABLE_READ */ + +} shard_plugin_con_t; + +NETWORK_API shard_plugin_con_t *shard_plugin_con_new(); +NETWORK_API void shard_plugin_con_free(network_mysqld_con *con, shard_plugin_con_t *st); + +#endif /*_SHARD_PLUGIN_CON_*/ diff --git a/src/sharding-config.c b/src/sharding-config.c new file mode 100644 index 0000000..2e86ebc --- /dev/null +++ b/src/sharding-config.c @@ -0,0 +1,816 @@ +#include "sharding-config.h" + +#include +#include +#include "glib-ext.h" +#include "sys-pedantic.h" +#include "cJSON.h" +#include "chassis-timings.h" + +static GList *shard_conf_vdbs = NULL; + +static GList *shard_conf_tables = NULL; + +static GHashTable *shard_conf_vdb_map = NULL; + +static GList *shard_conf_single_tables = NULL; + + +typedef struct sharding_database_t { + char *name; + GHashTable *tables; /* */ +} sharding_database_t; + +static sharding_database_t *sharding_database_new(const char *name) +{ + sharding_database_t *db = g_new0(sharding_database_t, 1); + db->tables = g_hash_table_new(g_str_hash, g_str_equal); + db->name = g_strdup(name); + return db; +} + +static void sharding_database_free(sharding_database_t *db) +{ + g_free(db->name); + g_hash_table_destroy(db->tables); + g_free(db); +} + +static void sharding_database_add_table(sharding_database_t *db, sharding_table_t *table) +{ + g_hash_table_insert(db->tables, table->name->str, table); +} + +static sharding_table_t *sharding_database_get_table(sharding_database_t *db, + const char *table) +{ + if (!table) + return NULL; + return g_hash_table_lookup(db->tables, table); +} + +static sharding_vdb_t *shard_vdbs_get_by_id(GList *vdbs, int id) +{ + GList *l = vdbs; + for (; l != NULL; l = l->next) { + sharding_vdb_t *vdb = l->data; + if (vdb->id == id) { + return vdb; + } + } + return NULL; +} + +void sharding_table_free(gpointer q) { + sharding_table_t *info = q; + if (NULL != info->db) g_string_free(info->db, TRUE); + if (NULL != info->name) g_string_free(info->name, TRUE); + if (NULL != info->pkey) g_string_free(info->pkey, TRUE); + g_free(info); +} + +gboolean sharding_partition_contain_hash(sharding_partition_t *partition, int val) +{ + g_assert(partition->vdb->method == SHARD_METHOD_HASH); + if (val >= partition->vdb->logic_shard_num) + return FALSE; + return TestBit(partition->hash_set, val); +} + +static sharding_vdb_t *sharding_vdb_new() +{ + sharding_vdb_t *vdb = g_new0(struct sharding_vdb_t, 1); + vdb->databases = g_ptr_array_new_with_free_func((GDestroyNotify)sharding_database_free); + vdb->partitions = g_ptr_array_new(); + return vdb; +} + +static void sharding_vdb_free(sharding_vdb_t *vdb) +{ + if (!vdb) { + return; + } + + int i; + for (i = 0; i < vdb->partitions->len; i++) { + sharding_partition_t *item = g_ptr_array_index(vdb->partitions, i); + if (vdb->method == SHARD_METHOD_RANGE) { + if (vdb->key_type == SHARD_DATA_TYPE_STR) { + if (item->value) { + g_free(item->value); + } + if (item->low_value) { + g_free(item->low_value); + } + } + } + if (item->group_name) { + g_string_free(item->group_name, TRUE); + } + g_free(item); + } + g_ptr_array_free(vdb->partitions, TRUE); + + g_ptr_array_free(vdb->databases, TRUE); + g_free(vdb); +} + +static gboolean sharding_vdb_is_valid(sharding_vdb_t *vdb) +{ + if (vdb->method == SHARD_METHOD_HASH) { + if (vdb->logic_shard_num <= 0 || vdb->logic_shard_num > MAX_HASH_VALUE_COUNT) { + return FALSE; + } + + /* make sure all hash values fall into a partition */ + char *value_set = g_malloc0(vdb->logic_shard_num); + int i, j; + /* collect hash values of all partitions */ + for (i = 0; i < vdb->partitions->len; i++) { + sharding_partition_t *part = g_ptr_array_index(vdb->partitions, i); + for (j = 0; j < vdb->logic_shard_num; ++j) { + if (TestBit(part->hash_set, j)) + value_set[j] = 1; + } + } + /* we expect that value_set all filled with 1 */ + for (i = 0; i < vdb->logic_shard_num; ++i) { + if (value_set[i] == 0) { + g_free(value_set); + return FALSE; + } + } + g_free(value_set); + } + return TRUE; +} + +static sharding_database_t *sharding_vdb_get_database(sharding_vdb_t *vdb, + const char *db_name) +{ + int i = 0; + for (i = 0; i < vdb->databases->len; ++i) { + sharding_database_t *db = g_ptr_array_index(vdb->databases, i); + if (strcasecmp(db_name, db->name) == 0) { + return db; + } + } + return NULL; +} + +static sharding_database_t *sharding_vdb_add_database(sharding_vdb_t *vdb, + const char *name) +{ + sharding_database_t *db = sharding_database_new(name); + g_ptr_array_add(vdb->databases, db); + return db; +} + +GPtrArray *shard_conf_get_all_groups(GPtrArray *visited_groups, const char *db) +{ + if (!db) { + g_warning(G_STRLOC " db name is NULL"); + return visited_groups; + } + sharding_vdb_t *vdb = g_hash_table_lookup(shard_conf_vdb_map, db); + if (vdb) { + int i = 0; + for (i = 0; i < vdb->partitions->len; ++i) { + sharding_partition_t *part = g_ptr_array_index(vdb->partitions, i); + GString *gp = part->group_name; + g_ptr_array_add(visited_groups, gp); + } + } else { + g_warning(G_STRLOC " fail to get all groups for db: %s", db); + } + return visited_groups; +} + +void shard_conf_find_groups(GPtrArray *groups, const char *match, const char *db) +{ + if (strcasecmp(match, "all") == 0) { + shard_conf_get_all_groups(groups, db); + return; + } + sharding_vdb_t *vdb = g_hash_table_lookup(shard_conf_vdb_map, db); + if (vdb) { + int i = 0; + for (i = 0; i < vdb->partitions->len; ++i) { + sharding_partition_t *part = g_ptr_array_index(vdb->partitions, i); + GString *gp = part->group_name; + if (strcmp(gp->str, match) == 0) { + g_ptr_array_add(groups, gp); + return; + } + } + } +} + +GPtrArray *shard_conf_get_any_group(GPtrArray *visited_groups, char *db, + char *UNUSED_PARAM(table)) +{ + if (!db) { + g_warning(G_STRLOC " db name is NULL"); + return NULL; + } + sharding_vdb_t *vdb = g_hash_table_lookup(shard_conf_vdb_map, db); + if (vdb == NULL) { + return NULL; + } + + GPtrArray *partitions = vdb->partitions; + int i = rand() % partitions->len; + + sharding_partition_t *part = g_ptr_array_index(partitions, i); + g_ptr_array_add(visited_groups, part->group_name); + + return visited_groups; +} + +GPtrArray *shard_conf_get_table_groups(GPtrArray *visited_groups, char *db, + char *UNUSED_PARAM(table)) +{ + if (!db) { + g_warning(G_STRLOC " db name is NULL"); + return NULL; + } + + sharding_vdb_t *vdb = g_hash_table_lookup(shard_conf_vdb_map, db); + if (vdb == NULL) { + return NULL; + } + + GPtrArray *partitions = vdb->partitions; + int i, j; + for (i = 0; i < partitions->len; i++) { + sharding_partition_t *partition = g_ptr_array_index(partitions, i); + int already_added = 0; + for (j = 0; j < visited_groups->len; j++) { + GString *added_group_name = visited_groups->pdata[j]; + if (g_string_equal(partition->group_name, added_group_name)) { + already_added = 1; + break; + } + } + if (already_added == 0) { + g_ptr_array_add(visited_groups, partition->group_name); + } + } + return visited_groups; +} + +/** + * get array of groups pointers of a table + * no more duplication check cause one group correspond multiple range value + */ +GPtrArray *shard_conf_table_partitions(GPtrArray *partitions, + const char *db, const char *UNUSED_PARAM(table)) { + if (!db) { + g_warning(G_STRLOC " db name is NULL"); + return NULL; + } + + sharding_vdb_t *vdb = g_hash_table_lookup(shard_conf_vdb_map, db); + if (!vdb) { + return NULL; + } + GPtrArray *all_partitions = vdb->partitions; + int i; + for (i = 0; i < all_partitions->len; i++) { + sharding_partition_t *part = g_ptr_array_index(all_partitions, i); + g_ptr_array_add(partitions, part); + } + return partitions; +} + +sharding_table_t *shard_conf_get_info(const char *db_name, const char *table) +{ + if (!db_name) + return NULL; + sharding_vdb_t *vdb = g_hash_table_lookup(shard_conf_vdb_map, db_name); + if (!vdb) { + return NULL; + } + sharding_database_t *db = sharding_vdb_get_database(vdb, db_name); + if (!db) { + return NULL; + } + return sharding_database_get_table(db, table); +} + +gboolean shard_conf_is_shard_table(const char *db, const char *table) +{ + return shard_conf_get_info(db, table) ? TRUE : FALSE; +} + +GPtrArray *shard_conf_get_fixed_group(GPtrArray *groups, const char *db, guint32 fixture) +{ + if (!db) { + g_warning(G_STRLOC " db name is NULL"); + return groups; + } + sharding_vdb_t *vdb = g_hash_table_lookup(shard_conf_vdb_map, db); + if (vdb) { + int base = vdb->partitions->len; + if (base == 0) { + return groups; + } + int index = fixture % base; + sharding_partition_t *part = g_ptr_array_index(vdb->partitions, index); + g_ptr_array_add(groups, part->group_name); + } + return groups; +} + +typedef struct single_table_t { /* single table only resides on 1 group */ + GString *name; + GString *db; + GString *group; +} single_table_t; + +void single_table_free(single_table_t *t) +{ + if (t) { + g_string_free(t->name, TRUE); + g_string_free(t->db, TRUE); + g_string_free(t->group, TRUE); + g_free(t); + } +} + +static void shard_conf_set_vdb_list(GList *vdbs) +{ + g_list_free_full(shard_conf_vdbs, (GDestroyNotify)sharding_vdb_free); + shard_conf_vdbs = vdbs; +} + +static void shard_conf_set_table_list(GList *tables) +{ + g_list_free_full(shard_conf_tables, (GDestroyNotify)sharding_table_free); + shard_conf_tables = tables; +} + +static void shard_conf_set_vdb_map(GHashTable *vdbmap) +{ + if (shard_conf_vdb_map) { + g_hash_table_destroy(shard_conf_vdb_map); + } + shard_conf_vdb_map = vdbmap; +} + +static void shard_conf_set_single_tables(GList *tables) +{ + g_list_free_full(shard_conf_single_tables, (GDestroyNotify)single_table_free); + shard_conf_single_tables = tables; +} + +/** + * setup index & validate configurations + */ +gboolean shard_conf_try_setup(GList *vdbs, GList *tables, GList *single_tables) +{ + if (!vdbs || !tables) { + g_critical("empty vdb/table list"); + return FALSE; + } + GList *l = vdbs; + for (; l != NULL; l = l->next) { + sharding_vdb_t *vdb = l->data; + if (!sharding_vdb_is_valid(vdb)) { + g_warning("invalid vdb config"); + return FALSE; + } + } + GHashTable *vdbmap = g_hash_table_new(g_str_hash, g_str_equal); + + l = tables; + for (; l != NULL; l = l->next) { + sharding_table_t *table = l->data; + sharding_vdb_t *vdb = shard_vdbs_get_by_id(vdbs, table->vdb_id); + + /* Fill table with vdb data */ + if (!vdb) { + g_critical(G_STRLOC " table:%s VDB ID cannot be found: %d", + table->name->str, table->vdb_id); + g_hash_table_destroy(vdbmap); + return FALSE; + } else { + table->shard_key_type = vdb->key_type; + table->logic_shard_num = vdb->logic_shard_num; + table->method = vdb->method; + table->partitions = vdb->partitions; + } + + /* collect database into vdb */ + sharding_database_t *database = sharding_vdb_get_database(vdb, table->db->str); + if (!database) { + database = sharding_vdb_add_database(vdb, table->db->str); + } + + /* setup table map in database */ + if (sharding_database_get_table(database, table->name->str)) { + g_hash_table_destroy(vdbmap); + g_critical(G_STRLOC " same table name inside same db: %s", table->name->str); + return FALSE; + } + sharding_database_add_table(database, table); + + /* setup db to vdb map */ + sharding_vdb_t *vdb_res = g_hash_table_lookup(vdbmap, database->name); + if (vdb_res && vdb_res != vdb) { + g_hash_table_destroy(vdbmap); + g_critical(G_STRLOC " same db inside different vdb: %s", database->name); + return FALSE; + } else { + g_hash_table_insert(vdbmap, database->name, vdb); + } + } + shard_conf_set_vdb_list(vdbs); + shard_conf_set_table_list(tables); + shard_conf_set_vdb_map(vdbmap); + shard_conf_set_single_tables(single_tables); + return TRUE; +} + +void shard_conf_destroy(void) +{ + if (shard_conf_vdbs) { + g_list_free_full(shard_conf_vdbs, (GDestroyNotify)sharding_vdb_free); + } + g_list_free_full(shard_conf_tables, (GDestroyNotify)sharding_table_free); + if (shard_conf_vdb_map) { + g_hash_table_destroy(shard_conf_vdb_map); + } +} + +static GHashTable *load_shard_from_json(gchar *json_str); + +gboolean shard_conf_load(char *json_str) +{ + GHashTable *ht = load_shard_from_json(json_str); + if (!ht) + return FALSE; + + GList *tables = g_hash_table_lookup(ht, "table_list"); + GList *vdbs = g_hash_table_lookup(ht, "vdb_list"); + GList *single_tables = g_hash_table_lookup(ht, "single_tables"); + gboolean success = shard_conf_try_setup(vdbs, tables, single_tables); + if (!success) { + g_list_free_full(vdbs, (GDestroyNotify)sharding_vdb_free); + g_list_free_full(tables, (GDestroyNotify)sharding_table_free); + } + g_hash_table_destroy(ht); + return success; +} + +static single_table_t *shard_conf_get_single_table(const char *db, const char *name) +{ + GList *l = shard_conf_single_tables; + for (; l; l = l->next) { + single_table_t *t = l->data; + if (strcasecmp(t->name->str, name) == 0 + && strcasecmp(t->db->str, db) == 0) { + return t; + } + } + return NULL; +} + +gboolean shard_conf_is_single_table(const char *db, const char *name) +{ + single_table_t *t = shard_conf_get_single_table(db, name); + return t != NULL; +} + +static gboolean shard_conf_group_contains(GPtrArray *groups, GString *match) +{ + int i; + for (i = 0; i < groups->len; ++i) { + GString *gp = g_ptr_array_index(groups, i); + if (g_string_equal(gp, match)) { + return TRUE; + } + } + return FALSE; +} + +GPtrArray *shard_conf_get_single_table_distinct_group(GPtrArray *groups, + const char *db, const char *name) +{ + single_table_t *t = shard_conf_get_single_table(db, name); + if (t && !shard_conf_group_contains(groups, t->group)) { + g_ptr_array_add(groups, t->group); + } + return groups; +} + +static int sharding_type(const char *str) { + struct code_map_t { + const char *name; + int code; + } map[] = { + {"INT", SHARD_DATA_TYPE_INT}, + {"STR", SHARD_DATA_TYPE_STR}, + {"DATE", SHARD_DATA_TYPE_DATE}, + {"DATETIME", SHARD_DATA_TYPE_DATETIME}, + }; + int i; + for (i = 0; i < sizeof(map)/sizeof(*map); ++i) { + if (strcasecmp(map[i].name, str) == 0) + return map[i].code; + } + g_critical("Wrong sharding setting ", str); + return -1; +} + +static int sharding_method(const char *str) { + if (strcasecmp(str, "hash") == 0) { + return SHARD_METHOD_HASH; + } else if (strcasecmp(str, "range") == 0) { + return SHARD_METHOD_RANGE; + } else { + return SHARD_METHOD_UNKNOWN; + } +} +/* + * Parse partitions from JSON to Hash Table + * exmpale: + * {"data1":[0], "data2":[1], "data3":[2], "data4":[3]} + */ +static void parse_partitions(cJSON *root, const sharding_vdb_t *vdb, + GPtrArray *partitions /* out */) +{ + cJSON *cur = root->child; + sharding_partition_t *item; + for (; cur; cur = cur->next) { /* { "groupA":xx, "groupB":xx, "groupC":xx} */ + /* null means unlimited */ + switch (cur->type) { + case cJSON_NULL: /* range: null */ + item = g_new0(sharding_partition_t, 1); + item->vdb = vdb; + item->group_name = g_string_new(cur->string); + if (vdb->key_type == SHARD_DATA_TYPE_STR) { + item->value = NULL; + } else { + item->value = (void *) (uint64_t) INT_MAX; + } + g_ptr_array_add(partitions, item); + break; + case cJSON_Number: /* range > 123 */ + item = g_new0(sharding_partition_t, 1); + item->vdb = vdb; + item->group_name = g_string_new(cur->string); + item->value = (void *) (uint64_t) cur->valueint; + g_ptr_array_add(partitions, item); + break; + case cJSON_String: /* range > "str" */ + item = g_new0(sharding_partition_t, 1); + item->vdb = vdb; + item->group_name = g_string_new(cur->string); + if (vdb->key_type == SHARD_DATA_TYPE_DATETIME + || vdb->key_type == SHARD_DATA_TYPE_DATE) { + gboolean ok; + int epoch = chassis_epoch_from_string(cur->valuestring, &ok); + if (ok) + item->value = (void *) (uint64_t)epoch; + else + g_warning("Wrong sharding setting ", + cur->valuestring); + } else { + item->value = g_strdup(cur->valuestring); + } + g_ptr_array_add(partitions, item); + break; + case cJSON_Array: { + cJSON *elem = cur->child; + if (cJSON_Number == elem->type) { /* hash in [0,3,5] */ + item = g_new0(sharding_partition_t, 1); + item->vdb = vdb; + item->group_name = g_string_new(cur->string); + for (; elem; elem = elem->next) { + if (elem->type != cJSON_Number) { + g_critical(G_STRLOC "array has different type"); + continue; + } + if (elem->valueint >= 0 && elem->valueint < vdb->logic_shard_num) { + SetBit(item->hash_set, elem->valueint); + } else { + g_critical(G_STRLOC "hash value exceeds logic_shard_num"); + } + } + g_ptr_array_add(partitions, item); + } else if (cJSON_String == elem->type) { /* TODO: range in [0, 100, 200] */ + while (elem != NULL) { + item = g_new0(sharding_partition_t, 1); + item->vdb = vdb; + item->group_name = g_string_new(cur->string); + if (vdb->key_type == SHARD_DATA_TYPE_DATETIME + || vdb->key_type == SHARD_DATA_TYPE_DATE) { + gboolean ok; + int epoch = chassis_epoch_from_string(elem->valuestring, &ok); + if (ok) + item->value = (void *) (uint64_t)epoch; + else + g_warning("Wrong sharding setting ", + elem->valuestring); + } else { + item->value = g_strdup(elem->valuestring); + } + g_ptr_array_add(partitions, item); + elem = elem->next; + } + } + break; + } + default: + g_warning("JSON TYPE: %d, GROUP: %s", cur->type, cur->string); + } /* end switch */ + } +} + +static gint cmp_shard_range_groups_int(gconstpointer a, gconstpointer b) { + sharding_partition_t *item1 = *(sharding_partition_t **)a; + sharding_partition_t *item2 = *(sharding_partition_t **)b; + int n1 = (int) (int64_t) item1->value; + int n2 = (int) (int64_t) item2->value; + return n1-n2; +} + +static gint cmp_shard_range_groups_str(gconstpointer a, gconstpointer b) { + sharding_partition_t *item1 = *(sharding_partition_t **)a; + sharding_partition_t *item2 = *(sharding_partition_t **)b; + const char *s1 = item1->value; + const char *s2 = item2->value; + if (s1 == NULL) { + return 1; + } else if (s2 == NULL) { + return -1; + } + return strcmp(s1, s2); +} + +static void setup_partitions(GPtrArray *partitions, sharding_vdb_t *vdb) +{ + if (vdb->method == SHARD_METHOD_RANGE) { + /* sort partitions */ + if (vdb->key_type == SHARD_DATA_TYPE_INT + || vdb->key_type == SHARD_DATA_TYPE_DATETIME + || vdb->key_type == SHARD_DATA_TYPE_DATE) { + g_ptr_array_sort(partitions, cmp_shard_range_groups_int); + } else { + g_ptr_array_sort(partitions, cmp_shard_range_groups_str); + } + /* record the lower range, get from previous group */ + int64_t prev_value = INT_MIN; + char *prev_str = NULL; + int i; + for (i = 0; i < partitions->len; ++i) { + sharding_partition_t *part = g_ptr_array_index(partitions, i); + if (vdb->key_type == SHARD_DATA_TYPE_STR) { + part->low_value = prev_str; + if (i != partitions->len-1) { + prev_str = g_strdup(part->value); + } + } else { + part->low_value = (void *)prev_value; + prev_value = (int64_t)part->value; + } + } + } +} +/** + * @return GList + */ +static GList *parse_vdbs(cJSON *vdb_root) +{ + GList *vdb_list = NULL; + cJSON *p = vdb_root->child; + for (; p != NULL; p = p->next) { + cJSON *id = cJSON_GetObjectItem(p, "id"); + cJSON *key_type = cJSON_GetObjectItem(p, "type"); + cJSON *method = cJSON_GetObjectItem(p, "method"); + cJSON *num = cJSON_GetObjectItem(p, "num"); + cJSON *partitions = cJSON_GetObjectItem(p, "partitions"); + if (!(id && key_type && method && num && partitions)) { + g_critical("parse vdbs error, neglected"); + continue; + } + + struct sharding_vdb_t *vdb = sharding_vdb_new(); + if (id->type == cJSON_Number) { + vdb->id = id->valueint; + } else { + vdb->id = atoi(id->valuestring); + } + vdb->key_type = sharding_type(key_type->valuestring); + if (vdb->key_type < 0) { + g_critical("Wrong sharding settings ", key_type->valuestring); + } + vdb->method = sharding_method(method->valuestring); + if (vdb->method < 0) { + g_critical("Wrong sharding settings ", method->valuestring); + } + + if (num->type == cJSON_Number) { + vdb->logic_shard_num = num->valueint; + } else { + g_critical("no match num: %s", num->valuestring); + } + + parse_partitions(partitions, vdb, vdb->partitions); + setup_partitions(vdb->partitions, vdb); + + vdb_list = g_list_append(vdb_list, vdb); + } + return vdb_list; +} + +static GList *parse_tables(cJSON *root) +{ + GList *tables = NULL; + cJSON *p = root->child; + while (p) { + cJSON *db = cJSON_GetObjectItem(p, "db"); + cJSON *table_root = cJSON_GetObjectItem(p, "table"); + cJSON *pkey = cJSON_GetObjectItem(p, "pkey"); + cJSON *vdb = cJSON_GetObjectItem(p, "vdb"); + if (db && table_root && pkey && vdb) { + sharding_table_t *table = g_new0(sharding_table_t, 1); + if (vdb->type == cJSON_String) { + table->vdb_id = atoi(vdb->string); + } else if (vdb->type == cJSON_Number) { + table->vdb_id = vdb->valueint; + } + table->db = g_string_new(db->valuestring); + table->name = g_string_new(table_root->valuestring); + table->pkey = g_string_new(pkey->valuestring); + + tables = g_list_append(tables, table); + } else { + g_critical("parse_tables error"); + } + p = p->next; + } + return tables; +} + +static GList *parse_single_tables(cJSON *root) +{ + GList *tables = NULL; + cJSON *p = root->child; + while (p) { + cJSON *name = cJSON_GetObjectItem(p, "table"); + cJSON *db = cJSON_GetObjectItem(p, "db"); + cJSON *group = cJSON_GetObjectItem(p, "group"); + if (name && db && group) { + single_table_t *table = g_new0(single_table_t, 1); + table->group = g_string_new(group->valuestring); + table->db = g_string_new(db->valuestring); + table->name = g_string_new(name->valuestring); + tables = g_list_append(tables, table); + } else { + g_critical("single_table parse error"); + } + p = p->next; + } + return tables; +} + +static GHashTable *load_shard_from_json(gchar *json_str) +{ + cJSON *root = cJSON_Parse(json_str); + if (!root) { + g_critical("JSON format is not correct!"); + return NULL; + } + + /* parse vdbs */ + cJSON *vdb_root = cJSON_GetObjectItem(root, "vdb"); + if (!vdb_root) { + g_critical(G_STRLOC "vdb config file error"); + } + GList *vdb_list = parse_vdbs(vdb_root); + + /* parse tables */ + cJSON *table_root = cJSON_GetObjectItem(root, "table"); + if (!table_root) { + g_critical(G_STRLOC "table config error"); + } + GList *table_list = parse_tables(table_root); + + /* parse single tables */ + cJSON *single_root = cJSON_GetObjectItem(root, "single_tables"); + GList *single_list = NULL; + if (single_root) { + single_list = parse_single_tables(single_root); + } + + cJSON_Delete(root); + + GHashTable *shard_hash = g_hash_table_new(g_str_hash, g_str_equal); + g_hash_table_insert(shard_hash, "table_list", table_list); + g_hash_table_insert(shard_hash, "vdb_list", vdb_list); + g_hash_table_insert(shard_hash, "single_tables", single_list); /* NULLable */ + return shard_hash; +} + diff --git a/src/sharding-config.h b/src/sharding-config.h new file mode 100644 index 0000000..55ddf55 --- /dev/null +++ b/src/sharding-config.h @@ -0,0 +1,103 @@ +#ifndef __SHARDING_CONFIG_H__ +#define __SHARDING_CONFIG_H__ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "glib-ext.h" +#include "cetus-util.h" + +#define SHARD_DATA_TYPE_UNSUPPORTED 0 +#define SHARD_DATA_TYPE_INT 1 +#define SHARD_DATA_TYPE_STR 2 +#define SHARD_DATA_TYPE_DATE 3 +#define SHARD_DATA_TYPE_DATETIME 4 + +enum sharding_method_t { + SHARD_METHOD_UNKNOWN = -1, + SHARD_METHOD_RANGE = 0, + SHARD_METHOD_HASH = 1, + SHARD_METHOD_LIST = 2 +}; + +typedef struct sharding_vdb_t sharding_vdb_t; +typedef struct sharding_table_t sharding_table_t; + +#define MAX_HASH_VALUE_COUNT 1024 + +typedef struct sharding_partition_t { + char *value; /* high range OR hash value */ + char *low_value; /* low range OR null */ + + BitArray hash_set[MAX_HASH_VALUE_COUNT/32]; /* hash values of this partition */ + + GString *group_name; + const sharding_vdb_t *vdb; /* references the vdb it belongs to */ +} sharding_partition_t; + +gboolean sharding_partition_contain_hash(sharding_partition_t *, int); +//gboolean sharding_partition_cover_range(sharding_partition_t *, ); + +struct sharding_vdb_t { + int id; + enum sharding_method_t method; + int key_type; + int logic_shard_num; + GPtrArray *partitions; /* GPtrArray */ + GPtrArray *databases; /* GPtrArray */ +}; + +struct sharding_table_t { + GString *db; + GString *name; + GString *pkey; + int shard_key_type; + int logic_shard_num; + enum sharding_method_t method; + GPtrArray *partitions; /* GPtrArray ref from VDB */ + + int vdb_id; + struct sharding_vdb_t *vdb; +}; + +GPtrArray *shard_conf_get_table_groups(GPtrArray *groups, char *db, char *table); + +GPtrArray *shard_conf_get_any_group(GPtrArray *groups, char *db, char *table); + +GPtrArray *shard_conf_get_all_groups(GPtrArray *groups, const char *db); + +/* same fixture will get same group */ +GPtrArray *shard_conf_get_fixed_group(GPtrArray *groups, const char *db, guint32 fixture); + +gboolean shard_conf_is_shard_table(const char *db, const char *table); + +gboolean shard_conf_is_single_table(const char *db, const char *table); + +GPtrArray *shard_conf_get_single_table_distinct_group(GPtrArray *groups, + const char *db, const char *table); + +sharding_table_t *shard_conf_get_info(const char *db, const char *table); + +/** + * similar with shard_conf_get_table_groups + * @return array of group_item_t * + */ +GPtrArray *shard_conf_table_partitions(GPtrArray *partitions, + const char *db, const char *table); + +/** + * find partition by group name + * special name "all" will get all groups + */ +void shard_conf_find_groups(GPtrArray *groups, const char *match, const char *db); + + +gboolean shard_conf_load(char *); + +void shard_conf_destroy(void); + + + +#endif /* __SHARDING_CONFIG_H__ */ + diff --git a/src/sharding-query-plan.c b/src/sharding-query-plan.c new file mode 100644 index 0000000..5279b1e --- /dev/null +++ b/src/sharding-query-plan.c @@ -0,0 +1,137 @@ +#include "sharding-query-plan.h" + +#include + +sharding_plan_t *sharding_plan_new(const GString *orig_sql) +{ + sharding_plan_t *plan = g_new0(sharding_plan_t, 1); + plan->orig_sql = orig_sql; + plan->groups = g_ptr_array_new(); + return plan; +} + +void sharding_plan_free(sharding_plan_t *plan) +{ + g_ptr_array_free(plan->groups, TRUE); + if (plan->sql_list) { + g_list_free_full(plan->sql_list, g_string_true_free); + } + if (plan->mapping) { + GList *l = plan->mapping; + for (; l != NULL; l = l->next) { + g_free(l->data); + } + g_list_free(plan->mapping); + } + + g_free(plan); +} + +static struct _group_sql_pair *sharding_plan_get_mapping(sharding_plan_t *plan, + const GString *gp) +{ + if (plan->mapping) { + GList *l = plan->mapping; + for (; l != NULL; l = l->next) { + struct _group_sql_pair *group = l->data; + if (gp == group->gp_name || g_string_equal(gp, group->gp_name)) { + return group; + } + } + } + return NULL; +} + +gboolean sharding_plan_has_group(sharding_plan_t *plan, const GString *gp) +{ + if (plan->groups) { + int i; + for (i = 0; i < plan->groups->len; ++i) { + const GString *group = g_ptr_array_index(plan->groups, i); + if (group == gp || g_string_equal(gp, group)) { + return TRUE; + } + } + } + return FALSE; +} + +static void sharding_plan_add_mapping(sharding_plan_t *plan, + const GString *group, const GString *sql) +{ + struct _group_sql_pair *pair = sharding_plan_get_mapping(plan, group); + if (pair) { + pair->sql = sql; + } else { + struct _group_sql_pair *new_pair = g_new0(struct _group_sql_pair, 1); + new_pair->gp_name = group; + new_pair->sql = sql; + plan->mapping = g_list_append(plan->mapping, new_pair); + } +} + +void sharding_plan_add_group(sharding_plan_t *plan, GString *gp_name) +{ + if (!sharding_plan_has_group(plan, gp_name)) { + g_ptr_array_add(plan->groups, gp_name); + } + sharding_plan_add_mapping(plan, gp_name, NULL); +} + +void sharding_plan_add_groups(sharding_plan_t *plan, GPtrArray *groups) +{ + if (!groups) { + return; + } + int i; + for (i = 0; i < groups->len; ++i) { + sharding_plan_add_group(plan, g_ptr_array_index(groups, i)); + } +} + +void sharding_plan_clear_group(sharding_plan_t *plan) +{ + GPtrArray *groups = plan->groups; + g_ptr_array_remove_range(groups, 0, groups->len); +} + +void sharding_plan_add_group_sql(sharding_plan_t *plan, GString *gp_name, GString *sql) +{ + plan->sql_list = g_list_append(plan->sql_list, sql); + if (!sharding_plan_has_group(plan, gp_name)) { + g_ptr_array_add(plan->groups, gp_name); + } + sharding_plan_add_mapping(plan, gp_name, sql); +} + +const GString *sharding_plan_get_sql(sharding_plan_t *plan, const GString *group) +{ + struct _group_sql_pair *pair = sharding_plan_get_mapping(plan, group); + if (pair) { + if (pair->sql) { + return pair->sql; + } else { + return plan->is_modified ? plan->modified_sql : plan->orig_sql; + } + } + return NULL; +} + + +void sharding_plan_set_modified_sql(sharding_plan_t *plan, GString *sql) +{ + plan->is_modified = TRUE; + plan->modified_sql = sql; +} + +static gint gstr_comp(gconstpointer a1, gconstpointer a2) +{ + GString *s1 = *(GString **)a1; + GString *s2 = *(GString **)a2; + return strcmp(s1->str, s2->str); +} + +void sharding_plan_sort_groups(sharding_plan_t *plan) +{ + g_ptr_array_sort(plan->groups, gstr_comp); +} diff --git a/src/sharding-query-plan.h b/src/sharding-query-plan.h new file mode 100644 index 0000000..ad84f0f --- /dev/null +++ b/src/sharding-query-plan.h @@ -0,0 +1,55 @@ +#ifndef SHARDING_QUERY_PLAN +#define SHARDING_QUERY_PLAN + +#include "glib-ext.h" + +struct _group_sql_pair { + /* group names references sharding_partition_t.group_name */ + const GString *gp_name; + + /* sql references con->orig_sql or sharding_plan_t.sql_list */ + const GString *sql; +}; + +enum sharding_table_type_t { + GLOBAL_TABLE, + SHARDED_TABLE, + SINGLE_TABLE, +}; + +typedef struct sharding_plan_t { + GPtrArray *groups; /* GPtrArray */ + + GList *sql_list; /* GList */ + + GList *mapping; /* GList */ + + gboolean is_modified; + const GString *orig_sql; + const GString *modified_sql; + enum sharding_table_type_t table_type; +} sharding_plan_t; + +sharding_plan_t *sharding_plan_new(const GString *orig_sql); + +void sharding_plan_free(sharding_plan_t *); + +void sharding_plan_set_modified_sql(sharding_plan_t *, GString *sql); + +const GString *sharding_plan_get_sql(sharding_plan_t *, const GString *group); + +gboolean sharding_plan_has_group(sharding_plan_t *plan, const GString *gp); + +/* use orig_sql or modified sql */ +void sharding_plan_add_group(sharding_plan_t *, GString *gp_name); + +void sharding_plan_add_groups(sharding_plan_t *, GPtrArray *groups); + +void sharding_plan_clear_group(sharding_plan_t *); + +/* use group-specific sql */ +void sharding_plan_add_group_sql(sharding_plan_t *, GString *gp_name, GString *sql); + +void sharding_plan_sort_groups(sharding_plan_t *); + +#endif /* SHARDING_QUERY_PLAN */ diff --git a/src/sys-pedantic.h b/src/sys-pedantic.h new file mode 100644 index 0000000..c15aa24 --- /dev/null +++ b/src/sys-pedantic.h @@ -0,0 +1,20 @@ +#ifndef _SYS_PEDANTIC_H_ +#define _SYS_PEDANTIC_H_ + +/** @file + * a set of macros to make programming C easier + */ + +#ifdef UNUSED_PARAM +#elif defined(__GNUC__) +# define UNUSED_PARAM(x) UNUSED_ ## x __attribute__((unused)) +#elif defined(__LCLINT__) +# define UNUSED_PARAM(x) /*@unused@*/ x +#else +# define UNUSED_PARAM(x) x +#endif + +#define F_SIZE_T "%"G_GSIZE_FORMAT +#define F_U64 "%"G_GUINT64_FORMAT + +#endif diff --git a/tools/lemon.c b/tools/lemon.c new file mode 100644 index 0000000..d40773d --- /dev/null +++ b/tools/lemon.c @@ -0,0 +1,5432 @@ +/* +** This file contains all sources (including headers) to the LEMON +** LALR(1) parser generator. The sources have been combined into a +** single file to make it easy to include LEMON in the source tree +** and Makefile of another program. +** +** The author of this program disclaims copyright. +*/ +#include +#include +#include +#include +#include +#include + +#define ISSPACE(X) isspace((unsigned char)(X)) +#define ISDIGIT(X) isdigit((unsigned char)(X)) +#define ISALNUM(X) isalnum((unsigned char)(X)) +#define ISALPHA(X) isalpha((unsigned char)(X)) +#define ISUPPER(X) isupper((unsigned char)(X)) +#define ISLOWER(X) islower((unsigned char)(X)) + + +#ifndef __WIN32__ +# if defined(_WIN32) || defined(WIN32) +# define __WIN32__ +# endif +#endif + +#ifdef __WIN32__ +#ifdef __cplusplus +extern "C" { +#endif +extern int access(const char *path, int mode); +#ifdef __cplusplus +} +#endif +#else +#include +#endif + +/* #define PRIVATE static */ +#define PRIVATE + +#ifdef TEST +#define MAXRHS 5 /* Set low to exercise exception code */ +#else +#define MAXRHS 1000 +#endif + +static int showPrecedenceConflict = 0; +static char *msort(char *,char **,int(*)(const char *,const char *)); + +/* +** Compilers are getting increasingly pedantic about type conversions +** as C evolves ever closer to Ada.... To work around the latest problems +** we have to define the following variant of strlen(). +*/ +#define lemonStrlen(X) ((int)strlen(X)) + +/* +** Compilers are starting to complain about the use of sprintf() and strcpy(), +** saying they are unsafe. So we define our own versions of those routines too. +** +** There are three routines here: lemon_sprintf(), lemon_vsprintf(), and +** lemon_addtext(). The first two are replacements for sprintf() and vsprintf(). +** The third is a helper routine for vsnprintf() that adds texts to the end of a +** buffer, making sure the buffer is always zero-terminated. +** +** The string formatter is a minimal subset of stdlib sprintf() supporting only +** a few simply conversions: +** +** %d +** %s +** %.*s +** +*/ +static void lemon_addtext( + char *zBuf, /* The buffer to which text is added */ + int *pnUsed, /* Slots of the buffer used so far */ + const char *zIn, /* Text to add */ + int nIn, /* Bytes of text to add. -1 to use strlen() */ + int iWidth /* Field width. Negative to left justify */ +) { + if (nIn<0) for(nIn=0; zIn[nIn]; nIn++) {} + while(iWidth>nIn) { zBuf[(*pnUsed)++] = ' '; iWidth--; } + if (nIn == 0) return; + memcpy(&zBuf[*pnUsed], zIn, nIn); + *pnUsed += nIn; + while((-iWidth)>nIn) { zBuf[(*pnUsed)++] = ' '; iWidth++; } + zBuf[*pnUsed] = 0; +} +static int lemon_vsprintf(char *str, const char *zFormat, va_list ap) { + int i, j, k, c; + int nUsed = 0; + const char *z; + char zTemp[50]; + str[0] = 0; + for(i=j=0; (c = zFormat[i])!=0; i++) { + if (c == '%' ) { + int iWidth = 0; + lemon_addtext(str, &nUsed, &zFormat[j], i-j, 0); + c = zFormat[++i]; + if (ISDIGIT(c) || (c == '-' && ISDIGIT(zFormat[i+1])) ) { + if (c == '-' ) i++; + while(ISDIGIT(zFormat[i]) ) iWidth = iWidth *10 + zFormat[i++] - '0'; + if (c == '-' ) iWidth = -iWidth; + c = zFormat[i]; + } + if (c == 'd' ) { + int v = va_arg(ap, int); + if (v<0) { + lemon_addtext(str, &nUsed, "-", 1, iWidth); + v = -v; + } else if (v == 0) { + lemon_addtext(str, &nUsed, "0", 1, iWidth); + } + k = 0; + while(v>0) { + k++; + zTemp[sizeof(zTemp)-k] = (v%10) + '0'; + v /= 10; + } + lemon_addtext(str, &nUsed, &zTemp[sizeof(zTemp)-k], k, iWidth); + } else if (c == 's' ) { + z = va_arg(ap, const char *); + lemon_addtext(str, &nUsed, z, -1, iWidth); + } else if (c == '.' && memcmp(&zFormat[i], ".*s", 3) == 0) { + i += 2; + k = va_arg(ap, int); + z = va_arg(ap, const char *); + lemon_addtext(str, &nUsed, z, k, iWidth); + } else if (c == '%' ) { + lemon_addtext(str, &nUsed, "%", 1, 0); + } else{ + fprintf(stderr, "illegal format\n"); + exit(1); + } + j = i+1; + } + } + lemon_addtext(str, &nUsed, &zFormat[j], i-j, 0); + return nUsed; +} +static int lemon_sprintf(char *str, const char *format, ...) { + va_list ap; + int rc; + va_start(ap, format); + rc = lemon_vsprintf(str, format, ap); + va_end(ap); + return rc; +} +static void lemon_strcpy(char *dest, const char *src) { + while((*(dest++) = *(src++))!=0) {} +} +static void lemon_strcat(char *dest, const char *src) { + while(*dest) dest++; + lemon_strcpy(dest, src); +} + + +/* a few forward declarations... */ +struct rule; +struct lemon; +struct action; + +static struct action *Action_new(void); +static struct action *Action_sort(struct action *); + +/********** From the file "build.h" ************************************/ +void FindRulePrecedences(); +void FindFirstSets(); +void FindStates(); +void FindLinks(); +void FindFollowSets(); +void FindActions(); + +/********* From the file "configlist.h" *********************************/ +void Configlist_init(void); +struct config *Configlist_add(struct rule *, int); +struct config *Configlist_addbasis(struct rule *, int); +void Configlist_closure(struct lemon *); +void Configlist_sort(void); +void Configlist_sortbasis(void); +struct config *Configlist_return(void); +struct config *Configlist_basis(void); +void Configlist_eat(struct config *); +void Configlist_reset(void); + +/********* From the file "error.h" ***************************************/ +void ErrorMsg(const char *, int,const char *, ...); + +/****** From the file "option.h" ******************************************/ +enum option_type { OPT_FLAG=1, OPT_INT, OPT_DBL, OPT_STR, + OPT_FFLAG, OPT_FINT, OPT_FDBL, OPT_FSTR}; +struct s_options { + enum option_type type; + const char *label; + char *arg; + const char *message; +}; +int OptInit(char **,struct s_options *,FILE *); +int OptNArgs(void); +char *OptArg(int); +void OptErr(int); +void OptPrint(void); + +/******** From the file "parse.h" *****************************************/ +void Parse(struct lemon *lemp); + +/********* From the file "plink.h" ***************************************/ +struct plink *Plink_new(void); +void Plink_add(struct plink **, struct config *); +void Plink_copy(struct plink **, struct plink *); +void Plink_delete(struct plink *); + +/********** From the file "report.h" *************************************/ +void Reprint(struct lemon *); +void ReportOutput(struct lemon *); +void ReportTable(struct lemon *, int); +void ReportHeader(struct lemon *); +void CompressTables(struct lemon *); +void ResortStates(struct lemon *); + +/********** From the file "set.h" ****************************************/ +void SetSize(int); /* All sets will be of size N */ +char *SetNew(void); /* A new set for element 0..N */ +void SetFree(char *); /* Deallocate a set */ +int SetAdd(char *,int); /* Add element to a set */ +int SetUnion(char *,char *); /* A <- A U B, thru element N */ +#define SetFind(X,Y) (X[Y]) /* True if Y is in set X */ + +/********** From the file "struct.h" *************************************/ +/* +** Principal data structures for the LEMON parser generator. +*/ + +typedef enum {LEMON_FALSE=0, LEMON_TRUE} Boolean; + +/* Symbols (terminals and nonterminals) of the grammar are stored +** in the following: */ +enum symbol_type { + TERMINAL, + NONTERMINAL, + MULTITERMINAL +}; +enum e_assoc { + LEFT, + RIGHT, + NONE, + UNK +}; +struct symbol { + const char *name; /* Name of the symbol */ + int index; /* Index number for this symbol */ + enum symbol_type type; /* Symbols are all either TERMINALS or NTs */ + struct rule *rule; /* Linked list of rules of this (if an NT) */ + struct symbol *fallback; /* fallback token in case this token doesn't parse */ + int prec; /* Precedence if defined (-1 otherwise) */ + enum e_assoc assoc; /* Associativity if precedence is defined */ + char *firstset; /* First-set for all rules of this symbol */ + Boolean lambda; /* True if NT and can generate an empty string */ + int useCnt; /* Number of times used */ + char *destructor; /* Code which executes whenever this symbol is + ** popped from the stack during error processing */ + int destructor_emitted; /* Is the destructor code emitted */ + int destLineno; /* Line number for start of destructor */ + char *datatype; /* The data type of information held by this + ** object. Only used if type == NONTERMINAL */ + int dtnum; /* The data type number. In the parser, the value + ** stack is a union. The .yy%d element of this + ** union is the correct data type for this object */ + /* The following fields are used by MULTITERMINALs only */ + int nsubsym; /* Number of constituent symbols in the MULTI */ + struct symbol **subsym; /* Array of constituent symbols */ +}; + +/* Each production rule in the grammar is stored in the following +** structure. */ +struct rule { + struct symbol *lhs; /* Left-hand side of the rule */ + const char *lhsalias; /* Alias for the LHS (NULL if none) */ + int lhsStart; /* True if left-hand side is the start symbol */ + int ruleline; /* Line number for the rule */ + int nrhs; /* Number of RHS symbols */ + struct symbol **rhs; /* The RHS symbols */ + const char **rhsalias; /* An alias for each RHS symbol (NULL if none) */ + int line; /* Line number at which code begins */ + const char *code; /* The code executed when this rule is reduced */ + const char *codePrefix; /* Setup code before code[] above */ + const char *codeSuffix; /* Breakdown code after code[] above */ + struct symbol *precsym; /* Precedence symbol for this rule */ + int index; /* An index number for this rule */ + int iRule; /* Rule number as used in the generated tables */ + Boolean canReduce; /* True if this rule is ever reduced */ + struct rule *nextlhs; /* Next rule with the same LHS */ + struct rule *next; /* Next rule in the global list */ +}; + +/* A configuration is a production rule of the grammar together with +** a mark (dot) showing how much of that rule has been processed so far. +** Configurations also contain a follow-set which is a list of terminal +** symbols which are allowed to immediately follow the end of the rule. +** Every configuration is recorded as an instance of the following: */ +enum cfgstatus { + COMPLETE, + INCOMPLETE +}; +struct config { + struct rule *rp; /* The rule upon which the configuration is based */ + int dot; /* The parse point */ + char *fws; /* Follow-set for this configuration only */ + struct plink *fplp; /* Follow-set forward propagation links */ + struct plink *bplp; /* Follow-set backwards propagation links */ + struct state *stp; /* Pointer to state which contains this */ + enum cfgstatus status; /* used during followset and shift computations */ + struct config *next; /* Next configuration in the state */ + struct config *bp; /* The next basis configuration */ +}; + +enum e_action { + SHIFT, + ACCEPT, + REDUCE, + ERROR, + SSCONFLICT, /* A shift/shift conflict */ + SRCONFLICT, /* Was a reduce, but part of a conflict */ + RRCONFLICT, /* Was a reduce, but part of a conflict */ + SH_RESOLVED, /* Was a shift. Precedence resolved conflict */ + RD_RESOLVED, /* Was reduce. Precedence resolved conflict */ + NOT_USED, /* Deleted by compression */ + SHIFTREDUCE /* Shift first, then reduce */ +}; + +/* Every shift or reduce operation is stored as one of the following */ +struct action { + struct symbol *sp; /* The look-ahead symbol */ + enum e_action type; + union { + struct state *stp; /* The new state, if a shift */ + struct rule *rp; /* The rule, if a reduce */ + } x; + struct action *next; /* Next action for this state */ + struct action *collide; /* Next action with the same hash */ +}; + +/* Each state of the generated parser's finite state machine +** is encoded as an instance of the following structure. */ +struct state { + struct config *bp; /* The basis configurations for this state */ + struct config *cfp; /* All configurations in this set */ + int statenum; /* Sequential number for this state */ + struct action *ap; /* Array of actions for this state */ + int nTknAct, nNtAct; /* Number of actions on terminals and nonterminals */ + int iTknOfst, iNtOfst; /* yy_action[] offset for terminals and nonterms */ + int iDfltReduce; /* Default action is to REDUCE by this rule */ + struct rule *pDfltReduce;/* The default REDUCE rule. */ + int autoReduce; /* True if this is an auto-reduce state */ +}; +#define NO_OFFSET (-2147483647) + +/* A followset propagation link indicates that the contents of one +** configuration followset should be propagated to another whenever +** the first changes. */ +struct plink { + struct config *cfp; /* The configuration to which linked */ + struct plink *next; /* The next propagate link */ +}; + +/* The state vector for the entire parser generator is recorded as +** follows. (LEMON uses no global variables and makes little use of +** static variables. Fields in the following structure can be thought +** of as begin global variables in the program.) */ +struct lemon { + struct state **sorted; /* Table of states sorted by state number */ + struct rule *rule; /* List of all rules */ + struct rule *startRule; /* First rule */ + int nstate; /* Number of states */ + int nxstate; /* nstate with tail degenerate states removed */ + int nrule; /* Number of rules */ + int nsymbol; /* Number of terminal and nonterminal symbols */ + int nterminal; /* Number of terminal symbols */ + struct symbol **symbols; /* Sorted array of pointers to symbols */ + int errorcnt; /* Number of errors */ + struct symbol *errsym; /* The error symbol */ + struct symbol *wildcard; /* Token that matches anything */ + char *name; /* Name of the generated parser */ + char *arg; /* Declaration of the 3th argument to parser */ + char *tokentype; /* Type of terminal symbols in the parser stack */ + char *vartype; /* The default type of non-terminal symbols */ + char *start; /* Name of the start symbol for the grammar */ + char *stacksize; /* Size of the parser stack */ + char *include; /* Code to put at the start of the C file */ + char *error; /* Code to execute when an error is seen */ + char *overflow; /* Code to execute on a stack overflow */ + char *failure; /* Code to execute on parser failure */ + char *accept; /* Code to execute when the parser excepts */ + char *extracode; /* Code appended to the generated file */ + char *tokendest; /* Code to execute to destroy token data */ + char *vardest; /* Code for the default non-terminal destructor */ + char *filename; /* Name of the input file */ + char *outname; /* Name of the current output file, may be .c .h or .out */ + char *tokenprefix; /* A prefix added to token names in the .h file */ + int nconflict; /* Number of parsing conflicts */ + int nactiontab; /* Number of entries in the yy_action[] table */ + int tablesize; /* Total table size of all tables in bytes */ + int basisflag; /* Print only basis configurations */ + int has_fallback; /* True if any %fallback is seen in the grammar */ + int nolinenosflag; /* True if #line statements should not be printed */ + char *argv0; /* Name of the program */ +}; + +#define MemoryCheck(X) if ((X) == 0) { \ + extern void memory_error(); \ + memory_error(); \ +} + +/**************** From the file "table.h" *********************************/ +/* +** All code in this file has been automatically generated +** from a specification in the file +** "table.q" +** by the associative array code building program "aagen". +** Do not edit this file! Instead, edit the specification +** file, then rerun aagen. +*/ +/* +** Code for processing tables in the LEMON parser generator. +*/ +/* Routines for handling a strings */ + +const char *Strsafe(const char *); + +void Strsafe_init(void); +int Strsafe_insert(const char *); +const char *Strsafe_find(const char *); + +/* Routines for handling symbols of the grammar */ + +struct symbol *Symbol_new(const char *); +int Symbolcmpp(const void *, const void *); +void Symbol_init(void); +int Symbol_insert(struct symbol *, const char *); +struct symbol *Symbol_find(const char *); +struct symbol *Symbol_Nth(int); +int Symbol_count(void); +struct symbol **Symbol_arrayof(void); + +/* Routines to manage the state table */ + +int Configcmp(const char *, const char *); +struct state *State_new(void); +void State_init(void); +int State_insert(struct state *, struct config *); +struct state *State_find(struct config *); +struct state **State_arrayof(/* */); + +/* Routines used for efficiency in Configlist_add */ + +void Configtable_init(void); +int Configtable_insert(struct config *); +struct config *Configtable_find(struct config *); +void Configtable_clear(int(*)(struct config *)); + +/****************** From the file "action.c" *******************************/ +/* +** Routines processing parser actions in the LEMON parser generator. +*/ + +/* Allocate a new parser action */ +static struct action *Action_new(void) { + static struct action *freelist = 0; + struct action *newaction; + + if (freelist == 0) { + int i; + int amt = 100; + freelist = (struct action *)calloc(amt, sizeof(struct action)); + if (freelist == 0) { + fprintf(stderr,"Unable to allocate memory for a new parser action."); + exit(1); + } + for(i = 0 ; i < amt-1; i++) freelist[i].next = &freelist[i+1]; + freelist[amt-1].next = 0; + } + newaction = freelist; + freelist = freelist->next; + return newaction; +} + +/* Compare two actions for sorting purposes. Return negative, zero, or +** positive if the first action is less than, equal to, or greater than +** the first +*/ +static int actioncmp( + struct action *ap1, + struct action *ap2 +) { + int rc; + rc = ap1->sp->index - ap2->sp->index; + if (rc == 0) { + rc = (int)ap1->type - (int)ap2->type; + } + if (rc == 0 && (ap1->type == REDUCE || ap1->type == SHIFTREDUCE) ) { + rc = ap1->x.rp->index - ap2->x.rp->index; + } + if (rc == 0) { + rc = (int) (ap2 - ap1); + } + return rc; +} + +/* Sort parser actions */ +static struct action *Action_sort( + struct action *ap +) { + ap = (struct action *)msort((char *)ap,(char **)&ap->next, + (int(*)(const char *,const char *))actioncmp); + return ap; +} + +void Action_add( + struct action **app, + enum e_action type, + struct symbol *sp, + char *arg +) { + struct action *newaction; + newaction = Action_new(); + newaction->next = *app; + *app = newaction; + newaction->type = type; + newaction->sp = sp; + if (type == SHIFT) { + newaction->x.stp = (struct state *)arg; + } else{ + newaction->x.rp = (struct rule *)arg; + } +} +/********************** New code to implement the "acttab" module ***********/ +/* +** This module implements routines use to construct the yy_action[] table. +*/ + +/* +** The state of the yy_action table under construction is an instance of +** the following structure. +** +** The yy_action table maps the pair (state_number, lookahead) into an +** action_number. The table is an array of integers pairs. The state_number +** determines an initial offset into the yy_action array. The lookahead +** value is then added to this initial offset to get an index X into the +** yy_action array. If the aAction[X].lookahead equals the value of the +** of the lookahead input, then the value of the action_number output is +** aAction[X].action. If the lookaheads do not match then the +** default action for the state_number is returned. +** +** All actions associated with a single state_number are first entered +** into aLookahead[] using multiple calls to acttab_action(). Then the +** actions for that single state_number are placed into the aAction[] +** array with a single call to acttab_insert(). The acttab_insert() call +** also resets the aLookahead[] array in preparation for the next +** state number. +*/ +struct lookahead_action { + int lookahead; /* Value of the lookahead token */ + int action; /* Action to take on the given lookahead */ +}; +typedef struct acttab acttab; +struct acttab { + int nAction; /* Number of used slots in aAction[] */ + int nActionAlloc; /* Slots allocated for aAction[] */ + struct lookahead_action + *aAction, /* The yy_action[] table under construction */ + *aLookahead; /* A single new transaction set */ + int mnLookahead; /* Minimum aLookahead[].lookahead */ + int mnAction; /* Action associated with mnLookahead */ + int mxLookahead; /* Maximum aLookahead[].lookahead */ + int nLookahead; /* Used slots in aLookahead[] */ + int nLookaheadAlloc; /* Slots allocated in aLookahead[] */ +}; + +/* Return the number of entries in the yy_action table */ +#define acttab_size(X) ((X)->nAction) + +/* The value for the N-th entry in yy_action */ +#define acttab_yyaction(X,N) ((X)->aAction[N].action) + +/* The value for the N-th entry in yy_lookahead */ +#define acttab_yylookahead(X,N) ((X)->aAction[N].lookahead) + +/* Free all memory associated with the given acttab */ +void acttab_free(acttab *p) { + free( p->aAction); + free( p->aLookahead); + free( p); +} + +/* Allocate a new acttab structure */ +acttab *acttab_alloc(void) { + acttab *p = (acttab *) calloc( 1, sizeof(*p) ); + if (p == 0) { + fprintf(stderr,"Unable to allocate memory for a new acttab."); + exit(1); + } + memset(p, 0, sizeof(*p)); + return p; +} + +/* Add a new action to the current transaction set. +** +** This routine is called once for each lookahead for a particular +** state. +*/ +void acttab_action(acttab *p, int lookahead, int action) { + if (p->nLookahead>=p->nLookaheadAlloc) { + p->nLookaheadAlloc += 25; + p->aLookahead = (struct lookahead_action *) realloc( p->aLookahead, + sizeof(p->aLookahead[0])*p->nLookaheadAlloc); + if (p->aLookahead == 0) { + fprintf(stderr,"malloc failed\n"); + exit(1); + } + } + if (p->nLookahead == 0) { + p->mxLookahead = lookahead; + p->mnLookahead = lookahead; + p->mnAction = action; + } else{ + if (p->mxLookaheadmxLookahead = lookahead; + if (p->mnLookahead>lookahead) { + p->mnLookahead = lookahead; + p->mnAction = action; + } + } + p->aLookahead[p->nLookahead].lookahead = lookahead; + p->aLookahead[p->nLookahead].action = action; + p->nLookahead++; +} + +/* +** Add the transaction set built up with prior calls to acttab_action() +** into the current action table. Then reset the transaction set back +** to an empty set in preparation for a new round of acttab_action() calls. +** +** Return the offset into the action table of the new transaction. +*/ +int acttab_insert(acttab *p) { + int i, j, k, n; + assert( p->nLookahead>0); + + /* Make sure we have enough space to hold the expanded action table + ** in the worst case. The worst case occurs if the transaction set + ** must be appended to the current action table + */ + n = p->mxLookahead + 1; + if (p->nAction + n >= p->nActionAlloc) { + int oldAlloc = p->nActionAlloc; + p->nActionAlloc = p->nAction + n + p->nActionAlloc + 20; + p->aAction = (struct lookahead_action *) realloc( p->aAction, + sizeof(p->aAction[0])*p->nActionAlloc); + if (p->aAction == 0) { + fprintf(stderr,"malloc failed\n"); + exit(1); + } + for(i=oldAlloc; i < p->nActionAlloc; i++) { + p->aAction[i].lookahead = -1; + p->aAction[i].action = -1; + } + } + + /* Scan the existing action table looking for an offset that is a + ** duplicate of the current transaction set. Fall out of the loop + ** if and when the duplicate is found. + ** + ** i is the index in p->aAction[] where p->mnLookahead is inserted. + */ + for(i=p->nAction-1; i >= 0; i--) { + if (p->aAction[i].lookahead == p->mnLookahead) { + /* All lookaheads and actions in the aLookahead[] transaction + ** must match against the candidate aAction[i] entry. */ + if (p->aAction[i].action!=p->mnAction) continue; + for(j=0; jnLookahead; j++) { + k = p->aLookahead[j].lookahead - p->mnLookahead + i; + if (k<0 || k>=p->nAction) break; + if (p->aLookahead[j].lookahead!=p->aAction[k].lookahead) break; + if (p->aLookahead[j].action!=p->aAction[k].action) break; + } + if (jnLookahead) continue; + + /* No possible lookahead value that is not in the aLookahead[] + ** transaction is allowed to match aAction[i] */ + n = 0; + for(j=0; jnAction; j++) { + if (p->aAction[j].lookahead<0) continue; + if (p->aAction[j].lookahead == j+p->mnLookahead-i) n++; + } + if (n == p->nLookahead) { + break; /* An exact match is found at offset i */ + } + } + } + + /* If no existing offsets exactly match the current transaction, find an + ** an empty offset in the aAction[] table in which we can add the + ** aLookahead[] transaction. + */ + if (i < 0) { + /* Look for holes in the aAction[] table that fit the current + ** aLookahead[] transaction. Leave i set to the offset of the hole. + ** If no holes are found, i is left at p->nAction, which means the + ** transaction will be appended. */ + for(i = 0 ; i < p->nActionAlloc - p->mxLookahead; i++) { + if (p->aAction[i].lookahead<0) { + for(j=0; jnLookahead; j++) { + k = p->aLookahead[j].lookahead - p->mnLookahead + i; + if (k<0) break; + if (p->aAction[k].lookahead>=0) break; + } + if (jnLookahead) continue; + for(j=0; jnAction; j++) { + if (p->aAction[j].lookahead == j+p->mnLookahead-i) break; + } + if (j == p->nAction) { + break; /* Fits in empty slots */ + } + } + } + } + /* Insert transaction set at index i. */ + for(j=0; jnLookahead; j++) { + k = p->aLookahead[j].lookahead - p->mnLookahead + i; + p->aAction[k] = p->aLookahead[j]; + if (k>=p->nAction) p->nAction = k+1; + } + p->nLookahead = 0; + + /* Return the offset that is added to the lookahead in order to get the + ** index into yy_action of the action */ + return i - p->mnLookahead; +} + +/********************** From the file "build.c" *****************************/ +/* +** Routines to construction the finite state machine for the LEMON +** parser generator. +*/ + +/* Find a precedence symbol of every rule in the grammar. +** +** Those rules which have a precedence symbol coded in the input +** grammar using the "[symbol]" construct will already have the +** rp->precsym field filled. Other rules take as their precedence +** symbol the first RHS symbol with a defined precedence. If there +** are not RHS symbols with a defined precedence, the precedence +** symbol field is left blank. +*/ +void FindRulePrecedences(struct lemon *xp) +{ + struct rule *rp; + for(rp=xp->rule; rp; rp=rp->next) { + if (rp->precsym == 0) { + int i, j; + for(i = 0 ; i < rp->nrhs && rp->precsym == 0; i++) { + struct symbol *sp = rp->rhs[i]; + if (sp->type == MULTITERMINAL) { + for(j=0; jnsubsym; j++) { + if (sp->subsym[j]->prec>=0) { + rp->precsym = sp->subsym[j]; + break; + } + } + } else if (sp->prec>=0) { + rp->precsym = rp->rhs[i]; + } + } + } + } + return; +} + +/* Find all nonterminals which will generate the empty string. +** Then go back and compute the first sets of every nonterminal. +** The first set is the set of all terminal symbols which can begin +** a string generated by that nonterminal. +*/ +void FindFirstSets(struct lemon *lemp) +{ + int i, j; + struct rule *rp; + int progress; + + for(i = 0 ; i < lemp->nsymbol; i++) { + lemp->symbols[i]->lambda = LEMON_FALSE; + } + for(i=lemp->nterminal; i < lemp->nsymbol; i++) { + lemp->symbols[i]->firstset = SetNew(); + } + + /* First compute all lambdas */ + do{ + progress = 0; + for(rp=lemp->rule; rp; rp=rp->next) { + if (rp->lhs->lambda) continue; + for(i = 0 ; i < rp->nrhs; i++) { + struct symbol *sp = rp->rhs[i]; + assert( sp->type == NONTERMINAL || sp->lambda == LEMON_FALSE); + if (sp->lambda == LEMON_FALSE) break; + } + if (i == rp->nrhs) { + rp->lhs->lambda = LEMON_TRUE; + progress = 1; + } + } + }while(progress); + + /* Now compute all first sets */ + do{ + struct symbol *s1, *s2; + progress = 0; + for(rp=lemp->rule; rp; rp=rp->next) { + s1 = rp->lhs; + for(i = 0 ; i < rp->nrhs; i++) { + s2 = rp->rhs[i]; + if (s2->type == TERMINAL) { + progress += SetAdd(s1->firstset,s2->index); + break; + } else if (s2->type == MULTITERMINAL) { + for(j=0; jnsubsym; j++) { + progress += SetAdd(s1->firstset,s2->subsym[j]->index); + } + break; + } else if (s1==s2) { + if (s1->lambda == LEMON_FALSE) break; + } else{ + progress += SetUnion(s1->firstset,s2->firstset); + if (s2->lambda == LEMON_FALSE) break; + } + } + } + }while(progress); + return; +} + +/* Compute all LR(0) states for the grammar. Links +** are added to between some states so that the LR(1) follow sets +** can be computed later. +*/ +PRIVATE struct state *getstate(struct lemon *); /* forward reference */ +void FindStates(struct lemon *lemp) +{ + struct symbol *sp; + struct rule *rp; + + Configlist_init(); + + /* Find the start symbol */ + if (lemp->start) { + sp = Symbol_find(lemp->start); + if (sp == 0) { + ErrorMsg(lemp->filename,0, +"The specified start symbol \"%s\" is not \ +in a nonterminal of the grammar. \"%s\" will be used as the start \ +symbol instead.",lemp->start,lemp->startRule->lhs->name); + lemp->errorcnt++; + sp = lemp->startRule->lhs; + } + } else{ + sp = lemp->startRule->lhs; + } + + /* Make sure the start symbol doesn't occur on the right-hand side of + ** any rule. Report an error if it does. (YACC would generate a new + ** start symbol in this case.) */ + for(rp=lemp->rule; rp; rp=rp->next) { + int i; + for(i = 0 ; i < rp->nrhs; i++) { + if (rp->rhs[i] == sp) { /* FIX ME: Deal with multiterminals */ + ErrorMsg(lemp->filename,0, +"The start symbol \"%s\" occurs on the \ +right-hand side of a rule. This will result in a parser which \ +does not work properly.",sp->name); + lemp->errorcnt++; + } + } + } + + /* The basis configuration set for the first state + ** is all rules which have the start symbol as their + ** left-hand side */ + for(rp=sp->rule; rp; rp=rp->nextlhs) { + struct config *newcfp; + rp->lhsStart = 1; + newcfp = Configlist_addbasis(rp,0); + SetAdd(newcfp->fws,0); + } + + /* Compute the first state. All other states will be + ** computed automatically during the computation of the first one. + ** The returned pointer to the first state is not used. */ + (void)getstate(lemp); + return; +} + +/* Return a pointer to a state which is described by the configuration +** list which has been built from calls to Configlist_add. +*/ +PRIVATE void buildshifts(struct lemon *, struct state *); /* Forwd ref */ +PRIVATE struct state *getstate(struct lemon *lemp) +{ + struct config *cfp, *bp; + struct state *stp; + + /* Extract the sorted basis of the new state. The basis was constructed + ** by prior calls to "Configlist_addbasis()". */ + Configlist_sortbasis(); + bp = Configlist_basis(); + + /* Get a state with the same basis */ + stp = State_find(bp); + if (stp) { + /* A state with the same basis already exists! Copy all the follow-set + ** propagation links from the state under construction into the + ** preexisting state, then return a pointer to the preexisting state */ + struct config *x, *y; + for(x=bp, y=stp->bp; x && y; x=x->bp, y=y->bp) { + Plink_copy(&y->bplp,x->bplp); + Plink_delete(x->fplp); + x->fplp = x->bplp = 0; + } + cfp = Configlist_return(); + Configlist_eat(cfp); + } else{ + /* This really is a new state. Construct all the details */ + Configlist_closure(lemp); /* Compute the configuration closure */ + Configlist_sort(); /* Sort the configuration closure */ + cfp = Configlist_return(); /* Get a pointer to the config list */ + stp = State_new(); /* A new state structure */ + MemoryCheck(stp); + stp->bp = bp; /* Remember the configuration basis */ + stp->cfp = cfp; /* Remember the configuration closure */ + stp->statenum = lemp->nstate++; /* Every state gets a sequence number */ + stp->ap = 0; /* No actions, yet. */ + State_insert(stp,stp->bp); /* Add to the state table */ + buildshifts(lemp,stp); /* Recursively compute successor states */ + } + return stp; +} + +/* +** Return true if two symbols are the same. +*/ +int same_symbol(struct symbol *a, struct symbol *b) +{ + int i; + if (a == b) return 1; + if (a->type!=MULTITERMINAL) return 0; + if (b->type!=MULTITERMINAL) return 0; + if (a->nsubsym!=b->nsubsym) return 0; + for(i = 0 ; i < a->nsubsym; i++) { + if (a->subsym[i]!=b->subsym[i] ) return 0; + } + return 1; +} + +/* Construct all successor states to the given state. A "successor" +** state is any state which can be reached by a shift action. +*/ +PRIVATE void buildshifts(struct lemon *lemp, struct state *stp) +{ + struct config *cfp; /* For looping thru the config closure of "stp" */ + struct config *bcfp; /* For the inner loop on config closure of "stp" */ + struct config *newcfg; /* */ + struct symbol *sp; /* Symbol following the dot in configuration "cfp" */ + struct symbol *bsp; /* Symbol following the dot in configuration "bcfp" */ + struct state *newstp; /* A pointer to a successor state */ + + /* Each configuration becomes complete after it contibutes to a successor + ** state. Initially, all configurations are incomplete */ + for(cfp=stp->cfp; cfp; cfp=cfp->next) cfp->status = INCOMPLETE; + + /* Loop through all configurations of the state "stp" */ + for(cfp=stp->cfp; cfp; cfp=cfp->next) { + if (cfp->status == COMPLETE) continue; /* Already used by inner loop */ + if (cfp->dot>=cfp->rp->nrhs) continue; /* Can't shift this config */ + Configlist_reset(); /* Reset the new config set */ + sp = cfp->rp->rhs[cfp->dot]; /* Symbol after the dot */ + + /* For every configuration in the state "stp" which has the symbol "sp" + ** following its dot, add the same configuration to the basis set under + ** construction but with the dot shifted one symbol to the right. */ + for(bcfp=cfp; bcfp; bcfp=bcfp->next) { + if (bcfp->status == COMPLETE) continue; /* Already used */ + if (bcfp->dot>=bcfp->rp->nrhs) continue; /* Can't shift this one */ + bsp = bcfp->rp->rhs[bcfp->dot]; /* Get symbol after dot */ + if (!same_symbol(bsp,sp) ) continue; /* Must be same as for "cfp" */ + bcfp->status = COMPLETE; /* Mark this config as used */ + newcfg = Configlist_addbasis(bcfp->rp,bcfp->dot+1); + Plink_add(&newcfg->bplp,bcfp); + } + + /* Get a pointer to the state described by the basis configuration set + ** constructed in the preceding loop */ + newstp = getstate(lemp); + + /* The state "newstp" is reached from the state "stp" by a shift action + ** on the symbol "sp" */ + if (sp->type == MULTITERMINAL) { + int i; + for(i = 0 ; i < sp->nsubsym; i++) { + Action_add(&stp->ap,SHIFT,sp->subsym[i],(char *)newstp); + } + } else{ + Action_add(&stp->ap,SHIFT,sp,(char *)newstp); + } + } +} + +/* +** Construct the propagation links +*/ +void FindLinks(struct lemon *lemp) +{ + int i; + struct config *cfp, *other; + struct state *stp; + struct plink *plp; + + /* Housekeeping detail: + ** Add to every propagate link a pointer back to the state to + ** which the link is attached. */ + for(i = 0 ; i < lemp->nstate; i++) { + stp = lemp->sorted[i]; + for(cfp=stp->cfp; cfp; cfp=cfp->next) { + cfp->stp = stp; + } + } + + /* Convert all backlinks into forward links. Only the forward + ** links are used in the follow-set computation. */ + for(i = 0 ; i < lemp->nstate; i++) { + stp = lemp->sorted[i]; + for(cfp=stp->cfp; cfp; cfp=cfp->next) { + for(plp=cfp->bplp; plp; plp=plp->next) { + other = plp->cfp; + Plink_add(&other->fplp,cfp); + } + } + } +} + +/* Compute all followsets. +** +** A followset is the set of all symbols which can come immediately +** after a configuration. +*/ +void FindFollowSets(struct lemon *lemp) +{ + int i; + struct config *cfp; + struct plink *plp; + int progress; + int change; + + for(i = 0 ; i < lemp->nstate; i++) { + for(cfp=lemp->sorted[i]->cfp; cfp; cfp=cfp->next) { + cfp->status = INCOMPLETE; + } + } + + do{ + progress = 0; + for(i = 0 ; i < lemp->nstate; i++) { + for(cfp=lemp->sorted[i]->cfp; cfp; cfp=cfp->next) { + if (cfp->status == COMPLETE) continue; + for(plp=cfp->fplp; plp; plp=plp->next) { + change = SetUnion(plp->cfp->fws,cfp->fws); + if (change) { + plp->cfp->status = INCOMPLETE; + progress = 1; + } + } + cfp->status = COMPLETE; + } + } + }while(progress); +} + +static int resolve_conflict(struct action *,struct action *); + +/* Compute the reduce actions, and resolve conflicts. +*/ +void FindActions(struct lemon *lemp) +{ + int i,j; + struct config *cfp; + struct state *stp; + struct symbol *sp; + struct rule *rp; + + /* Add all of the reduce actions + ** A reduce action is added for each element of the followset of + ** a configuration which has its dot at the extreme right. + */ + for(i = 0 ; i < lemp->nstate; i++) { /* Loop over all states */ + stp = lemp->sorted[i]; + for(cfp=stp->cfp; cfp; cfp=cfp->next) { /* Loop over all configurations */ + if (cfp->rp->nrhs == cfp->dot) { /* Is dot at extreme right? */ + for(j=0; jnterminal; j++) { + if (SetFind(cfp->fws,j) ) { + /* Add a reduce action to the state "stp" which will reduce by the + ** rule "cfp->rp" if the lookahead symbol is "lemp->symbols[j]" */ + Action_add(&stp->ap,REDUCE,lemp->symbols[j],(char *)cfp->rp); + } + } + } + } + } + + /* Add the accepting token */ + if (lemp->start) { + sp = Symbol_find(lemp->start); + if (sp == 0) sp = lemp->startRule->lhs; + } else{ + sp = lemp->startRule->lhs; + } + /* Add to the first state (which is always the starting state of the + ** finite state machine) an action to ACCEPT if the lookahead is the + ** start nonterminal. */ + Action_add(&lemp->sorted[0]->ap,ACCEPT,sp,0); + + /* Resolve conflicts */ + for(i = 0 ; i < lemp->nstate; i++) { + struct action *ap, *nap; + stp = lemp->sorted[i]; + /* assert( stp->ap); */ + stp->ap = Action_sort(stp->ap); + for(ap=stp->ap; ap && ap->next; ap=ap->next) { + for(nap=ap->next; nap && nap->sp == ap->sp; nap=nap->next) { + /* The two actions "ap" and "nap" have the same lookahead. + ** Figure out which one should be used */ + lemp->nconflict += resolve_conflict(ap,nap); + } + } + } + + /* Report an error for each rule that can never be reduced. */ + for(rp=lemp->rule; rp; rp=rp->next) rp->canReduce = LEMON_FALSE; + for(i = 0 ; i < lemp->nstate; i++) { + struct action *ap; + for(ap=lemp->sorted[i]->ap; ap; ap=ap->next) { + if (ap->type == REDUCE) ap->x.rp->canReduce = LEMON_TRUE; + } + } + for(rp=lemp->rule; rp; rp=rp->next) { + if (rp->canReduce) continue; + ErrorMsg(lemp->filename,rp->ruleline,"This rule can not be reduced.\n"); + lemp->errorcnt++; + } +} + +/* Resolve a conflict between the two given actions. If the +** conflict can't be resolved, return non-zero. +** +** NO LONGER TRUE: +** To resolve a conflict, first look to see if either action +** is on an error rule. In that case, take the action which +** is not associated with the error rule. If neither or both +** actions are associated with an error rule, then try to +** use precedence to resolve the conflict. +** +** If either action is a SHIFT, then it must be apx. This +** function won't work if apx->type == REDUCE and apy->type == SHIFT. +*/ +static int resolve_conflict( + struct action *apx, + struct action *apy +) { + struct symbol *spx, *spy; + int errcnt = 0; + assert( apx->sp == apy->sp); /* Otherwise there would be no conflict */ + if (apx->type == SHIFT && apy->type == SHIFT) { + apy->type = SSCONFLICT; + errcnt++; + } + if (apx->type == SHIFT && apy->type == REDUCE) { + spx = apx->sp; + spy = apy->x.rp->precsym; + if (spy == 0 || spx->prec<0 || spy->prec<0) { + /* Not enough precedence information. */ + apy->type = SRCONFLICT; + errcnt++; + } else if (spx->prec>spy->prec) { /* higher precedence wins */ + apy->type = RD_RESOLVED; + } else if (spx->precprec) { + apx->type = SH_RESOLVED; + } else if (spx->prec == spy->prec && spx->assoc == RIGHT) { /* Use operator */ + apy->type = RD_RESOLVED; /* associativity */ + } else if (spx->prec == spy->prec && spx->assoc == LEFT) { /* to break tie */ + apx->type = SH_RESOLVED; + } else{ + assert( spx->prec == spy->prec && spx->assoc == NONE); + apx->type = ERROR; + } + } else if (apx->type == REDUCE && apy->type == REDUCE) { + spx = apx->x.rp->precsym; + spy = apy->x.rp->precsym; + if (spx == 0 || spy == 0 || spx->prec<0 || + spy->prec<0 || spx->prec == spy->prec) { + apy->type = RRCONFLICT; + errcnt++; + } else if (spx->prec>spy->prec) { + apy->type = RD_RESOLVED; + } else if (spx->precprec) { + apx->type = RD_RESOLVED; + } + } else{ + assert( + apx->type == SH_RESOLVED || + apx->type == RD_RESOLVED || + apx->type == SSCONFLICT || + apx->type == SRCONFLICT || + apx->type == RRCONFLICT || + apy->type == SH_RESOLVED || + apy->type == RD_RESOLVED || + apy->type == SSCONFLICT || + apy->type == SRCONFLICT || + apy->type == RRCONFLICT + ); + /* The REDUCE/SHIFT case cannot happen because SHIFTs come before + ** REDUCEs on the list. If we reach this point it must be because + ** the parser conflict had already been resolved. */ + } + return errcnt; +} +/********************* From the file "configlist.c" *************************/ +/* +** Routines to processing a configuration list and building a state +** in the LEMON parser generator. +*/ + +static struct config *freelist = 0; /* List of free configurations */ +static struct config *current = 0; /* Top of list of configurations */ +static struct config **currentend = 0; /* Last on list of configs */ +static struct config *basis = 0; /* Top of list of basis configs */ +static struct config **basisend = 0; /* End of list of basis configs */ + +/* Return a pointer to a new configuration */ +PRIVATE struct config *newconfig() { + struct config *newcfg; + if (freelist == 0) { + int i; + int amt = 3; + freelist = (struct config *)calloc( amt, sizeof(struct config) ); + if (freelist == 0) { + fprintf(stderr,"Unable to allocate memory for a new configuration."); + exit(1); + } + for(i = 0 ; i < amt-1; i++) freelist[i].next = &freelist[i+1]; + freelist[amt-1].next = 0; + } + newcfg = freelist; + freelist = freelist->next; + return newcfg; +} + +/* The configuration "old" is no longer used */ +PRIVATE void deleteconfig(struct config *old) +{ + old->next = freelist; + freelist = old; +} + +/* Initialized the configuration list builder */ +void Configlist_init() { + current = 0; + currentend = ¤t; + basis = 0; + basisend = &basis; + Configtable_init(); + return; +} + +/* Initialized the configuration list builder */ +void Configlist_reset() { + current = 0; + currentend = ¤t; + basis = 0; + basisend = &basis; + Configtable_clear(0); + return; +} + +/* Add another configuration to the configuration list */ +struct config *Configlist_add( + struct rule *rp, /* The rule */ + int dot /* Index into the RHS of the rule where the dot goes */ +) { + struct config *cfp, model; + + assert( currentend!=0); + model.rp = rp; + model.dot = dot; + cfp = Configtable_find(&model); + if (cfp == 0) { + cfp = newconfig(); + cfp->rp = rp; + cfp->dot = dot; + cfp->fws = SetNew(); + cfp->stp = 0; + cfp->fplp = cfp->bplp = 0; + cfp->next = 0; + cfp->bp = 0; + *currentend = cfp; + currentend = &cfp->next; + Configtable_insert(cfp); + } + return cfp; +} + +/* Add a basis configuration to the configuration list */ +struct config *Configlist_addbasis(struct rule *rp, int dot) +{ + struct config *cfp, model; + + assert( basisend!=0); + assert( currentend!=0); + model.rp = rp; + model.dot = dot; + cfp = Configtable_find(&model); + if (cfp == 0) { + cfp = newconfig(); + cfp->rp = rp; + cfp->dot = dot; + cfp->fws = SetNew(); + cfp->stp = 0; + cfp->fplp = cfp->bplp = 0; + cfp->next = 0; + cfp->bp = 0; + *currentend = cfp; + currentend = &cfp->next; + *basisend = cfp; + basisend = &cfp->bp; + Configtable_insert(cfp); + } + return cfp; +} + +/* Compute the closure of the configuration list */ +void Configlist_closure(struct lemon *lemp) +{ + struct config *cfp, *newcfp; + struct rule *rp, *newrp; + struct symbol *sp, *xsp; + int i, dot; + + assert( currentend!=0); + for(cfp=current; cfp; cfp=cfp->next) { + rp = cfp->rp; + dot = cfp->dot; + if (dot>=rp->nrhs) continue; + sp = rp->rhs[dot]; + if (sp->type == NONTERMINAL) { + if (sp->rule == 0 && sp!=lemp->errsym) { + ErrorMsg(lemp->filename,rp->line,"Nonterminal \"%s\" has no rules.", + sp->name); + lemp->errorcnt++; + } + for(newrp=sp->rule; newrp; newrp=newrp->nextlhs) { + newcfp = Configlist_add(newrp,0); + for(i=dot+1; i < rp->nrhs; i++) { + xsp = rp->rhs[i]; + if (xsp->type == TERMINAL) { + SetAdd(newcfp->fws,xsp->index); + break; + } else if (xsp->type == MULTITERMINAL) { + int k; + for(k=0; knsubsym; k++) { + SetAdd(newcfp->fws, xsp->subsym[k]->index); + } + break; + } else{ + SetUnion(newcfp->fws,xsp->firstset); + if (xsp->lambda == LEMON_FALSE) break; + } + } + if (i == rp->nrhs) Plink_add(&cfp->fplp,newcfp); + } + } + } + return; +} + +/* Sort the configuration list */ +void Configlist_sort() { + current = (struct config *)msort((char *)current,(char **)&(current->next), + Configcmp); + currentend = 0; + return; +} + +/* Sort the basis configuration list */ +void Configlist_sortbasis() { + basis = (struct config *)msort((char *)current,(char **)&(current->bp), + Configcmp); + basisend = 0; + return; +} + +/* Return a pointer to the head of the configuration list and +** reset the list */ +struct config *Configlist_return() { + struct config *old; + old = current; + current = 0; + currentend = 0; + return old; +} + +/* Return a pointer to the head of the configuration list and +** reset the list */ +struct config *Configlist_basis() { + struct config *old; + old = basis; + basis = 0; + basisend = 0; + return old; +} + +/* Free all elements of the given configuration list */ +void Configlist_eat(struct config *cfp) +{ + struct config *nextcfp; + for(; cfp; cfp=nextcfp) { + nextcfp = cfp->next; + assert( cfp->fplp == 0); + assert( cfp->bplp == 0); + if (cfp->fws) SetFree(cfp->fws); + deleteconfig(cfp); + } + return; +} +/***************** From the file "error.c" *********************************/ +/* +** Code for printing error message. +*/ + +void ErrorMsg(const char *filename, int lineno, const char *format, ...) { + va_list ap; + fprintf(stderr, "%s:%d: ", filename, lineno); + va_start(ap, format); + vfprintf(stderr,format,ap); + va_end(ap); + fprintf(stderr, "\n"); +} +/**************** From the file "main.c" ************************************/ +/* +** Main program file for the LEMON parser generator. +*/ + +/* Report an out-of-memory condition and abort. This function +** is used mostly by the "MemoryCheck" macro in struct.h +*/ +void memory_error() { + fprintf(stderr,"Out of memory. Aborting...\n"); + exit(1); +} + +static int nDefine = 0; /* Number of -D options on the command line */ +static char **azDefine = 0; /* Name of the -D macros */ + +/* This routine is called with the argument to each -D command-line option. +** Add the macro defined to the azDefine array. +*/ +static void handle_D_option(char *z) { + char **paz; + nDefine++; + azDefine = (char **) realloc(azDefine, sizeof(azDefine[0])*nDefine); + if (azDefine == 0) { + fprintf(stderr,"out of memory\n"); + exit(1); + } + paz = &azDefine[nDefine-1]; + *paz = (char *) malloc( lemonStrlen(z)+1); + if (*paz == 0) { + fprintf(stderr,"out of memory\n"); + exit(1); + } + lemon_strcpy(*paz, z); + for(z=*paz; *z && *z!='='; z++) {} + *z = 0; +} + +static char *user_templatename = NULL; +static void handle_T_option(char *z) { + user_templatename = (char *) malloc( lemonStrlen(z)+1); + if (user_templatename == 0) { + memory_error(); + } + lemon_strcpy(user_templatename, z); +} +static char *user_output_file = NULL; +static void handle_o_option(char *z) { + user_output_file = (char *) malloc( lemonStrlen(z)+1); + if (user_output_file == 0) { + memory_error(); + } + lemon_strcpy(user_output_file, z); +} +static char *user_output_header = NULL; +static void handle_h_option(char *z) { + user_output_header = (char *) malloc( lemonStrlen(z)+1); + if (user_output_header == 0) { + memory_error(); + } + lemon_strcpy(user_output_header, z); +} + +/* Merge together to lists of rules order by rule.iRule */ +static struct rule *Rule_merge(struct rule *pA, struct rule *pB) { + struct rule *pFirst = 0; + struct rule **ppPrev = &pFirst; + while(pA && pB) { + if (pA->iRuleiRule) { + *ppPrev = pA; + ppPrev = &pA->next; + pA = pA->next; + } else{ + *ppPrev = pB; + ppPrev = &pB->next; + pB = pB->next; + } + } + if (pA) { + *ppPrev = pA; + } else{ + *ppPrev = pB; + } + return pFirst; +} + +/* +** Sort a list of rules in order of increasing iRule value +*/ +static struct rule *Rule_sort(struct rule *rp) { + int i; + struct rule *pNext; + struct rule *x[32]; + memset(x, 0, sizeof(x)); + while(rp) { + pNext = rp->next; + rp->next = 0; + for(i = 0 ; i < sizeof(x)/sizeof(x[0]) && x[i]; i++) { + rp = Rule_merge(x[i], rp); + x[i] = 0; + } + x[i] = rp; + rp = pNext; + } + rp = 0; + for(i = 0 ; i < sizeof(x)/sizeof(x[0]); i++) { + rp = Rule_merge(x[i], rp); + } + return rp; +} + +/* forward reference */ +static const char *minimum_size_type(int lwr, int upr, int *pnByte); + +/* Print a single line of the "Parser Stats" output +*/ +static void stats_line(const char *zLabel, int iValue) { + int nLabel = lemonStrlen(zLabel); + printf(" %s%.*s %5d\n", zLabel, + 35-nLabel, "................................", + iValue); +} + +/* The main program. Parse the command line and do it... */ +int main(int argc, char **argv) +{ + static int version = 0; + static int rpflag = 0; + static int basisflag = 0; + static int compress = 0; + static int quiet = 0; + static int statistics = 0; + static int mhflag = 0; + static int nolinenosflag = 0; + static int noResort = 0; + static struct s_options options[] = { + {OPT_FLAG, "b", (char *)&basisflag, "Print only the basis in report."}, + {OPT_FLAG, "c", (char *)&compress, "Don't compress the action table."}, + {OPT_FSTR, "D", (char *)handle_D_option, "Define an %ifdef macro."}, + {OPT_FSTR, "f", 0, "Ignored. (Placeholder for -f compiler options.)"}, + {OPT_FLAG, "g", (char *)&rpflag, "Print grammar without actions."}, + {OPT_FSTR, "I", 0, "Ignored. (Placeholder for '-I' compiler options.)"}, + {OPT_FLAG, "m", (char *)&mhflag, "Output a makeheaders compatible file."}, + {OPT_FLAG, "l", (char *)&nolinenosflag, "Do not print #line statements."}, + {OPT_FSTR, "O", 0, "Ignored. (Placeholder for '-O' compiler options.)"}, + {OPT_FLAG, "p", (char *)&showPrecedenceConflict, + "Show conflicts resolved by precedence rules"}, + {OPT_FLAG, "q", (char *)&quiet, "(Quiet) Don't print the report file."}, + {OPT_FLAG, "r", (char *)&noResort, "Do not sort or renumber states"}, + {OPT_FLAG, "s", (char *)&statistics, + "Print parser stats to standard output."}, + {OPT_FLAG, "x", (char *)&version, "Print the version number."}, + {OPT_FSTR, "T", (char *)handle_T_option, "Specify a template file."}, + {OPT_FSTR, "W", 0, "Ignored. (Placeholder for '-W' compiler options.)"}, + {OPT_FSTR, "o", (char *)handle_o_option, "Specify output c file."}, + {OPT_FSTR, "h", (char *)handle_h_option, "Specify output header file."}, + {OPT_FLAG,0,0,0} + }; + int i; + int exitcode; + struct lemon lem; + struct rule *rp; + + OptInit(argv,options,stderr); + if (version) { + printf("Lemon version 1.0\n"); + exit(0); + } + if (OptNArgs()!=1) { + fprintf(stderr,"Exactly one filename argument is required.\n"); + exit(1); + } + memset(&lem, 0, sizeof(lem)); + lem.errorcnt = 0; + + /* Initialize the machine */ + Strsafe_init(); + Symbol_init(); + State_init(); + lem.argv0 = argv[0]; + lem.filename = OptArg(0); + lem.basisflag = basisflag; + lem.nolinenosflag = nolinenosflag; + Symbol_new("$"); + lem.errsym = Symbol_new("error"); + lem.errsym->useCnt = 0; + + /* Parse the input file */ + Parse(&lem); + if (lem.errorcnt) exit(lem.errorcnt); + if (lem.nrule == 0) { + fprintf(stderr,"Empty grammar.\n"); + exit(1); + } + + /* Count and index the symbols of the grammar */ + Symbol_new("{default}"); + lem.nsymbol = Symbol_count(); + lem.symbols = Symbol_arrayof(); + for(i = 0 ; i < lem.nsymbol; i++) lem.symbols[i]->index = i; + qsort(lem.symbols,lem.nsymbol,sizeof(struct symbol *), Symbolcmpp); + for(i = 0 ; i < lem.nsymbol; i++) lem.symbols[i]->index = i; + while(lem.symbols[i-1]->type == MULTITERMINAL) { i--; } + assert( strcmp(lem.symbols[i-1]->name,"{default}") == 0); + lem.nsymbol = i - 1; + for(i = 1; ISUPPER(lem.symbols[i]->name[0]); i++); + lem.nterminal = i; + + /* Assign sequential rule numbers */ + for(i = 0 , rp=lem.rule; rp; rp=rp->next) { + rp->iRule = rp->code ? i++ : -1; + } + for(rp=lem.rule; rp; rp=rp->next) { + if (rp->iRule<0) rp->iRule = i++; + } + lem.startRule = lem.rule; + lem.rule = Rule_sort(lem.rule); + + /* Generate a reprint of the grammar, if requested on the command line */ + if (rpflag) { + Reprint(&lem); + } else{ + /* Initialize the size for all follow and first sets */ + SetSize(lem.nterminal+1); + + /* Find the precedence for every production rule (that has one) */ + FindRulePrecedences(&lem); + + /* Compute the lambda-nonterminals and the first-sets for every + ** nonterminal */ + FindFirstSets(&lem); + + /* Compute all LR(0) states. Also record follow-set propagation + ** links so that the follow-set can be computed later */ + lem.nstate = 0; + FindStates(&lem); + lem.sorted = State_arrayof(); + + /* Tie up loose ends on the propagation links */ + FindLinks(&lem); + + /* Compute the follow set of every reducible configuration */ + FindFollowSets(&lem); + + /* Compute the action tables */ + FindActions(&lem); + + /* Compress the action tables */ + if (compress == 0) CompressTables(&lem); + + /* Reorder and renumber the states so that states with fewer choices + ** occur at the end. This is an optimization that helps make the + ** generated parser tables smaller. */ + if (noResort == 0) ResortStates(&lem); + + /* Generate a report of the parser generated. (the "y.output" file) */ + if (!quiet) ReportOutput(&lem); + + /* Generate the source code for the parser */ + ReportTable(&lem, mhflag); + + /* Produce a header file for use by the scanner. (This step is + ** omitted if the "-m" option is used because makeheaders will + ** generate the file for us.) */ + if (!mhflag) ReportHeader(&lem); + } + if (statistics) { + printf("Parser statistics:\n"); + stats_line("terminal symbols", lem.nterminal); + stats_line("non-terminal symbols", lem.nsymbol - lem.nterminal); + stats_line("total symbols", lem.nsymbol); + stats_line("rules", lem.nrule); + stats_line("states", lem.nxstate); + stats_line("conflicts", lem.nconflict); + stats_line("action table entries", lem.nactiontab); + stats_line("total table size (bytes)", lem.tablesize); + } + if (lem.nconflict > 0) { + fprintf(stderr,"%d parsing conflicts.\n",lem.nconflict); + } + + /* return 0 on success, 1 on failure. */ + exitcode = ((lem.errorcnt > 0) || (lem.nconflict > 0)) ? 1 : 0; + exit(exitcode); + return (exitcode); +} +/******************** From the file "msort.c" *******************************/ +/* +** A generic merge-sort program. +** +** USAGE: +** Let "ptr" be a pointer to some structure which is at the head of +** a null-terminated list. Then to sort the list call: +** +** ptr = msort(ptr,&(ptr->next),cmpfnc); +** +** In the above, "cmpfnc" is a pointer to a function which compares +** two instances of the structure and returns an integer, as in +** strcmp. The second argument is a pointer to the pointer to the +** second element of the linked list. This address is used to compute +** the offset to the "next" field within the structure. The offset to +** the "next" field must be constant for all structures in the list. +** +** The function returns a new pointer which is the head of the list +** after sorting. +** +** ALGORITHM: +** Merge-sort. +*/ + +/* +** Return a pointer to the next structure in the linked list. +*/ +#define NEXT(A) (*(char **)(((char *)A)+offset)) + +/* +** Inputs: +** a: A sorted, null-terminated linked list. (May be null). +** b: A sorted, null-terminated linked list. (May be null). +** cmp: A pointer to the comparison function. +** offset: Offset in the structure to the "next" field. +** +** Return Value: +** A pointer to the head of a sorted list containing the elements +** of both a and b. +** +** Side effects: +** The "next" pointers for elements in the lists a and b are +** changed. +*/ +static char *merge( + char *a, + char *b, + int (*cmp)(const char *,const char *), + int offset +) { + char *ptr, *head; + + if (a == 0) { + head = b; + } else if (b == 0) { + head = a; + } else{ + if ((*cmp)(a,b)<=0) { + ptr = a; + a = NEXT(a); + } else{ + ptr = b; + b = NEXT(b); + } + head = ptr; + while(a && b) { + if ((*cmp)(a,b)<=0) { + NEXT(ptr) = a; + ptr = a; + a = NEXT(a); + } else{ + NEXT(ptr) = b; + ptr = b; + b = NEXT(b); + } + } + if (a) NEXT(ptr) = a; + else NEXT(ptr) = b; + } + return head; +} + +/* +** Inputs: +** list: Pointer to a singly-linked list of structures. +** next: Pointer to pointer to the second element of the list. +** cmp: A comparison function. +** +** Return Value: +** A pointer to the head of a sorted list containing the elements +** orginally in list. +** +** Side effects: +** The "next" pointers for elements in list are changed. +*/ +#define LISTSIZE 30 +static char *msort( + char *list, + char **next, + int (*cmp)(const char *,const char *) +) { + unsigned long offset; + char *ep; + char *set[LISTSIZE]; + int i; + offset = (unsigned long)((char *)next - (char *)list); + for(i = 0 ; i < LISTSIZE; i++) set[i] = 0; + while(list) { + ep = list; + list = NEXT(list); + NEXT(ep) = 0; + for(i = 0 ; i < LISTSIZE-1 && set[i]!=0; i++) { + ep = merge(ep,set[i],cmp,offset); + set[i] = 0; + } + set[i] = ep; + } + ep = 0; + for(i = 0 ; i < LISTSIZE; i++) if (set[i] ) ep = merge(set[i],ep,cmp,offset); + return ep; +} +/************************ From the file "option.c" **************************/ +static char **argv; +static struct s_options *op; +static FILE *errstream; + +#define ISOPT(X) ((X)[0] == '-'||(X)[0] == '+'||strchr((X),'=')!=0) + +/* +** Print the command line with a carrot pointing to the k-th character +** of the n-th field. +*/ +static void errline(int n, int k, FILE *err) +{ + int spcnt, i; + if (argv[0] ) fprintf(err,"%s",argv[0]); + spcnt = lemonStrlen(argv[0]) + 1; + for(i = 1; i < n && argv[i]; i++) { + fprintf(err," %s",argv[i]); + spcnt += lemonStrlen(argv[i])+1; + } + spcnt += k; + for(; argv[i]; i++) fprintf(err," %s",argv[i]); + if (spcnt<20) { + fprintf(err,"\n%*s^-- here\n",spcnt,""); + } else{ + fprintf(err,"\n%*shere --^\n",spcnt-7,""); + } +} + +/* +** Return the index of the N-th non-switch argument. Return -1 +** if N is out of range. +*/ +static int argindex(int n) +{ + int i; + int dashdash = 0; + if (argv!=0 && *argv!=0) { + for(i = 1; argv[i]; i++) { + if (dashdash || !ISOPT(argv[i]) ) { + if (n == 0) return i; + n--; + } + if (strcmp(argv[i],"--") == 0) dashdash = 1; + } + } + return -1; +} + +static char emsg[] = "Command line syntax error: "; + +/* +** Process a flag command line argument. +*/ +static int handleflags(int i, FILE *err) +{ + int v; + int errcnt = 0; + int j; + for(j=0; op[j].label; j++) { + if (strncmp(&argv[i][1],op[j].label,lemonStrlen(op[j].label)) == 0) break; + } + v = argv[i][0] == '-' ? 1 : 0; + if (op[j].label == 0) { + if (err) { + fprintf(err,"%sundefined option.\n",emsg); + errline(i,1,err); + } + errcnt++; + } else if (op[j].arg == 0) { + /* Ignore this option */ + } else if (op[j].type == OPT_FLAG) { + *((int *)op[j].arg) = v; + } else if (op[j].type == OPT_FFLAG) { + (*(void(*)(int))(op[j].arg))(v); + } else if (op[j].type == OPT_FSTR) { + (*(void(*)(char *))(op[j].arg))(&argv[i][2]); + } else{ + if (err) { + fprintf(err,"%smissing argument on switch.\n",emsg); + errline(i,1,err); + } + errcnt++; + } + return errcnt; +} + +/* +** Process a command line switch which has an argument. +*/ +static int handleswitch(int i, FILE *err) +{ + int lv = 0; + double dv = 0.0; + char *sv = 0, *end; + char *cp; + int j; + int errcnt = 0; + cp = strchr(argv[i],'='); + assert( cp!=0); + *cp = 0; + for(j=0; op[j].label; j++) { + if (strcmp(argv[i],op[j].label) == 0) break; + } + *cp = '='; + if (op[j].label == 0) { + if (err) { + fprintf(err,"%sundefined option.\n",emsg); + errline(i,0,err); + } + errcnt++; + } else{ + cp++; + switch(op[j].type) { + case OPT_FLAG: + case OPT_FFLAG: + if (err) { + fprintf(err,"%soption requires an argument.\n",emsg); + errline(i,0,err); + } + errcnt++; + break; + case OPT_DBL: + case OPT_FDBL: + dv = strtod(cp,&end); + if (*end) { + if (err) { + fprintf(err, + "%sillegal character in floating-point argument.\n",emsg); + errline(i,(int)((char *)end-(char *)argv[i]),err); + } + errcnt++; + } + break; + case OPT_INT: + case OPT_FINT: + lv = strtol(cp,&end,0); + if (*end) { + if (err) { + fprintf(err,"%sillegal character in integer argument.\n",emsg); + errline(i,(int)((char *)end-(char *)argv[i]),err); + } + errcnt++; + } + break; + case OPT_STR: + case OPT_FSTR: + sv = cp; + break; + } + switch(op[j].type) { + case OPT_FLAG: + case OPT_FFLAG: + break; + case OPT_DBL: + *(double *)(op[j].arg) = dv; + break; + case OPT_FDBL: + (*(void(*)(double))(op[j].arg))(dv); + break; + case OPT_INT: + *(int *)(op[j].arg) = lv; + break; + case OPT_FINT: + (*(void(*)(int))(op[j].arg))((int)lv); + break; + case OPT_STR: + *(char **)(op[j].arg) = sv; + break; + case OPT_FSTR: + (*(void(*)(char *))(op[j].arg))(sv); + break; + } + } + return errcnt; +} + +int OptInit(char **a, struct s_options *o, FILE *err) +{ + int errcnt = 0; + argv = a; + op = o; + errstream = err; + if (argv && *argv && op) { + int i; + for(i = 1; argv[i]; i++) { + if (argv[i][0] == '+' || argv[i][0] == '-' ) { + errcnt += handleflags(i,err); + } else if (strchr(argv[i],'=') ) { + errcnt += handleswitch(i,err); + } + } + } + if (errcnt>0) { + fprintf(err,"Valid command line options for \"%s\" are:\n",*a); + OptPrint(); + exit(1); + } + return 0; +} + +int OptNArgs() { + int cnt = 0; + int dashdash = 0; + int i; + if (argv!=0 && argv[0]!=0) { + for(i = 1; argv[i]; i++) { + if (dashdash || !ISOPT(argv[i]) ) cnt++; + if (strcmp(argv[i],"--") == 0) dashdash = 1; + } + } + return cnt; +} + +char *OptArg(int n) +{ + int i; + i = argindex(n); + return i >= 0 ? argv[i] : 0; +} + +void OptErr(int n) +{ + int i; + i = argindex(n); + if (i >= 0) errline(i,0,errstream); +} + +void OptPrint() { + int i; + int max, len; + max = 0; + for(i = 0 ; op[i].label; i++) { + len = lemonStrlen(op[i].label) + 1; + switch(op[i].type) { + case OPT_FLAG: + case OPT_FFLAG: + break; + case OPT_INT: + case OPT_FINT: + len += 9; /* length of "" */ + break; + case OPT_DBL: + case OPT_FDBL: + len += 6; /* length of "" */ + break; + case OPT_STR: + case OPT_FSTR: + len += 8; /* length of "" */ + break; + } + if (len>max) max = len; + } + for(i = 0 ; op[i].label; i++) { + switch(op[i].type) { + case OPT_FLAG: + case OPT_FFLAG: + fprintf(errstream," -%-*s %s\n",max,op[i].label,op[i].message); + break; + case OPT_INT: + case OPT_FINT: + fprintf(errstream," -%s%*s %s\n",op[i].label, + (int)(max-lemonStrlen(op[i].label)-9),"",op[i].message); + break; + case OPT_DBL: + case OPT_FDBL: + fprintf(errstream," -%s%*s %s\n",op[i].label, + (int)(max-lemonStrlen(op[i].label)-6),"",op[i].message); + break; + case OPT_STR: + case OPT_FSTR: + fprintf(errstream," -%s%*s %s\n",op[i].label, + (int)(max-lemonStrlen(op[i].label)-8),"",op[i].message); + break; + } + } +} +/*********************** From the file "parse.c" ****************************/ +/* +** Input file parser for the LEMON parser generator. +*/ + +/* The state of the parser */ +enum e_state { + INITIALIZE, + WAITING_FOR_DECL_OR_RULE, + WAITING_FOR_DECL_KEYWORD, + WAITING_FOR_DECL_ARG, + WAITING_FOR_PRECEDENCE_SYMBOL, + WAITING_FOR_ARROW, + IN_RHS, + LHS_ALIAS_1, + LHS_ALIAS_2, + LHS_ALIAS_3, + RHS_ALIAS_1, + RHS_ALIAS_2, + PRECEDENCE_MARK_1, + PRECEDENCE_MARK_2, + RESYNC_AFTER_RULE_ERROR, + RESYNC_AFTER_DECL_ERROR, + WAITING_FOR_DESTRUCTOR_SYMBOL, + WAITING_FOR_DATATYPE_SYMBOL, + WAITING_FOR_FALLBACK_ID, + WAITING_FOR_WILDCARD_ID, + WAITING_FOR_CLASS_ID, + WAITING_FOR_CLASS_TOKEN +}; +struct pstate { + char *filename; /* Name of the input file */ + int tokenlineno; /* Linenumber at which current token starts */ + int errorcnt; /* Number of errors so far */ + char *tokenstart; /* Text of current token */ + struct lemon *gp; /* Global state vector */ + enum e_state state; /* The state of the parser */ + struct symbol *fallback; /* The fallback token */ + struct symbol *tkclass; /* Token class symbol */ + struct symbol *lhs; /* Left-hand side of current rule */ + const char *lhsalias; /* Alias for the LHS */ + int nrhs; /* Number of right-hand side symbols seen */ + struct symbol *rhs[MAXRHS]; /* RHS symbols */ + const char *alias[MAXRHS]; /* Aliases for each RHS symbol (or NULL) */ + struct rule *prevrule; /* Previous rule parsed */ + const char *declkeyword; /* Keyword of a declaration */ + char **declargslot; /* Where the declaration argument should be put */ + int insertLineMacro; /* Add #line before declaration insert */ + int *decllinenoslot; /* Where to write declaration line number */ + enum e_assoc declassoc; /* Assign this association to decl arguments */ + int preccounter; /* Assign this precedence to decl arguments */ + struct rule *firstrule; /* Pointer to first rule in the grammar */ + struct rule *lastrule; /* Pointer to the most recently parsed rule */ +}; + +/* Parse a single token */ +static void parseonetoken(struct pstate *psp) +{ + const char *x; + x = Strsafe(psp->tokenstart); /* Save the token permanently */ +#if 0 + printf("%s:%d: Token=[%s] state=%d\n",psp->filename,psp->tokenlineno, + x,psp->state); +#endif + switch(psp->state) { + case INITIALIZE: + psp->prevrule = 0; + psp->preccounter = 0; + psp->firstrule = psp->lastrule = 0; + psp->gp->nrule = 0; + /* Fall thru to next case */ + case WAITING_FOR_DECL_OR_RULE: + if (x[0] == '%' ) { + psp->state = WAITING_FOR_DECL_KEYWORD; + } else if (ISLOWER(x[0]) ) { + psp->lhs = Symbol_new(x); + psp->nrhs = 0; + psp->lhsalias = 0; + psp->state = WAITING_FOR_ARROW; + } else if (x[0] == '{' ) { + if (psp->prevrule == 0) { + ErrorMsg(psp->filename,psp->tokenlineno, +"There is no prior rule upon which to attach the code \ +fragment which begins on this line."); + psp->errorcnt++; + } else if (psp->prevrule->code!=0) { + ErrorMsg(psp->filename,psp->tokenlineno, +"Code fragment beginning on this line is not the first \ +to follow the previous rule."); + psp->errorcnt++; + } else{ + psp->prevrule->line = psp->tokenlineno; + psp->prevrule->code = &x[1]; + } + } else if (x[0] == '[' ) { + psp->state = PRECEDENCE_MARK_1; + } else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Token \"%s\" should be either \"%%\" or a nonterminal name.", + x); + psp->errorcnt++; + } + break; + case PRECEDENCE_MARK_1: + if (!ISUPPER(x[0]) ) { + ErrorMsg(psp->filename,psp->tokenlineno, + "The precedence symbol must be a terminal."); + psp->errorcnt++; + } else if (psp->prevrule == 0) { + ErrorMsg(psp->filename,psp->tokenlineno, + "There is no prior rule to assign precedence \"[%s]\".",x); + psp->errorcnt++; + } else if (psp->prevrule->precsym!=0) { + ErrorMsg(psp->filename,psp->tokenlineno, +"Precedence mark on this line is not the first \ +to follow the previous rule."); + psp->errorcnt++; + } else{ + psp->prevrule->precsym = Symbol_new(x); + } + psp->state = PRECEDENCE_MARK_2; + break; + case PRECEDENCE_MARK_2: + if (x[0]!=']' ) { + ErrorMsg(psp->filename,psp->tokenlineno, + "Missing \"]\" on precedence mark."); + psp->errorcnt++; + } + psp->state = WAITING_FOR_DECL_OR_RULE; + break; + case WAITING_FOR_ARROW: + if (x[0] == ':' && x[1] == ':' && x[2] == '=' ) { + psp->state = IN_RHS; + } else if (x[0] == '(' ) { + psp->state = LHS_ALIAS_1; + } else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Expected to see a \":\" following the LHS symbol \"%s\".", + psp->lhs->name); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case LHS_ALIAS_1: + if (ISALPHA(x[0]) ) { + psp->lhsalias = x; + psp->state = LHS_ALIAS_2; + } else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "\"%s\" is not a valid alias for the LHS \"%s\"\n", + x,psp->lhs->name); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case LHS_ALIAS_2: + if (x[0] == ')' ) { + psp->state = LHS_ALIAS_3; + } else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Missing \")\" following LHS alias name \"%s\".",psp->lhsalias); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case LHS_ALIAS_3: + if (x[0] == ':' && x[1] == ':' && x[2] == '=' ) { + psp->state = IN_RHS; + } else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Missing \"->\" following: \"%s(%s)\".", + psp->lhs->name,psp->lhsalias); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case IN_RHS: + if (x[0] == '.' ) { + struct rule *rp; + rp = (struct rule *)calloc( sizeof(struct rule) + + sizeof(struct symbol *)*psp->nrhs + sizeof(char *)*psp->nrhs, 1); + if (rp == 0) { + ErrorMsg(psp->filename,psp->tokenlineno, + "Can't allocate enough memory for this rule."); + psp->errorcnt++; + psp->prevrule = 0; + } else{ + int i; + rp->ruleline = psp->tokenlineno; + rp->rhs = (struct symbol **)&rp[1]; + rp->rhsalias = (const char **)&(rp->rhs[psp->nrhs]); + for(i = 0 ; i < psp->nrhs; i++) { + rp->rhs[i] = psp->rhs[i]; + rp->rhsalias[i] = psp->alias[i]; + } + rp->lhs = psp->lhs; + rp->lhsalias = psp->lhsalias; + rp->nrhs = psp->nrhs; + rp->code = 0; + rp->precsym = 0; + rp->index = psp->gp->nrule++; + rp->nextlhs = rp->lhs->rule; + rp->lhs->rule = rp; + rp->next = 0; + if (psp->firstrule == 0) { + psp->firstrule = psp->lastrule = rp; + } else{ + psp->lastrule->next = rp; + psp->lastrule = rp; + } + psp->prevrule = rp; + } + psp->state = WAITING_FOR_DECL_OR_RULE; + } else if (ISALPHA(x[0]) ) { + if (psp->nrhs>=MAXRHS) { + ErrorMsg(psp->filename,psp->tokenlineno, + "Too many symbols on RHS of rule beginning at \"%s\".", + x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } else{ + psp->rhs[psp->nrhs] = Symbol_new(x); + psp->alias[psp->nrhs] = 0; + psp->nrhs++; + } + } else if ((x[0] == '|' || x[0] == '/') && psp->nrhs>0) { + struct symbol *msp = psp->rhs[psp->nrhs-1]; + if (msp->type!=MULTITERMINAL) { + struct symbol *origsp = msp; + msp = (struct symbol *) calloc(1,sizeof(*msp)); + memset(msp, 0, sizeof(*msp)); + msp->type = MULTITERMINAL; + msp->nsubsym = 1; + msp->subsym = (struct symbol **) calloc(1,sizeof(struct symbol *)); + msp->subsym[0] = origsp; + msp->name = origsp->name; + psp->rhs[psp->nrhs-1] = msp; + } + msp->nsubsym++; + msp->subsym = (struct symbol **) realloc(msp->subsym, + sizeof(struct symbol *)*msp->nsubsym); + msp->subsym[msp->nsubsym-1] = Symbol_new(&x[1]); + if (!ISUPPER(x[1])) { + ErrorMsg(psp->filename,psp->tokenlineno, + "Compound should be TERM|TERM without spaces"); + psp->errorcnt++; + } + if (ISLOWER(x[1]) || ISLOWER(msp->subsym[0]->name[0]) ) { + ErrorMsg(psp->filename,psp->tokenlineno, + "Cannot form a compound containing a non-terminal"); + psp->errorcnt++; + } + } else if (x[0] == '(' && psp->nrhs>0) { + psp->state = RHS_ALIAS_1; + } else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Illegal character on RHS of rule: \"%s\".",x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case RHS_ALIAS_1: + if (ISALPHA(x[0]) ) { + psp->alias[psp->nrhs-1] = x; + psp->state = RHS_ALIAS_2; + } else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "\"%s\" is not a valid alias for the RHS symbol \"%s\"\n", + x,psp->rhs[psp->nrhs-1]->name); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case RHS_ALIAS_2: + if (x[0] == ')' ) { + psp->state = IN_RHS; + } else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Missing \")\" following LHS alias name \"%s\".",psp->lhsalias); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case WAITING_FOR_DECL_KEYWORD: + if (ISALPHA(x[0]) ) { + psp->declkeyword = x; + psp->declargslot = 0; + psp->decllinenoslot = 0; + psp->insertLineMacro = 1; + psp->state = WAITING_FOR_DECL_ARG; + if (strcmp(x,"name") == 0) { + psp->declargslot = &(psp->gp->name); + psp->insertLineMacro = 0; + } else if (strcmp(x,"include") == 0) { + psp->declargslot = &(psp->gp->include); + } else if (strcmp(x,"code") == 0) { + psp->declargslot = &(psp->gp->extracode); + } else if (strcmp(x,"token_destructor") == 0) { + psp->declargslot = &psp->gp->tokendest; + } else if (strcmp(x,"default_destructor") == 0) { + psp->declargslot = &psp->gp->vardest; + } else if (strcmp(x,"token_prefix") == 0) { + psp->declargslot = &psp->gp->tokenprefix; + psp->insertLineMacro = 0; + } else if (strcmp(x,"syntax_error") == 0) { + psp->declargslot = &(psp->gp->error); + } else if (strcmp(x,"parse_accept") == 0) { + psp->declargslot = &(psp->gp->accept); + } else if (strcmp(x,"parse_failure") == 0) { + psp->declargslot = &(psp->gp->failure); + } else if (strcmp(x,"stack_overflow") == 0) { + psp->declargslot = &(psp->gp->overflow); + } else if (strcmp(x,"extra_argument") == 0) { + psp->declargslot = &(psp->gp->arg); + psp->insertLineMacro = 0; + } else if (strcmp(x,"token_type") == 0) { + psp->declargslot = &(psp->gp->tokentype); + psp->insertLineMacro = 0; + } else if (strcmp(x,"default_type") == 0) { + psp->declargslot = &(psp->gp->vartype); + psp->insertLineMacro = 0; + } else if (strcmp(x,"stack_size") == 0) { + psp->declargslot = &(psp->gp->stacksize); + psp->insertLineMacro = 0; + } else if (strcmp(x,"start_symbol") == 0) { + psp->declargslot = &(psp->gp->start); + psp->insertLineMacro = 0; + } else if (strcmp(x,"left") == 0) { + psp->preccounter++; + psp->declassoc = LEFT; + psp->state = WAITING_FOR_PRECEDENCE_SYMBOL; + } else if (strcmp(x,"right") == 0) { + psp->preccounter++; + psp->declassoc = RIGHT; + psp->state = WAITING_FOR_PRECEDENCE_SYMBOL; + } else if (strcmp(x,"nonassoc") == 0) { + psp->preccounter++; + psp->declassoc = NONE; + psp->state = WAITING_FOR_PRECEDENCE_SYMBOL; + } else if (strcmp(x,"destructor") == 0) { + psp->state = WAITING_FOR_DESTRUCTOR_SYMBOL; + } else if (strcmp(x,"type") == 0) { + psp->state = WAITING_FOR_DATATYPE_SYMBOL; + } else if (strcmp(x,"fallback") == 0) { + psp->fallback = 0; + psp->state = WAITING_FOR_FALLBACK_ID; + } else if (strcmp(x,"wildcard") == 0) { + psp->state = WAITING_FOR_WILDCARD_ID; + } else if (strcmp(x,"token_class") == 0) { + psp->state = WAITING_FOR_CLASS_ID; + } else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Unknown declaration keyword: \"%%%s\".",x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } + } else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Illegal declaration keyword: \"%s\".",x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } + break; + case WAITING_FOR_DESTRUCTOR_SYMBOL: + if (!ISALPHA(x[0]) ) { + ErrorMsg(psp->filename,psp->tokenlineno, + "Symbol name missing after %%destructor keyword"); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } else{ + struct symbol *sp = Symbol_new(x); + psp->declargslot = &sp->destructor; + psp->decllinenoslot = &sp->destLineno; + psp->insertLineMacro = 1; + psp->state = WAITING_FOR_DECL_ARG; + } + break; + case WAITING_FOR_DATATYPE_SYMBOL: + if (!ISALPHA(x[0]) ) { + ErrorMsg(psp->filename,psp->tokenlineno, + "Symbol name missing after %%type keyword"); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } else{ + struct symbol *sp = Symbol_find(x); + if ((sp) && (sp->datatype)) { + ErrorMsg(psp->filename,psp->tokenlineno, + "Symbol %%type \"%s\" already defined", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } else{ + if (!sp) { + sp = Symbol_new(x); + } + psp->declargslot = &sp->datatype; + psp->insertLineMacro = 0; + psp->state = WAITING_FOR_DECL_ARG; + } + } + break; + case WAITING_FOR_PRECEDENCE_SYMBOL: + if (x[0] == '.' ) { + psp->state = WAITING_FOR_DECL_OR_RULE; + } else if (ISUPPER(x[0]) ) { + struct symbol *sp; + sp = Symbol_new(x); + if (sp->prec>=0) { + ErrorMsg(psp->filename,psp->tokenlineno, + "Symbol \"%s\" has already be given a precedence.",x); + psp->errorcnt++; + } else{ + sp->prec = psp->preccounter; + sp->assoc = psp->declassoc; + } + } else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Can't assign a precedence to \"%s\".",x); + psp->errorcnt++; + } + break; + case WAITING_FOR_DECL_ARG: + if (x[0] == '{' || x[0] == '\"' || ISALNUM(x[0]) ) { + const char *zOld, *zNew; + char *zBuf, *z; + int nOld, n, nLine = 0, nNew, nBack; + int addLineMacro; + char zLine[50]; + zNew = x; + if (zNew[0] == '"' || zNew[0] == '{' ) zNew++; + nNew = lemonStrlen(zNew); + if (*psp->declargslot) { + zOld = *psp->declargslot; + } else{ + zOld = ""; + } + nOld = lemonStrlen(zOld); + n = nOld + nNew + 20; + addLineMacro = !psp->gp->nolinenosflag && psp->insertLineMacro && + (psp->decllinenoslot == 0 || psp->decllinenoslot[0]!=0); + if (addLineMacro) { + for(z=psp->filename, nBack=0; *z; z++) { + if (*z == '\\' ) nBack++; + } + lemon_sprintf(zLine, "#line %d ", psp->tokenlineno); + nLine = lemonStrlen(zLine); + n += nLine + lemonStrlen(psp->filename) + nBack; + } + *psp->declargslot = (char *) realloc(*psp->declargslot, n); + zBuf = *psp->declargslot + nOld; + if (addLineMacro) { + if (nOld && zBuf[-1]!='\n' ) { + *(zBuf++) = '\n'; + } + memcpy(zBuf, zLine, nLine); + zBuf += nLine; + *(zBuf++) = '"'; + for(z=psp->filename; *z; z++) { + if (*z == '\\' ) { + *(zBuf++) = '\\'; + } + *(zBuf++) = *z; + } + *(zBuf++) = '"'; + *(zBuf++) = '\n'; + } + if (psp->decllinenoslot && psp->decllinenoslot[0] == 0) { + psp->decllinenoslot[0] = psp->tokenlineno; + } + memcpy(zBuf, zNew, nNew); + zBuf += nNew; + *zBuf = 0; + psp->state = WAITING_FOR_DECL_OR_RULE; + } else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Illegal argument to %%%s: %s",psp->declkeyword,x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } + break; + case WAITING_FOR_FALLBACK_ID: + if (x[0] == '.' ) { + psp->state = WAITING_FOR_DECL_OR_RULE; + } else if (!ISUPPER(x[0]) ) { + ErrorMsg(psp->filename, psp->tokenlineno, + "%%fallback argument \"%s\" should be a token", x); + psp->errorcnt++; + } else{ + struct symbol *sp = Symbol_new(x); + if (psp->fallback == 0) { + psp->fallback = sp; + } else if (sp->fallback) { + ErrorMsg(psp->filename, psp->tokenlineno, + "More than one fallback assigned to token %s", x); + psp->errorcnt++; + } else{ + sp->fallback = psp->fallback; + psp->gp->has_fallback = 1; + } + } + break; + case WAITING_FOR_WILDCARD_ID: + if (x[0] == '.' ) { + psp->state = WAITING_FOR_DECL_OR_RULE; + } else if (!ISUPPER(x[0]) ) { + ErrorMsg(psp->filename, psp->tokenlineno, + "%%wildcard argument \"%s\" should be a token", x); + psp->errorcnt++; + } else{ + struct symbol *sp = Symbol_new(x); + if (psp->gp->wildcard == 0) { + psp->gp->wildcard = sp; + } else{ + ErrorMsg(psp->filename, psp->tokenlineno, + "Extra wildcard to token: %s", x); + psp->errorcnt++; + } + } + break; + case WAITING_FOR_CLASS_ID: + if (!ISLOWER(x[0]) ) { + ErrorMsg(psp->filename, psp->tokenlineno, + "%%token_class must be followed by an identifier: ", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } else if (Symbol_find(x) ) { + ErrorMsg(psp->filename, psp->tokenlineno, + "Symbol \"%s\" already used", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } else{ + psp->tkclass = Symbol_new(x); + psp->tkclass->type = MULTITERMINAL; + psp->state = WAITING_FOR_CLASS_TOKEN; + } + break; + case WAITING_FOR_CLASS_TOKEN: + if (x[0] == '.' ) { + psp->state = WAITING_FOR_DECL_OR_RULE; + } else if (ISUPPER(x[0]) || ((x[0] == '|' || x[0] == '/') && ISUPPER(x[1])) ) { + struct symbol *msp = psp->tkclass; + msp->nsubsym++; + msp->subsym = (struct symbol **) realloc(msp->subsym, + sizeof(struct symbol *)*msp->nsubsym); + if (!ISUPPER(x[0]) ) x++; + msp->subsym[msp->nsubsym-1] = Symbol_new(x); + } else{ + ErrorMsg(psp->filename, psp->tokenlineno, + "%%token_class argument \"%s\" should be a token", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } + break; + case RESYNC_AFTER_RULE_ERROR: +/* if (x[0] == '.' ) psp->state = WAITING_FOR_DECL_OR_RULE; +** break; */ + case RESYNC_AFTER_DECL_ERROR: + if (x[0] == '.' ) psp->state = WAITING_FOR_DECL_OR_RULE; + if (x[0] == '%' ) psp->state = WAITING_FOR_DECL_KEYWORD; + break; + } +} + +/* Run the preprocessor over the input file text. The global variables +** azDefine[0] through azDefine[nDefine-1] contains the names of all defined +** macros. This routine looks for "%ifdef" and "%ifndef" and "%endif" and +** comments them out. Text in between is also commented out as appropriate. +*/ +static void preprocess_input(char *z) { + int i, j, k, n; + int exclude = 0; + int start = 0; + int lineno = 1; + int start_lineno = 1; + for(i = 0 ; z[i]; i++) { + if (z[i] == '\n' ) lineno++; + if (z[i]!='%' || (i > 0 && z[i-1]!='\n') ) continue; + if (strncmp(&z[i],"%endif",6) == 0 && ISSPACE(z[i+6]) ) { + if (exclude) { + exclude--; + if (exclude == 0) { + for(j=start; jfilename; + ps.errorcnt = 0; + ps.state = INITIALIZE; + + /* Begin by reading the input file */ + fp = fopen(ps.filename,"rb"); + if (fp == 0) { + ErrorMsg(ps.filename,0,"Can't open this file for reading."); + gp->errorcnt++; + return; + } + fseek(fp,0,2); + filesize = ftell(fp); + rewind(fp); + filebuf = (char *)malloc( filesize+1); + if (filesize>100000000 || filebuf == 0) { + ErrorMsg(ps.filename,0,"Input file too large."); + gp->errorcnt++; + fclose(fp); + return; + } + if (fread(filebuf,1,filesize,fp)!=filesize) { + ErrorMsg(ps.filename,0,"Can't read in all %d bytes of this file.", + filesize); + free(filebuf); + gp->errorcnt++; + fclose(fp); + return; + } + fclose(fp); + filebuf[filesize] = 0; + + /* Make an initial pass through the file to handle %ifdef and %ifndef */ + preprocess_input(filebuf); + + /* Now scan the text of the input file */ + lineno = 1; + for(cp=filebuf; (c= *cp)!=0; ) { + if (c == '\n' ) lineno++; /* Keep track of the line number */ + if (ISSPACE(c) ) { cp++; continue; } /* Skip all white space */ + if (c == '/' && cp[1] == '/' ) { /* Skip C++ style comments */ + cp+=2; + while((c= *cp)!=0 && c!='\n' ) cp++; + continue; + } + if (c == '/' && cp[1] == '*' ) { /* Skip C style comments */ + cp+=2; + while((c= *cp)!=0 && (c!='/' || cp[-1]!='*') ) { + if (c == '\n' ) lineno++; + cp++; + } + if (c) cp++; + continue; + } + ps.tokenstart = cp; /* Mark the beginning of the token */ + ps.tokenlineno = lineno; /* Linenumber on which token begins */ + if (c == '\"' ) { /* String literals */ + cp++; + while((c= *cp)!=0 && c!='\"' ) { + if (c == '\n' ) lineno++; + cp++; + } + if (c == 0) { + ErrorMsg(ps.filename,startline, +"String starting on this line is not terminated before the end of the file."); + ps.errorcnt++; + nextcp = cp; + } else{ + nextcp = cp+1; + } + } else if (c == '{' ) { /* A block of C code */ + int level; + cp++; + for(level=1; (c= *cp)!=0 && (level>1 || c!='}'); cp++) { + if (c == '\n' ) lineno++; + else if (c == '{' ) level++; + else if (c == '}' ) level--; + else if (c == '/' && cp[1] == '*' ) { /* Skip comments */ + int prevc; + cp = &cp[2]; + prevc = 0; + while((c= *cp)!=0 && (c!='/' || prevc!='*') ) { + if (c == '\n' ) lineno++; + prevc = c; + cp++; + } + } else if (c == '/' && cp[1] == '/' ) { /* Skip C++ style comments too */ + cp = &cp[2]; + while((c= *cp)!=0 && c!='\n' ) cp++; + if (c) lineno++; + } else if (c == '\'' || c == '\"' ) { /* String a character literals */ + int startchar, prevc; + startchar = c; + prevc = 0; + for(cp++; (c= *cp)!=0 && (c!=startchar || prevc == '\\'); cp++) { + if (c == '\n' ) lineno++; + if (prevc == '\\' ) prevc = 0; + else prevc = c; + } + } + } + if (c == 0) { + ErrorMsg(ps.filename,ps.tokenlineno, +"C code starting on this line is not terminated before the end of the file."); + ps.errorcnt++; + nextcp = cp; + } else{ + nextcp = cp+1; + } + } else if (ISALNUM(c) ) { /* Identifiers */ + while((c= *cp)!=0 && (ISALNUM(c) || c == '_') ) cp++; + nextcp = cp; + } else if (c == ':' && cp[1] == ':' && cp[2] == '=' ) { /* The operator "::=" */ + cp += 3; + nextcp = cp; + } else if ((c == '/' || c == '|') && ISALPHA(cp[1]) ) { + cp += 2; + while((c = *cp)!=0 && (ISALNUM(c) || c == '_') ) cp++; + nextcp = cp; + } else{ /* All other (one character) operators */ + cp++; + nextcp = cp; + } + c = *cp; + *cp = 0; /* Null terminate the token */ + parseonetoken(&ps); /* Parse the token */ + *cp = (char)c; /* Restore the buffer */ + cp = nextcp; + } + free(filebuf); /* Release the buffer after parsing */ + gp->rule = ps.firstrule; + gp->errorcnt = ps.errorcnt; +} +/*************************** From the file "plink.c" *********************/ +/* +** Routines processing configuration follow-set propagation links +** in the LEMON parser generator. +*/ +static struct plink *plink_freelist = 0; + +/* Allocate a new plink */ +struct plink *Plink_new() { + struct plink *newlink; + + if (plink_freelist == 0) { + int i; + int amt = 100; + plink_freelist = (struct plink *)calloc( amt, sizeof(struct plink) ); + if (plink_freelist == 0) { + fprintf(stderr, + "Unable to allocate memory for a new follow-set propagation link.\n"); + exit(1); + } + for(i = 0 ; i < amt-1; i++) plink_freelist[i].next = &plink_freelist[i+1]; + plink_freelist[amt-1].next = 0; + } + newlink = plink_freelist; + plink_freelist = plink_freelist->next; + return newlink; +} + +/* Add a plink to a plink list */ +void Plink_add(struct plink **plpp, struct config *cfp) +{ + struct plink *newlink; + newlink = Plink_new(); + newlink->next = *plpp; + *plpp = newlink; + newlink->cfp = cfp; +} + +/* Transfer every plink on the list "from" to the list "to" */ +void Plink_copy(struct plink **to, struct plink *from) +{ + struct plink *nextpl; + while(from) { + nextpl = from->next; + from->next = *to; + *to = from; + from = nextpl; + } +} + +/* Delete every plink on the list */ +void Plink_delete(struct plink *plp) +{ + struct plink *nextpl; + + while(plp) { + nextpl = plp->next; + plp->next = plink_freelist; + plink_freelist = plp; + plp = nextpl; + } +} +/*********************** From the file "report.c" **************************/ +/* +** Procedures for generating reports and tables in the LEMON parser generator. +*/ + +/* Generate a filename with the given suffix. Space to hold the +** name comes from malloc() and must be freed by the calling +** function. +*/ +PRIVATE char *file_makename(struct lemon *lemp, const char *suffix) +{ + char *name; + char *cp; + + name = (char *)malloc( lemonStrlen(lemp->filename) + lemonStrlen(suffix) + 5); + if (name == 0) { + fprintf(stderr,"Can't allocate space for a filename.\n"); + exit(1); + } + lemon_strcpy(name,lemp->filename); + cp = strrchr(name,'.'); + if (cp) *cp = 0; + lemon_strcat(name,suffix); + return name; +} + +/* Open a file with a name based on the name of the input file, +** but with a different (specified) suffix, and return a pointer +** to the stream */ +PRIVATE FILE *file_open( + struct lemon *lemp, + const char *suffix, + const char *mode +) { + FILE *fp; + + if (lemp->outname) free(lemp->outname); + lemp->outname = file_makename(lemp, suffix); + fp = fopen(lemp->outname,mode); + if (fp == 0 && *mode == 'w' ) { + fprintf(stderr,"Can't open file \"%s\".\n",lemp->outname); + lemp->errorcnt++; + return 0; + } + return fp; +} + +/* Duplicate the input file without comments and without actions +** on rules */ +void Reprint(struct lemon *lemp) +{ + struct rule *rp; + struct symbol *sp; + int i, j, maxlen, len, ncolumns, skip; + printf("// Reprint of input file \"%s\".\n// Symbols:\n",lemp->filename); + maxlen = 10; + for(i = 0 ; i < lemp->nsymbol; i++) { + sp = lemp->symbols[i]; + len = lemonStrlen(sp->name); + if (len>maxlen) maxlen = len; + } + ncolumns = 76/(maxlen+5); + if (ncolumns<1) ncolumns = 1; + skip = (lemp->nsymbol + ncolumns - 1)/ncolumns; + for(i = 0 ; i < skip; i++) { + printf("//"); + for(j=i; jnsymbol; j+=skip) { + sp = lemp->symbols[j]; + assert( sp->index == j); + printf(" %3d %-*.*s",j,maxlen,maxlen,sp->name); + } + printf("\n"); + } + for(rp=lemp->rule; rp; rp=rp->next) { + printf("%s",rp->lhs->name); + /* if (rp->lhsalias) printf("(%s)",rp->lhsalias); */ + printf(" ::="); + for(i = 0 ; i < rp->nrhs; i++) { + sp = rp->rhs[i]; + if (sp->type == MULTITERMINAL) { + printf(" %s", sp->subsym[0]->name); + for(j=1; jnsubsym; j++) { + printf("|%s", sp->subsym[j]->name); + } + } else{ + printf(" %s", sp->name); + } + /* if (rp->rhsalias[i] ) printf("(%s)",rp->rhsalias[i]); */ + } + printf("."); + if (rp->precsym) printf(" [%s]",rp->precsym->name); + /* if (rp->code) printf("\n %s",rp->code); */ + printf("\n"); + } +} + +/* Print a single rule. +*/ +void RulePrint(FILE *fp, struct rule *rp, int iCursor) { + struct symbol *sp; + int i, j; + fprintf(fp,"%s ::=",rp->lhs->name); + for(i = 0 ; i <= rp->nrhs; i++) { + if (i == iCursor) fprintf(fp," *"); + if (i == rp->nrhs) break; + sp = rp->rhs[i]; + if (sp->type == MULTITERMINAL) { + fprintf(fp," %s", sp->subsym[0]->name); + for(j=1; jnsubsym; j++) { + fprintf(fp,"|%s",sp->subsym[j]->name); + } + } else{ + fprintf(fp," %s", sp->name); + } + } +} + +/* Print the rule for a configuration. +*/ +void ConfigPrint(FILE *fp, struct config *cfp) { + RulePrint(fp, cfp->rp, cfp->dot); +} + +/* #define TEST */ +#if 0 +/* Print a set */ +PRIVATE void SetPrint(out,set,lemp) +FILE *out; +char *set; +struct lemon *lemp; +{ + int i; + char *spacer; + spacer = ""; + fprintf(out,"%12s[",""); + for(i = 0 ; i < lemp->nterminal; i++) { + if (SetFind(set,i) ) { + fprintf(out,"%s%s",spacer,lemp->symbols[i]->name); + spacer = " "; + } + } + fprintf(out,"]\n"); +} + +/* Print a plink chain */ +PRIVATE void PlinkPrint(out,plp,tag) +FILE *out; +struct plink *plp; +char *tag; +{ + while(plp) { + fprintf(out,"%12s%s (state %2d) ","",tag,plp->cfp->stp->statenum); + ConfigPrint(out,plp->cfp); + fprintf(out,"\n"); + plp = plp->next; + } +} +#endif + +/* Print an action to the given file descriptor. Return FALSE if +** nothing was actually printed. +*/ +int PrintAction( + struct action *ap, /* The action to print */ + FILE *fp, /* Print the action here */ + int indent /* Indent by this amount */ +) { + int result = 1; + switch(ap->type) { + case SHIFT: { + struct state *stp = ap->x.stp; + fprintf(fp,"%*s shift %-7d",indent,ap->sp->name,stp->statenum); + break; + } + case REDUCE: { + struct rule *rp = ap->x.rp; + fprintf(fp,"%*s reduce %-7d",indent,ap->sp->name,rp->iRule); + RulePrint(fp, rp, -1); + break; + } + case SHIFTREDUCE: { + struct rule *rp = ap->x.rp; + fprintf(fp,"%*s shift-reduce %-7d",indent,ap->sp->name,rp->iRule); + RulePrint(fp, rp, -1); + break; + } + case ACCEPT: + fprintf(fp,"%*s accept",indent,ap->sp->name); + break; + case ERROR: + fprintf(fp,"%*s error",indent,ap->sp->name); + break; + case SRCONFLICT: + case RRCONFLICT: + fprintf(fp,"%*s reduce %-7d **Parsing conflict **", + indent,ap->sp->name,ap->x.rp->iRule); + break; + case SSCONFLICT: + fprintf(fp,"%*s shift %-7d **Parsing conflict **", + indent,ap->sp->name,ap->x.stp->statenum); + break; + case SH_RESOLVED: + if (showPrecedenceConflict) { + fprintf(fp,"%*s shift %-7d -- dropped by precedence", + indent,ap->sp->name,ap->x.stp->statenum); + } else{ + result = 0; + } + break; + case RD_RESOLVED: + if (showPrecedenceConflict) { + fprintf(fp,"%*s reduce %-7d -- dropped by precedence", + indent,ap->sp->name,ap->x.rp->iRule); + } else{ + result = 0; + } + break; + case NOT_USED: + result = 0; + break; + } + return result; +} + +/* Generate the "*.out" log file */ +void ReportOutput(struct lemon *lemp) +{ + int i; + struct state *stp; + struct config *cfp; + struct action *ap; + FILE *fp; + + fp = file_open(lemp,".out","wb"); + if (fp == 0) return; + for(i = 0 ; i < lemp->nxstate; i++) { + stp = lemp->sorted[i]; + fprintf(fp,"State %d:\n",stp->statenum); + if (lemp->basisflag) cfp=stp->bp; + else cfp=stp->cfp; + while(cfp) { + char buf[20]; + if (cfp->dot == cfp->rp->nrhs) { + lemon_sprintf(buf,"(%d)",cfp->rp->iRule); + fprintf(fp," %5s ",buf); + } else{ + fprintf(fp," "); + } + ConfigPrint(fp,cfp); + fprintf(fp,"\n"); +#if 0 + SetPrint(fp,cfp->fws,lemp); + PlinkPrint(fp,cfp->fplp,"To "); + PlinkPrint(fp,cfp->bplp,"From"); +#endif + if (lemp->basisflag) cfp=cfp->bp; + else cfp=cfp->next; + } + fprintf(fp,"\n"); + for(ap=stp->ap; ap; ap=ap->next) { + if (PrintAction(ap,fp,30) ) fprintf(fp,"\n"); + } + fprintf(fp,"\n"); + } + fprintf(fp, "----------------------------------------------------\n"); + fprintf(fp, "Symbols:\n"); + for(i = 0 ; i < lemp->nsymbol; i++) { + int j; + struct symbol *sp; + + sp = lemp->symbols[i]; + fprintf(fp, " %3d: %s", i, sp->name); + if (sp->type == NONTERMINAL) { + fprintf(fp, ":"); + if (sp->lambda) { + fprintf(fp, " "); + } + for(j=0; jnterminal; j++) { + if (sp->firstset && SetFind(sp->firstset, j) ) { + fprintf(fp, " %s", lemp->symbols[j]->name); + } + } + } + fprintf(fp, "\n"); + } + fclose(fp); + return; +} + +/* Search for the file "name" which is in the same directory as +** the exacutable */ +PRIVATE char *pathsearch(char *argv0, char *name, int modemask) +{ + const char *pathlist; + char *pathbufptr; + char *pathbuf; + char *path,*cp; + char c; + +#ifdef __WIN32__ + cp = strrchr(argv0,'\\'); +#else + cp = strrchr(argv0,'/'); +#endif + if (cp) { + c = *cp; + *cp = 0; + path = (char *)malloc( lemonStrlen(argv0) + lemonStrlen(name) + 2); + if (path) lemon_sprintf(path,"%s/%s",argv0,name); + *cp = c; + } else{ + pathlist = getenv("PATH"); + if (pathlist == 0) pathlist = ".:/bin:/usr/bin"; + pathbuf = (char *) malloc( lemonStrlen(pathlist) + 1); + path = (char *)malloc( lemonStrlen(pathlist)+lemonStrlen(name)+2); + if ((pathbuf != 0) && (path!=0) ) { + pathbufptr = pathbuf; + lemon_strcpy(pathbuf, pathlist); + while(*pathbuf) { + cp = strchr(pathbuf,':'); + if (cp == 0) cp = &pathbuf[lemonStrlen(pathbuf)]; + c = *cp; + *cp = 0; + lemon_sprintf(path,"%s/%s",pathbuf,name); + *cp = c; + if (c == 0) pathbuf[0] = 0; + else pathbuf = &cp[1]; + if (access(path,modemask) == 0) break; + } + free(pathbufptr); + } + } + return path; +} + +/* Given an action, compute the integer value for that action +** which is to be put in the action table of the generated machine. +** Return negative if no action should be generated. +*/ +PRIVATE int compute_action(struct lemon *lemp, struct action *ap) +{ + int act; + switch(ap->type) { + case SHIFT: act = ap->x.stp->statenum; break; + case SHIFTREDUCE: act = ap->x.rp->iRule + lemp->nstate; break; + case REDUCE: act = ap->x.rp->iRule + lemp->nstate+lemp->nrule; break; + case ERROR: act = lemp->nstate + lemp->nrule *2; break; + case ACCEPT: act = lemp->nstate + lemp->nrule *2 + 1; break; + default: act = -1; break; + } + return act; +} + +#define LINESIZE 1000 +/* The next cluster of routines are for reading the template file +** and writing the results to the generated parser */ +/* The first function transfers data from "in" to "out" until +** a line is seen which begins with "%%". The line number is +** tracked. +** +** if name!=0, then any word that begin with "Parse" is changed to +** begin with *name instead. +*/ +PRIVATE void tplt_xfer(char *name, FILE *in, FILE *out, int *lineno) +{ + int i, iStart; + char line[LINESIZE]; + while(fgets(line,LINESIZE,in) && (line[0]!='%' || line[1]!='%') ) { + (*lineno)++; + iStart = 0; + if (name) { + for(i = 0 ; line[i]; i++) { + if (line[i] == 'P' && strncmp(&line[i],"Parse",5) == 0 + && (i == 0 || !ISALPHA(line[i-1])) + ) { + if (i > iStart) fprintf(out,"%.*s",i-iStart,&line[iStart]); + fprintf(out,"%s",name); + i += 4; + iStart = i+1; + } + } + } + fprintf(out,"%s",&line[iStart]); + } +} + +/* The next function finds the template file and opens it, returning +** a pointer to the opened file. */ +PRIVATE FILE *tplt_open(struct lemon *lemp) +{ + static char templatename[] = "lempar.c"; + char buf[1000]; + FILE *in; + char *tpltname; + char *cp; + + /* first, see if user specified a template filename on the command line. */ + if (user_templatename != 0) { + if (access(user_templatename,004) == -1) { + fprintf(stderr,"Can't find the parser driver template file \"%s\".\n", + user_templatename); + lemp->errorcnt++; + return 0; + } + in = fopen(user_templatename,"rb"); + if (in == 0) { + fprintf(stderr,"Can't open the template file \"%s\".\n", + user_templatename); + lemp->errorcnt++; + return 0; + } + return in; + } + + cp = strrchr(lemp->filename,'.'); + if (cp) { + lemon_sprintf(buf,"%.*s.lt",(int)(cp-lemp->filename),lemp->filename); + } else{ + lemon_sprintf(buf,"%s.lt",lemp->filename); + } + if (access(buf,004) == 0) { + tpltname = buf; + } else if (access(templatename,004) == 0) { + tpltname = templatename; + } else{ + tpltname = pathsearch(lemp->argv0,templatename,0); + } + if (tpltname == 0) { + fprintf(stderr,"Can't find the parser driver template file \"%s\".\n", + templatename); + lemp->errorcnt++; + return 0; + } + in = fopen(tpltname,"rb"); + if (in == 0) { + fprintf(stderr,"Can't open the template file \"%s\".\n",templatename); + lemp->errorcnt++; + return 0; + } + return in; +} + +/* Print a #line directive line to the output file. */ +PRIVATE void tplt_linedir(FILE *out, int lineno, char *filename) +{ + fprintf(out,"#line %d \"",lineno); + while(*filename) { + if (*filename == '\\' ) putc('\\',out); + putc(*filename,out); + filename++; + } + fprintf(out,"\"\n"); +} + +/* Print a string to the file and keep the linenumber up to date */ +PRIVATE void tplt_print(FILE *out, struct lemon *lemp, char *str, int *lineno) +{ + if (str == 0) return; + while(*str) { + putc(*str,out); + if (*str == '\n' ) (*lineno)++; + str++; + } + if (str[-1]!='\n' ) { + putc('\n',out); + (*lineno)++; + } + if (!lemp->nolinenosflag) { + (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); + } + return; +} + +/* +** The following routine emits code for the destructor for the +** symbol sp +*/ +void emit_destructor_code( + FILE *out, + struct symbol *sp, + struct lemon *lemp, + int *lineno +) { + char *cp = 0; + + if (sp->type == TERMINAL) { + cp = lemp->tokendest; + if (cp == 0) return; + fprintf(out,"{\n"); (*lineno)++; + } else if (sp->destructor) { + cp = sp->destructor; + fprintf(out,"{\n"); (*lineno)++; + if (!lemp->nolinenosflag) { + (*lineno)++; + tplt_linedir(out,sp->destLineno,lemp->filename); + } + } else if (lemp->vardest) { + cp = lemp->vardest; + if (cp == 0) return; + fprintf(out,"{\n"); (*lineno)++; + } else{ + assert( 0); /* Cannot happen */ + } + for(; *cp; cp++) { + if (*cp == '$' && cp[1] == '$' ) { + fprintf(out,"(yypminor->yy%d)",sp->dtnum); + cp++; + continue; + } + if (*cp == '\n' ) (*lineno)++; + fputc(*cp,out); + } + fprintf(out,"\n"); (*lineno)++; + if (!lemp->nolinenosflag) { + (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); + } + fprintf(out,"}\n"); (*lineno)++; + return; +} + +/* +** Return TRUE (non-zero) if the given symbol has a destructor. +*/ +int has_destructor(struct symbol *sp, struct lemon *lemp) +{ + int ret; + if (sp->type == TERMINAL) { + ret = lemp->tokendest!=0; + } else{ + ret = lemp->vardest!=0 || sp->destructor!=0; + } + return ret; +} + +/* +** Append text to a dynamically allocated string. If zText is 0 then +** reset the string to be empty again. Always return the complete text +** of the string (which is overwritten with each call). +** +** n bytes of zText are stored. If n == 0 then all of zText up to the first +** \000 terminator is stored. zText can contain up to two instances of +** %d. The values of p1 and p2 are written into the first and second +** %d. +** +** If n == -1, then the previous character is overwritten. +*/ +PRIVATE char *append_str(const char *zText, int n, int p1, int p2) { + static char empty[1] = { 0 }; + static char *z = 0; + static int alloced = 0; + static int used = 0; + int c; + char zInt[40]; + if (zText == 0) { + if (used == 0 && z!=0) z[0] = 0; + used = 0; + return z; + } + if (n<=0) { + if (n<0) { + used += n; + assert( used>=0); + } + n = lemonStrlen(zText); + } + if ((int) (n+sizeof(zInt)*2+used) >= alloced) { + alloced = n + sizeof(zInt)*2 + used + 200; + z = (char *) realloc(z, alloced); + } + if (z == 0) return empty; + while(n-- > 0) { + c = *(zText++); + if (c == '%' && n>0 && zText[0] == 'd' ) { + lemon_sprintf(zInt, "%d", p1); + p1 = p2; + lemon_strcpy(&z[used], zInt); + used += lemonStrlen(&z[used]); + zText++; + n--; + } else{ + z[used++] = (char)c; + } + } + z[used] = 0; + return z; +} + +/* +** zCode is a string that is the action associated with a rule. Expand +** the symbols in this string so that the refer to elements of the parser +** stack. +** +** Return 1 if the expanded code requires that "yylhsminor" local variable +** to be defined. +*/ +PRIVATE int translate_code(struct lemon *lemp, struct rule *rp) { + char *cp, *xp; + int i; + int rc = 0; /* True if yylhsminor is used */ + int dontUseRhs0 = 0; /* If true, use of left-most RHS label is illegal */ + const char *zSkip = 0; /* The zOvwrt comment within rp->code, or NULL */ + char lhsused = 0; /* True if the LHS element has been used */ + char lhsdirect; /* True if LHS writes directly into stack */ + char used[MAXRHS]; /* True for each RHS element which is used */ + char zLhs[50]; /* Convert the LHS symbol into this string */ + char zOvwrt[900]; /* Comment that to allow LHS to overwrite RHS */ + + for(i = 0 ; i < rp->nrhs; i++) used[i] = 0; + lhsused = 0; + + if (rp->code == 0) { + static char newlinestr[2] = { '\n', '\0' }; + rp->code = newlinestr; + rp->line = rp->ruleline; + } + + + if (rp->nrhs == 0) { + /* If there are no RHS symbols, then writing directly to the LHS is ok */ + lhsdirect = 1; + } else if (rp->rhsalias[0] == 0) { + /* The left-most RHS symbol has no value. LHS direct is ok. But + ** we have to call the distructor on the RHS symbol first. */ + lhsdirect = 1; + if (has_destructor(rp->rhs[0],lemp) ) { + append_str(0,0,0,0); + append_str(" yy_destructor(yypParser,%d,&yymsp[%d].minor);\n", 0, + rp->rhs[0]->index,1-rp->nrhs); + rp->codePrefix = Strsafe(append_str(0,0,0,0)); + } + } else if (rp->lhsalias == 0) { + /* There is no LHS value symbol. */ + lhsdirect = 1; + } else if (strcmp(rp->lhsalias,rp->rhsalias[0]) == 0) { + /* The LHS symbol and the left-most RHS symbol are the same, so + ** direct writing is allowed */ + lhsdirect = 1; + lhsused = 1; + used[0] = 1; + if (rp->lhs->dtnum!=rp->rhs[0]->dtnum) { + ErrorMsg(lemp->filename,rp->ruleline, + "%s(%s) and %s(%s) share the same label but have " + "different datatypes.", + rp->lhs->name, rp->lhsalias, rp->rhs[0]->name, rp->rhsalias[0]); + lemp->errorcnt++; + } + } else{ + lemon_sprintf(zOvwrt, "/*%s-overwrites-%s */", + rp->lhsalias, rp->rhsalias[0]); + zSkip = strstr(rp->code, zOvwrt); + if (zSkip!=0) { + /* The code contains a special comment that indicates that it is safe + ** for the LHS label to overwrite left-most RHS label. */ + lhsdirect = 1; + } else{ + lhsdirect = 0; + } + } + if (lhsdirect) { + sprintf(zLhs, "yymsp[%d].minor.yy%d",1-rp->nrhs,rp->lhs->dtnum); + } else{ + rc = 1; + sprintf(zLhs, "yylhsminor.yy%d",rp->lhs->dtnum); + } + + append_str(0,0,0,0); + + /* This const cast is wrong but harmless, if we're careful. */ + for(cp=(char *)rp->code; *cp; cp++) { + if (cp == zSkip) { + append_str(zOvwrt,0,0,0); + cp += lemonStrlen(zOvwrt)-1; + dontUseRhs0 = 1; + continue; + } + if (ISALPHA(*cp) && (cp == rp->code || (!ISALNUM(cp[-1]) && cp[-1]!='_')) ) { + char saved; + for(xp= &cp[1]; ISALNUM(*xp) || *xp == '_'; xp++); + saved = *xp; + *xp = 0; + if (rp->lhsalias && strcmp(cp,rp->lhsalias) == 0) { + append_str(zLhs,0,0,0); + cp = xp; + lhsused = 1; + } else{ + for(i = 0 ; i < rp->nrhs; i++) { + if (rp->rhsalias[i] && strcmp(cp,rp->rhsalias[i]) == 0) { + if (i == 0 && dontUseRhs0) { + ErrorMsg(lemp->filename,rp->ruleline, + "Label %s used after '%s'.", + rp->rhsalias[0], zOvwrt); + lemp->errorcnt++; + } else if (cp!=rp->code && cp[-1] == '@' ) { + /* If the argument is of the form @X then substituted + ** the token number of X, not the value of X */ + append_str("yymsp[%d].major",-1,i-rp->nrhs+1,0); + } else{ + struct symbol *sp = rp->rhs[i]; + int dtnum; + if (sp->type == MULTITERMINAL) { + dtnum = sp->subsym[0]->dtnum; + } else{ + dtnum = sp->dtnum; + } + append_str("yymsp[%d].minor.yy%d",0,i-rp->nrhs+1, dtnum); + } + cp = xp; + used[i] = 1; + break; + } + } + } + *xp = saved; + } + append_str(cp, 1, 0, 0); + } /* End loop */ + + /* Main code generation completed */ + cp = append_str(0,0,0,0); + if (cp && cp[0] ) rp->code = Strsafe(cp); + append_str(0,0,0,0); + + /* Check to make sure the LHS has been used */ + if (rp->lhsalias && !lhsused) { + ErrorMsg(lemp->filename,rp->ruleline, + "Label \"%s\" for \"%s(%s)\" is never used.", + rp->lhsalias,rp->lhs->name,rp->lhsalias); + lemp->errorcnt++; + } + + /* Generate destructor code for RHS minor values which are not referenced. + ** Generate error messages for unused labels and duplicate labels. + */ + for(i = 0 ; i < rp->nrhs; i++) { + if (rp->rhsalias[i] ) { + if (i > 0) { + int j; + if (rp->lhsalias && strcmp(rp->lhsalias,rp->rhsalias[i]) == 0) { + ErrorMsg(lemp->filename,rp->ruleline, + "%s(%s) has the same label as the LHS but is not the left-most " + "symbol on the RHS.", + rp->rhs[i]->name, rp->rhsalias); + lemp->errorcnt++; + } + for(j=0; jrhsalias[j] && strcmp(rp->rhsalias[j],rp->rhsalias[i]) == 0) { + ErrorMsg(lemp->filename,rp->ruleline, + "Label %s used for multiple symbols on the RHS of a rule.", + rp->rhsalias[i]); + lemp->errorcnt++; + break; + } + } + } + if (!used[i] ) { + ErrorMsg(lemp->filename,rp->ruleline, + "Label %s for \"%s(%s)\" is never used.", + rp->rhsalias[i],rp->rhs[i]->name,rp->rhsalias[i]); + lemp->errorcnt++; + } + } else if (i > 0 && has_destructor(rp->rhs[i],lemp) ) { + append_str(" yy_destructor(yypParser,%d,&yymsp[%d].minor);\n", 0, + rp->rhs[i]->index,i-rp->nrhs+1); + } + } + + /* If unable to write LHS values directly into the stack, write the + ** saved LHS value now. */ + if (lhsdirect == 0) { + append_str(" yymsp[%d].minor.yy%d = ", 0, 1-rp->nrhs, rp->lhs->dtnum); + append_str(zLhs, 0, 0, 0); + append_str(";\n", 0, 0, 0); + } + + /* Suffix code generation complete */ + cp = append_str(0,0,0,0); + if (cp && cp[0] ) rp->codeSuffix = Strsafe(cp); + + return rc; +} + +/* +** Generate code which executes when the rule "rp" is reduced. Write +** the code to "out". Make sure lineno stays up-to-date. +*/ +PRIVATE void emit_code( + FILE *out, + struct rule *rp, + struct lemon *lemp, + int *lineno +) { + const char *cp; + + /* Setup code prior to the #line directive */ + if (rp->codePrefix && rp->codePrefix[0] ) { + fprintf(out, "{%s", rp->codePrefix); + for(cp=rp->codePrefix; *cp; cp++) { if (*cp == '\n' ) (*lineno)++; } + } + + /* Generate code to do the reduce action */ + if (rp->code) { + if (!lemp->nolinenosflag) { + (*lineno)++; + tplt_linedir(out,rp->line,lemp->filename); + } + fprintf(out,"{%s",rp->code); + for(cp=rp->code; *cp; cp++) { if (*cp == '\n' ) (*lineno)++; } + fprintf(out,"}\n"); (*lineno)++; + if (!lemp->nolinenosflag) { + (*lineno)++; + tplt_linedir(out,*lineno,lemp->outname); + } + } + + /* Generate breakdown code that occurs after the #line directive */ + if (rp->codeSuffix && rp->codeSuffix[0] ) { + fprintf(out, "%s", rp->codeSuffix); + for(cp=rp->codeSuffix; *cp; cp++) { if (*cp == '\n' ) (*lineno)++; } + } + + if (rp->codePrefix) { + fprintf(out, "}\n"); (*lineno)++; + } + + return; +} + +/* +** Print the definition of the union used for the parser's data stack. +** This union contains fields for every possible data type for tokens +** and nonterminals. In the process of computing and printing this +** union, also set the ".dtnum" field of every terminal and nonterminal +** symbol. +*/ +void print_stack_union( + FILE *out, /* The output stream */ + struct lemon *lemp, /* The main info structure for this parser */ + int *plineno, /* Pointer to the line number */ + int mhflag /* True if generating makeheaders output */ +) { + int lineno = *plineno; /* The line number of the output */ + char **types; /* A hash table of datatypes */ + int arraysize; /* Size of the "types" array */ + int maxdtlength; /* Maximum length of any ".datatype" field. */ + char *stddt; /* Standardized name for a datatype */ + int i,j; /* Loop counters */ + unsigned hash; /* For hashing the name of a type */ + const char *name; /* Name of the parser */ + + /* Allocate and initialize types[] and allocate stddt[] */ + arraysize = lemp->nsymbol *2; + types = (char **)calloc( arraysize, sizeof(char *) ); + if (types == 0) { + fprintf(stderr,"Out of memory.\n"); + exit(1); + } + for(i = 0 ; i < arraysize; i++) types[i] = 0; + maxdtlength = 0; + if (lemp->vartype) { + maxdtlength = lemonStrlen(lemp->vartype); + } + for(i = 0 ; i < lemp->nsymbol; i++) { + int len; + struct symbol *sp = lemp->symbols[i]; + if (sp->datatype == 0) continue; + len = lemonStrlen(sp->datatype); + if (len>maxdtlength) maxdtlength = len; + } + stddt = (char *)malloc( maxdtlength *2 + 1); + if (stddt == 0) { + fprintf(stderr,"Out of memory.\n"); + exit(1); + } + + /* Build a hash table of datatypes. The ".dtnum" field of each symbol + ** is filled in with the hash index plus 1. A ".dtnum" value of 0 is + ** used for terminal symbols. If there is no %default_type defined then + ** 0 is also used as the .dtnum value for nonterminals which do not specify + ** a datatype using the %type directive. + */ + for(i = 0 ; i < lemp->nsymbol; i++) { + struct symbol *sp = lemp->symbols[i]; + char *cp; + if (sp == lemp->errsym) { + sp->dtnum = arraysize+1; + continue; + } + if (sp->type!=NONTERMINAL || (sp->datatype == 0 && lemp->vartype == 0) ) { + sp->dtnum = 0; + continue; + } + cp = sp->datatype; + if (cp == 0) cp = lemp->vartype; + j = 0; + while(ISSPACE(*cp) ) cp++; + while(*cp) stddt[j++] = *cp++; + while(j>0 && ISSPACE(stddt[j-1]) ) j--; + stddt[j] = 0; + if (lemp->tokentype && strcmp(stddt, lemp->tokentype) == 0) { + sp->dtnum = 0; + continue; + } + hash = 0; + for(j=0; stddt[j]; j++) { + hash = hash *53 + stddt[j]; + } + hash = (hash & 0x7fffffff)%arraysize; + while(types[hash] ) { + if (strcmp(types[hash],stddt) == 0) { + sp->dtnum = hash + 1; + break; + } + hash++; + if (hash>=(unsigned)arraysize) hash = 0; + } + if (types[hash] == 0) { + sp->dtnum = hash + 1; + types[hash] = (char *)malloc( lemonStrlen(stddt)+1); + if (types[hash] == 0) { + fprintf(stderr,"Out of memory.\n"); + exit(1); + } + lemon_strcpy(types[hash],stddt); + } + } + + /* Print out the definition of YYTOKENTYPE and YYMINORTYPE */ + name = lemp->name ? lemp->name : "Parse"; + lineno = *plineno; + if (mhflag) { fprintf(out,"#if INTERFACE\n"); lineno++; } + fprintf(out,"#define %sTOKENTYPE %s\n",name, + lemp->tokentype?lemp->tokentype:"void *"); lineno++; + if (mhflag) { fprintf(out,"#endif\n"); lineno++; } + fprintf(out,"typedef union {\n"); lineno++; + fprintf(out," int yyinit;\n"); lineno++; + fprintf(out," %sTOKENTYPE yy0;\n",name); lineno++; + for(i = 0 ; i < arraysize; i++) { + if (types[i] == 0) continue; + fprintf(out," %s yy%d;\n",types[i],i+1); lineno++; + free(types[i]); + } + if (lemp->errsym->useCnt) { + fprintf(out," int yy%d;\n",lemp->errsym->dtnum); lineno++; + } + free(stddt); + free(types); + fprintf(out,"} YYMINORTYPE;\n"); lineno++; + *plineno = lineno; +} + +/* +** Return the name of a C datatype able to represent values between +** lwr and upr, inclusive. If pnByte!=NULL then also write the sizeof +** for that type (1, 2, or 4) into *pnByte. +*/ +static const char *minimum_size_type(int lwr, int upr, int *pnByte) { + const char *zType = "int"; + int nByte = 4; + if (lwr>=0) { + if (upr<=255) { + zType = "unsigned char"; + nByte = 1; + } else if (upr<65535) { + zType = "unsigned short int"; + nByte = 2; + } else{ + zType = "unsigned int"; + nByte = 4; + } + } else if (lwr>=-127 && upr<=127) { + zType = "signed char"; + nByte = 1; + } else if (lwr>=-32767 && upr<32767) { + zType = "short"; + nByte = 2; + } + if (pnByte) *pnByte = nByte; + return zType; +} + +/* +** Each state contains a set of token transaction and a set of +** nonterminal transactions. Each of these sets makes an instance +** of the following structure. An array of these structures is used +** to order the creation of entries in the yy_action[] table. +*/ +struct axset { + struct state *stp; /* A pointer to a state */ + int isTkn; /* True to use tokens. False for non-terminals */ + int nAction; /* Number of actions */ + int iOrder; /* Original order of action sets */ +}; + +/* +** Compare to axset structures for sorting purposes +*/ +static int axset_compare(const void *a, const void *b) { + struct axset *p1 = (struct axset *)a; + struct axset *p2 = (struct axset *)b; + int c; + c = p2->nAction - p1->nAction; + if (c == 0) { + c = p1->iOrder - p2->iOrder; + } + assert( c!=0 || p1==p2); + return c; +} + +/* +** Write text on "out" that describes the rule "rp". +*/ +static void writeRuleText(FILE *out, struct rule *rp) { + int j; + fprintf(out,"%s ::=", rp->lhs->name); + for(j=0; jnrhs; j++) { + struct symbol *sp = rp->rhs[j]; + if (sp->type!=MULTITERMINAL) { + fprintf(out," %s", sp->name); + } else{ + int k; + fprintf(out," %s", sp->subsym[0]->name); + for(k=1; knsubsym; k++) { + fprintf(out,"|%s",sp->subsym[k]->name); + } + } + } +} + +static FILE *source_open(struct lemon *lemp) +{ + if (user_output_file) { + if (lemp->outname) free(lemp->outname); + lemp->outname = strdup(user_output_file); + FILE *fp = fopen(lemp->outname, "wb"); + if (fp == 0) { + fprintf(stderr,"Can't open file \"%s\".\n",lemp->outname); + lemp->errorcnt++; + return 0; + } + return fp; + } else { + return file_open(lemp,".c","wb"); + } +} + +/* Generate C source code for the parser */ +void ReportTable( + struct lemon *lemp, + int mhflag /* Output in makeheaders format if true */ +) { + FILE *out, *in; + char line[LINESIZE]; + int lineno; + struct state *stp; + struct action *ap; + struct rule *rp; + struct acttab *pActtab; + int i, j, n, sz; + int szActionType; /* sizeof(YYACTIONTYPE) */ + int szCodeType; /* sizeof(YYCODETYPE) */ + const char *name; + int mnTknOfst, mxTknOfst; + int mnNtOfst, mxNtOfst; + struct axset *ax; + + in = tplt_open(lemp); + if (in == 0) return; + out = source_open(lemp); + if (out == 0) { + fclose(in); + return; + } + lineno = 1; + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate the include code, if any */ + tplt_print(out,lemp,lemp->include,&lineno); + if (mhflag) { + char *incName = file_makename(lemp, ".h"); + fprintf(out,"#include \"%s\"\n", incName); lineno++; + free(incName); + } + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate #defines for all tokens */ + if (mhflag) { + const char *prefix; + fprintf(out,"#if INTERFACE\n"); lineno++; + if (lemp->tokenprefix) prefix = lemp->tokenprefix; + else prefix = ""; + for(i = 1; i < lemp->nterminal; i++) { + fprintf(out,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i); + lineno++; + } + fprintf(out,"#endif\n"); lineno++; + } + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate the defines */ + fprintf(out,"#define YYCODETYPE %s\n", + minimum_size_type(0, lemp->nsymbol+1, &szCodeType)); lineno++; + fprintf(out,"#define YYNOCODE %d\n",lemp->nsymbol+1); lineno++; + fprintf(out,"#define YYACTIONTYPE %s\n", + minimum_size_type(0,lemp->nstate+lemp->nrule *2+5,&szActionType)); lineno++; + if (lemp->wildcard) { + fprintf(out,"#define YYWILDCARD %d\n", + lemp->wildcard->index); lineno++; + } + print_stack_union(out,lemp,&lineno,mhflag); + fprintf(out, "#ifndef YYSTACKDEPTH\n"); lineno++; + if (lemp->stacksize) { + fprintf(out,"#define YYSTACKDEPTH %s\n",lemp->stacksize); lineno++; + } else{ + fprintf(out,"#define YYSTACKDEPTH 100\n"); lineno++; + } + fprintf(out, "#endif\n"); lineno++; + if (mhflag) { + fprintf(out,"#if INTERFACE\n"); lineno++; + } + name = lemp->name ? lemp->name : "Parse"; + if (lemp->arg && lemp->arg[0] ) { + i = lemonStrlen(lemp->arg); + while(i >= 1 && ISSPACE(lemp->arg[i-1]) ) i--; + while(i >= 1 && (ISALNUM(lemp->arg[i-1]) || lemp->arg[i-1] == '_') ) i--; + fprintf(out,"#define %sARG_SDECL %s;\n",name,lemp->arg); lineno++; + fprintf(out,"#define %sARG_PDECL ,%s\n",name,lemp->arg); lineno++; + fprintf(out,"#define %sARG_FETCH %s = yypParser->%s\n", + name,lemp->arg,&lemp->arg[i]); lineno++; + fprintf(out,"#define %sARG_STORE yypParser->%s = %s\n", + name,&lemp->arg[i],&lemp->arg[i]); lineno++; + } else{ + fprintf(out,"#define %sARG_SDECL\n",name); lineno++; + fprintf(out,"#define %sARG_PDECL\n",name); lineno++; + fprintf(out,"#define %sARG_FETCH\n",name); lineno++; + fprintf(out,"#define %sARG_STORE\n",name); lineno++; + } + if (mhflag) { + fprintf(out,"#endif\n"); lineno++; + } + if (lemp->errsym->useCnt) { + fprintf(out,"#define YYERRORSYMBOL %d\n",lemp->errsym->index); lineno++; + fprintf(out,"#define YYERRSYMDT yy%d\n",lemp->errsym->dtnum); lineno++; + } + if (lemp->has_fallback) { + fprintf(out,"#define YYFALLBACK 1\n"); lineno++; + } + + /* Compute the action table, but do not output it yet. The action + ** table must be computed before generating the YYNSTATE macro because + ** we need to know how many states can be eliminated. + */ + ax = (struct axset *) calloc(lemp->nxstate *2, sizeof(ax[0])); + if (ax == 0) { + fprintf(stderr,"malloc failed\n"); + exit(1); + } + for(i = 0 ; i < lemp->nxstate; i++) { + stp = lemp->sorted[i]; + ax[i *2].stp = stp; + ax[i *2].isTkn = 1; + ax[i *2].nAction = stp->nTknAct; + ax[i *2+1].stp = stp; + ax[i *2+1].isTkn = 0; + ax[i *2+1].nAction = stp->nNtAct; + } + mxTknOfst = mnTknOfst = 0; + mxNtOfst = mnNtOfst = 0; + /* In an effort to minimize the action table size, use the heuristic + ** of placing the largest action sets first */ + for(i = 0 ; i < lemp->nxstate *2; i++) ax[i].iOrder = i; + qsort(ax, lemp->nxstate *2, sizeof(ax[0]), axset_compare); + pActtab = acttab_alloc(); + for(i = 0 ; i < lemp->nxstate *2 && ax[i].nAction>0; i++) { + stp = ax[i].stp; + if (ax[i].isTkn) { + for(ap=stp->ap; ap; ap=ap->next) { + int action; + if (ap->sp->index>=lemp->nterminal) continue; + action = compute_action(lemp, ap); + if (action<0) continue; + acttab_action(pActtab, ap->sp->index, action); + } + stp->iTknOfst = acttab_insert(pActtab); + if (stp->iTknOfstiTknOfst; + if (stp->iTknOfst>mxTknOfst) mxTknOfst = stp->iTknOfst; + } else{ + for(ap=stp->ap; ap; ap=ap->next) { + int action; + if (ap->sp->indexnterminal) continue; + if (ap->sp->index == lemp->nsymbol) continue; + action = compute_action(lemp, ap); + if (action<0) continue; + acttab_action(pActtab, ap->sp->index, action); + } + stp->iNtOfst = acttab_insert(pActtab); + if (stp->iNtOfstiNtOfst; + if (stp->iNtOfst>mxNtOfst) mxNtOfst = stp->iNtOfst; + } +#if 0 /* Uncomment for a trace of how the yy_action[] table fills out */ + { int jj, nn; + for(jj=nn=0; jjnAction; jj++) { + if (pActtab->aAction[jj].action<0) nn++; + } + printf("%4d: State %3d %s n: %2d size: %5d freespace: %d\n", + i, stp->statenum, ax[i].isTkn ? "Token" : "Var ", + ax[i].nAction, pActtab->nAction, nn); + } +#endif + } + free(ax); + + /* Finish rendering the constants now that the action table has + ** been computed */ + fprintf(out,"#define YYNSTATE %d\n",lemp->nxstate); lineno++; + fprintf(out,"#define YYNRULE %d\n",lemp->nrule); lineno++; + fprintf(out,"#define YY_MAX_SHIFT %d\n",lemp->nxstate-1); lineno++; + fprintf(out,"#define YY_MIN_SHIFTREDUCE %d\n",lemp->nstate); lineno++; + i = lemp->nstate + lemp->nrule; + fprintf(out,"#define YY_MAX_SHIFTREDUCE %d\n", i-1); lineno++; + fprintf(out,"#define YY_MIN_REDUCE %d\n", i); lineno++; + i = lemp->nstate + lemp->nrule *2; + fprintf(out,"#define YY_MAX_REDUCE %d\n", i-1); lineno++; + fprintf(out,"#define YY_ERROR_ACTION %d\n", i); lineno++; + fprintf(out,"#define YY_ACCEPT_ACTION %d\n", i+1); lineno++; + fprintf(out,"#define YY_NO_ACTION %d\n", i+2); lineno++; + tplt_xfer(lemp->name,in,out,&lineno); + + /* Now output the action table and its associates: + ** + ** yy_action[] A single table containing all actions. + ** yy_lookahead[] A table containing the lookahead for each entry in + ** yy_action. Used to detect hash collisions. + ** yy_shift_ofst[] For each state, the offset into yy_action for + ** shifting terminals. + ** yy_reduce_ofst[] For each state, the offset into yy_action for + ** shifting non-terminals after a reduce. + ** yy_default[] Default action for each state. + */ + + /* Output the yy_action table */ + lemp->nactiontab = n = acttab_size(pActtab); + lemp->tablesize += n *szActionType; + fprintf(out,"#define YY_ACTTAB_COUNT (%d)\n", n); lineno++; + fprintf(out,"static const YYACTIONTYPE yy_action[] = {\n"); lineno++; + for(i=j=0; i < n; i++) { + int action = acttab_yyaction(pActtab, i); + if (action<0) action = lemp->nstate + lemp->nrule + 2; + if (j == 0) fprintf(out," /* %5d */ ", i); + fprintf(out, " %4d,", action); + if (j == 9 || i == n-1) { + fprintf(out, "\n"); lineno++; + j = 0; + } else{ + j++; + } + } + fprintf(out, "};\n"); lineno++; + + /* Output the yy_lookahead table */ + lemp->tablesize += n *szCodeType; + fprintf(out,"static const YYCODETYPE yy_lookahead[] = {\n"); lineno++; + for(i=j=0; i < n; i++) { + int la = acttab_yylookahead(pActtab, i); + if (la<0) la = lemp->nsymbol; + if (j == 0) fprintf(out," /* %5d */ ", i); + fprintf(out, " %4d,", la); + if (j == 9 || i == n-1) { + fprintf(out, "\n"); lineno++; + j = 0; + } else{ + j++; + } + } + fprintf(out, "};\n"); lineno++; + + /* Output the yy_shift_ofst[] table */ + fprintf(out, "#define YY_SHIFT_USE_DFLT (%d)\n", mnTknOfst-1); lineno++; + n = lemp->nxstate; + while(n>0 && lemp->sorted[n-1]->iTknOfst == NO_OFFSET) n--; + fprintf(out, "#define YY_SHIFT_COUNT (%d)\n", n-1); lineno++; + fprintf(out, "#define YY_SHIFT_MIN (%d)\n", mnTknOfst); lineno++; + fprintf(out, "#define YY_SHIFT_MAX (%d)\n", mxTknOfst); lineno++; + fprintf(out, "static const %s yy_shift_ofst[] = {\n", + minimum_size_type(mnTknOfst-1, mxTknOfst, &sz)); lineno++; + lemp->tablesize += n *sz; + for(i=j=0; i < n; i++) { + int ofst; + stp = lemp->sorted[i]; + ofst = stp->iTknOfst; + if (ofst == NO_OFFSET) ofst = mnTknOfst - 1; + if (j == 0) fprintf(out," /* %5d */ ", i); + fprintf(out, " %4d,", ofst); + if (j == 9 || i == n-1) { + fprintf(out, "\n"); lineno++; + j = 0; + } else{ + j++; + } + } + fprintf(out, "};\n"); lineno++; + + /* Output the yy_reduce_ofst[] table */ + fprintf(out, "#define YY_REDUCE_USE_DFLT (%d)\n", mnNtOfst-1); lineno++; + n = lemp->nxstate; + while(n>0 && lemp->sorted[n-1]->iNtOfst == NO_OFFSET) n--; + fprintf(out, "#define YY_REDUCE_COUNT (%d)\n", n-1); lineno++; + fprintf(out, "#define YY_REDUCE_MIN (%d)\n", mnNtOfst); lineno++; + fprintf(out, "#define YY_REDUCE_MAX (%d)\n", mxNtOfst); lineno++; + fprintf(out, "static const %s yy_reduce_ofst[] = {\n", + minimum_size_type(mnNtOfst-1, mxNtOfst, &sz)); lineno++; + lemp->tablesize += n *sz; + for(i=j=0; i < n; i++) { + int ofst; + stp = lemp->sorted[i]; + ofst = stp->iNtOfst; + if (ofst == NO_OFFSET) ofst = mnNtOfst - 1; + if (j == 0) fprintf(out," /* %5d */ ", i); + fprintf(out, " %4d,", ofst); + if (j == 9 || i == n-1) { + fprintf(out, "\n"); lineno++; + j = 0; + } else{ + j++; + } + } + fprintf(out, "};\n"); lineno++; + + /* Output the default action table */ + fprintf(out, "static const YYACTIONTYPE yy_default[] = {\n"); lineno++; + n = lemp->nxstate; + lemp->tablesize += n *szActionType; + for(i=j=0; i < n; i++) { + stp = lemp->sorted[i]; + if (j == 0) fprintf(out," /* %5d */ ", i); + fprintf(out, " %4d,", stp->iDfltReduce+lemp->nstate+lemp->nrule); + if (j == 9 || i == n-1) { + fprintf(out, "\n"); lineno++; + j = 0; + } else{ + j++; + } + } + fprintf(out, "};\n"); lineno++; + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate the table of fallback tokens. + */ + if (lemp->has_fallback) { + int mx = lemp->nterminal - 1; + while(mx>0 && lemp->symbols[mx]->fallback == 0) { mx--; } + lemp->tablesize += (mx+1)*szCodeType; + for(i = 0 ; i <= mx; i++) { + struct symbol *p = lemp->symbols[i]; + if (p->fallback == 0) { + fprintf(out, " 0, /* %10s => nothing */\n", p->name); + } else{ + fprintf(out, " %3d, /* %10s => %s */\n", p->fallback->index, + p->name, p->fallback->name); + } + lineno++; + } + } + tplt_xfer(lemp->name, in, out, &lineno); + + /* Generate a table containing the symbolic name of every symbol + */ + for(i = 0 ; i < lemp->nsymbol; i++) { + lemon_sprintf(line,"\"%s\",",lemp->symbols[i]->name); + fprintf(out," %-15s",line); + if ((i&3) == 3) { fprintf(out,"\n"); lineno++; } + } + if ((i&3)!=0) { fprintf(out,"\n"); lineno++; } + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate a table containing a text string that describes every + ** rule in the rule set of the grammar. This information is used + ** when tracing REDUCE actions. + */ + for(i = 0 , rp=lemp->rule; rp; rp=rp->next, i++) { + assert( rp->iRule == i); + fprintf(out," /* %3d */ \"", i); + writeRuleText(out, rp); + fprintf(out,"\",\n"); lineno++; + } + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate code which executes every time a symbol is popped from + ** the stack while processing errors or while destroying the parser. + ** (In other words, generate the %destructor actions) + */ + if (lemp->tokendest) { + int once = 1; + for(i = 0 ; i < lemp->nsymbol; i++) { + struct symbol *sp = lemp->symbols[i]; + if (sp == 0 || sp->type!=TERMINAL) continue; + if (once) { + fprintf(out, " /* TERMINAL Destructor */\n"); lineno++; + once = 0; + } + fprintf(out," case %d: /* %s */\n", sp->index, sp->name); lineno++; + } + for(i = 0 ; i < lemp->nsymbol && lemp->symbols[i]->type!=TERMINAL; i++); + if (i < lemp->nsymbol) { + emit_destructor_code(out,lemp->symbols[i],lemp,&lineno); + fprintf(out," break;\n"); lineno++; + } + } + if (lemp->vardest) { + struct symbol *dflt_sp = 0; + int once = 1; + for(i = 0 ; i < lemp->nsymbol; i++) { + struct symbol *sp = lemp->symbols[i]; + if (sp == 0 || sp->type == TERMINAL || + sp->index<=0 || sp->destructor!=0) continue; + if (once) { + fprintf(out, " /* Default NON-TERMINAL Destructor */\n"); lineno++; + once = 0; + } + fprintf(out," case %d: /* %s */\n", sp->index, sp->name); lineno++; + dflt_sp = sp; + } + if (dflt_sp!=0) { + emit_destructor_code(out,dflt_sp,lemp,&lineno); + } + fprintf(out," break;\n"); lineno++; + } + for(i = 0 ; i < lemp->nsymbol; i++) { + struct symbol *sp = lemp->symbols[i]; + if (sp == 0 || sp->type == TERMINAL || sp->destructor == 0 || sp->destructor_emitted) continue; + fprintf(out," case %d: /* %s */\n", sp->index, sp->name); lineno++; + sp->destructor_emitted = 1; + + /* Combine duplicate destructors into a single case */ + for(j=i+1; jnsymbol; j++) { + struct symbol *sp2 = lemp->symbols[j]; + if (sp2 && sp2->type!=TERMINAL && sp2->destructor + && sp2->dtnum == sp->dtnum + && strcmp(sp->destructor,sp2->destructor) == 0) { + fprintf(out," case %d: /* %s */\n", + sp2->index, sp2->name); lineno++; + sp2->destructor_emitted = 1; + } + } + + emit_destructor_code(out,lemp->symbols[i],lemp,&lineno); + fprintf(out," break;\n"); lineno++; + } + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate code which executes whenever the parser stack overflows */ + tplt_print(out,lemp,lemp->overflow,&lineno); + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate the table of rule information + ** + ** Note: This code depends on the fact that rules are number + ** sequentually beginning with 0. + */ + for(rp=lemp->rule; rp; rp=rp->next) { + fprintf(out," { %d, %d },\n",rp->lhs->index,rp->nrhs); lineno++; + } + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate code which execution during each REDUCE action */ + i = 0; + for(rp=lemp->rule; rp; rp=rp->next) { + i += translate_code(lemp, rp); + } + if (i) { + fprintf(out," YYMINORTYPE yylhsminor;\n"); lineno++; + } + /* First output rules other than the default: rule */ + for(rp=lemp->rule; rp; rp=rp->next) { + struct rule *rp2; /* Other rules with the same action */ + if (rp->code == 0) continue; + if (rp->code[0] == '\n' + && rp->code[1] == 0 + && rp->codePrefix == 0 + && rp->codeSuffix == 0 + ) { + /* No actions, so this will be part of the "default:" rule */ + continue; + } + fprintf(out," case %d: /* ", rp->iRule); + writeRuleText(out, rp); + fprintf(out, " */\n"); lineno++; + for(rp2=rp->next; rp2; rp2=rp2->next) { + if (rp2->code == rp->code && rp2->codePrefix == rp->codePrefix + && rp2->codeSuffix == rp->codeSuffix) { + fprintf(out," case %d: /* ", rp2->iRule); + writeRuleText(out, rp2); + fprintf(out," */ yytestcase(yyruleno == %d);\n", rp2->iRule); lineno++; + rp2->code = 0; + } + } + emit_code(out,rp,lemp,&lineno); + fprintf(out," break;\n"); lineno++; + rp->code = 0; + } + /* Finally, output the default: rule. We choose as the default: all + ** empty actions. */ + fprintf(out," default:\n"); lineno++; + for(rp=lemp->rule; rp; rp=rp->next) { + if (rp->code == 0) continue; + assert( rp->code[0] == '\n' && rp->code[1] == 0); + assert( rp->codePrefix == 0); + assert( rp->codeSuffix == 0); + fprintf(out," /* (%d) ", rp->iRule); + writeRuleText(out, rp); + fprintf(out, " */ yytestcase(yyruleno == %d);\n", rp->iRule); lineno++; + } + fprintf(out," break;\n"); lineno++; + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate code which executes if a parse fails */ + tplt_print(out,lemp,lemp->failure,&lineno); + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate code which executes when a syntax error occurs */ + tplt_print(out,lemp,lemp->error,&lineno); + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate code which executes when the parser accepts its input */ + tplt_print(out,lemp,lemp->accept,&lineno); + tplt_xfer(lemp->name,in,out,&lineno); + + /* Append any addition code the user desires */ + tplt_print(out,lemp,lemp->extracode,&lineno); + + fclose(in); + fclose(out); + return; +} + +static FILE *header_open(struct lemon *lemp, const char *mode) +{ + if (user_output_header) { + if (lemp->outname) free(lemp->outname); + lemp->outname = strdup(user_output_header); + FILE *fp = fopen(lemp->outname, mode); + if (fp == 0 && *mode == 'w' ) { + fprintf(stderr,"Can't open file \"%s\".\n",lemp->outname); + lemp->errorcnt++; + return 0; + } + return fp; + } else { + return file_open(lemp,".h",mode); + } +} + +/* Generate a header file for the parser */ +void ReportHeader(struct lemon *lemp) +{ + FILE *out, *in; + const char *prefix; + char line[LINESIZE]; + char pattern[LINESIZE]; + int i; + + if (lemp->tokenprefix) prefix = lemp->tokenprefix; + else prefix = ""; + in = header_open(lemp,"rb"); + if (in) { + int nextChar; + for(i = 1; i < lemp->nterminal && fgets(line,LINESIZE,in); i++) { + lemon_sprintf(pattern,"#define %s%-30s %3d\n", + prefix,lemp->symbols[i]->name,i); + if (strcmp(line,pattern) ) break; + } + nextChar = fgetc(in); + fclose(in); + if (i == lemp->nterminal && nextChar == EOF) { + /* No change in the file. Don't rewrite it. */ + return; + } + } + out = header_open(lemp,"wb"); + if (out) { + for(i = 1; i < lemp->nterminal; i++) { + fprintf(out,"#define %s%-30s %3d\n",prefix,lemp->symbols[i]->name,i); + } + fclose(out); + } + return; +} + +/* Reduce the size of the action tables, if possible, by making use +** of defaults. +** +** In this version, we take the most frequent REDUCE action and make +** it the default. Except, there is no default if the wildcard token +** is a possible look-ahead. +*/ +void CompressTables(struct lemon *lemp) +{ + struct state *stp; + struct action *ap, *ap2; + struct rule *rp, *rp2, *rbest; + int nbest, n; + int i; + int usesWildcard; + + for(i = 0 ; i < lemp->nstate; i++) { + stp = lemp->sorted[i]; + nbest = 0; + rbest = 0; + usesWildcard = 0; + + for(ap=stp->ap; ap; ap=ap->next) { + if (ap->type == SHIFT && ap->sp == lemp->wildcard) { + usesWildcard = 1; + } + if (ap->type!=REDUCE) continue; + rp = ap->x.rp; + if (rp->lhsStart) continue; + if (rp == rbest) continue; + n = 1; + for(ap2=ap->next; ap2; ap2=ap2->next) { + if (ap2->type!=REDUCE) continue; + rp2 = ap2->x.rp; + if (rp2==rbest) continue; + if (rp2==rp) n++; + } + if (n>nbest) { + nbest = n; + rbest = rp; + } + } + + /* Do not make a default if the number of rules to default + ** is not at least 1 or if the wildcard token is a possible + ** lookahead. + */ + if (nbest<1 || usesWildcard) continue; + + + /* Combine matching REDUCE actions into a single default */ + for(ap=stp->ap; ap; ap=ap->next) { + if (ap->type == REDUCE && ap->x.rp == rbest) break; + } + assert( ap); + ap->sp = Symbol_new("{default}"); + for(ap=ap->next; ap; ap=ap->next) { + if (ap->type == REDUCE && ap->x.rp == rbest) ap->type = NOT_USED; + } + stp->ap = Action_sort(stp->ap); + + for(ap=stp->ap; ap; ap=ap->next) { + if (ap->type == SHIFT) break; + if (ap->type == REDUCE && ap->x.rp!=rbest) break; + } + if (ap == 0) { + stp->autoReduce = 1; + stp->pDfltReduce = rbest; + } + } + + /* Make a second pass over all states and actions. Convert + ** every action that is a SHIFT to an autoReduce state into + ** a SHIFTREDUCE action. + */ + for(i = 0 ; i < lemp->nstate; i++) { + stp = lemp->sorted[i]; + for(ap=stp->ap; ap; ap=ap->next) { + struct state *pNextState; + if (ap->type!=SHIFT) continue; + pNextState = ap->x.stp; + if (pNextState->autoReduce && pNextState->pDfltReduce!=0) { + ap->type = SHIFTREDUCE; + ap->x.rp = pNextState->pDfltReduce; + } + } + } +} + + +/* +** Compare two states for sorting purposes. The smaller state is the +** one with the most non-terminal actions. If they have the same number +** of non-terminal actions, then the smaller is the one with the most +** token actions. +*/ +static int stateResortCompare(const void *a, const void *b) { + const struct state *pA = *(const struct state **)a; + const struct state *pB = *(const struct state **)b; + int n; + + n = pB->nNtAct - pA->nNtAct; + if (n == 0) { + n = pB->nTknAct - pA->nTknAct; + if (n == 0) { + n = pB->statenum - pA->statenum; + } + } + assert( n!=0); + return n; +} + + +/* +** Renumber and resort states so that states with fewer choices +** occur at the end. Except, keep state 0 as the first state. +*/ +void ResortStates(struct lemon *lemp) +{ + int i; + struct state *stp; + struct action *ap; + + for(i = 0 ; i < lemp->nstate; i++) { + stp = lemp->sorted[i]; + stp->nTknAct = stp->nNtAct = 0; + stp->iDfltReduce = lemp->nrule; /* Init dflt action to "syntax error" */ + stp->iTknOfst = NO_OFFSET; + stp->iNtOfst = NO_OFFSET; + for(ap=stp->ap; ap; ap=ap->next) { + int iAction = compute_action(lemp,ap); + if (iAction>=0) { + if (ap->sp->indexnterminal) { + stp->nTknAct++; + } else if (ap->sp->indexnsymbol) { + stp->nNtAct++; + } else{ + assert( stp->autoReduce == 0 || stp->pDfltReduce == ap->x.rp); + stp->iDfltReduce = iAction - lemp->nstate - lemp->nrule; + } + } + } + } + qsort(&lemp->sorted[1], lemp->nstate-1, sizeof(lemp->sorted[0]), + stateResortCompare); + for(i = 0 ; i < lemp->nstate; i++) { + lemp->sorted[i]->statenum = i; + } + lemp->nxstate = lemp->nstate; + while(lemp->nxstate>1 && lemp->sorted[lemp->nxstate-1]->autoReduce) { + lemp->nxstate--; + } +} + + +/***************** From the file "set.c" ************************************/ +/* +** Set manipulation routines for the LEMON parser generator. +*/ + +static int size = 0; + +/* Set the set size */ +void SetSize(int n) +{ + size = n+1; +} + +/* Allocate a new set */ +char *SetNew() { + char *s; + s = (char *)calloc( size, 1); + if (s == 0) { + extern void memory_error(); + memory_error(); + } + return s; +} + +/* Deallocate a set */ +void SetFree(char *s) +{ + free(s); +} + +/* Add a new element to the set. Return TRUE if the element was added +** and FALSE if it was already there. */ +int SetAdd(char *s, int e) +{ + int rv; + assert( e>=0 && esize = 1024; + x1a->count = 0; + x1a->tbl = (x1node *)calloc(1024, sizeof(x1node) + sizeof(x1node *)); + if (x1a->tbl == 0) { + free(x1a); + x1a = 0; + } else{ + int i; + x1a->ht = (x1node **)&(x1a->tbl[1024]); + for(i = 0 ; i < 1024; i++) x1a->ht[i] = 0; + } + } +} +/* Insert a new record into the array. Return TRUE if successful. +** Prior data with the same key is NOT overwritten */ +int Strsafe_insert(const char *data) +{ + x1node *np; + unsigned h; + unsigned ph; + + if (x1a == 0) return 0; + ph = strhash(data); + h = ph & (x1a->size-1); + np = x1a->ht[h]; + while(np) { + if (strcmp(np->data,data) == 0) { + /* An existing entry with the same key is found. */ + /* Fail because overwrite is not allows. */ + return 0; + } + np = np->next; + } + if (x1a->count>=x1a->size) { + /* Need to make the hash table bigger */ + int i,arrSize; + struct s_x1 array; + array.size = arrSize = x1a->size *2; + array.count = x1a->count; + array.tbl = (x1node *)calloc(arrSize, sizeof(x1node) + sizeof(x1node *)); + if (array.tbl == 0) return 0; /* Fail due to malloc failure */ + array.ht = (x1node **)&(array.tbl[arrSize]); + for(i = 0 ; i < arrSize; i++) array.ht[i] = 0; + for(i = 0 ; i < x1a->count; i++) { + x1node *oldnp, *newnp; + oldnp = &(x1a->tbl[i]); + h = strhash(oldnp->data) & (arrSize-1); + newnp = &(array.tbl[i]); + if (array.ht[h] ) array.ht[h]->from = &(newnp->next); + newnp->next = array.ht[h]; + newnp->data = oldnp->data; + newnp->from = &(array.ht[h]); + array.ht[h] = newnp; + } + free(x1a->tbl); + *x1a = array; + } + /* Insert the new data */ + h = ph & (x1a->size-1); + np = &(x1a->tbl[x1a->count++]); + np->data = data; + if (x1a->ht[h] ) x1a->ht[h]->from = &(np->next); + np->next = x1a->ht[h]; + x1a->ht[h] = np; + np->from = &(x1a->ht[h]); + return 1; +} + +/* Return a pointer to data assigned to the given key. Return NULL +** if no such key. */ +const char *Strsafe_find(const char *key) +{ + unsigned h; + x1node *np; + + if (x1a == 0) return 0; + h = strhash(key) & (x1a->size-1); + np = x1a->ht[h]; + while(np) { + if (strcmp(np->data,key) == 0) break; + np = np->next; + } + return np ? np->data : 0; +} + +/* Return a pointer to the (terminal or nonterminal) symbol "x". +** Create a new symbol if this is the first time "x" has been seen. +*/ +struct symbol *Symbol_new(const char *x) +{ + struct symbol *sp; + + sp = Symbol_find(x); + if (sp == 0) { + sp = (struct symbol *)calloc(1, sizeof(struct symbol) ); + MemoryCheck(sp); + sp->name = Strsafe(x); + sp->type = ISUPPER(*x) ? TERMINAL : NONTERMINAL; + sp->rule = 0; + sp->fallback = 0; + sp->prec = -1; + sp->assoc = UNK; + sp->firstset = 0; + sp->lambda = LEMON_FALSE; + sp->destructor = 0; + sp->destructor_emitted = 0; + sp->destLineno = 0; + sp->datatype = 0; + sp->useCnt = 0; + Symbol_insert(sp,sp->name); + } + sp->useCnt++; + return sp; +} + +/* Compare two symbols for sorting purposes. Return negative, +** zero, or positive if a is less then, equal to, or greater +** than b. +** +** Symbols that begin with upper case letters (terminals or tokens) +** must sort before symbols that begin with lower case letters +** (non-terminals). And MULTITERMINAL symbols (created using the +** %token_class directive) must sort at the very end. Other than +** that, the order does not matter. +** +** We find experimentally that leaving the symbols in their original +** order (the order they appeared in the grammar file) gives the +** smallest parser tables in SQLite. +*/ +int Symbolcmpp(const void *_a, const void *_b) +{ + const struct symbol *a = *(const struct symbol **) _a; + const struct symbol *b = *(const struct symbol **) _b; + int i1 = a->type == MULTITERMINAL ? 3 : a->name[0]>'Z' ? 2 : 1; + int i2 = b->type == MULTITERMINAL ? 3 : b->name[0]>'Z' ? 2 : 1; + return i1==i2 ? a->index - b->index : i1 - i2; +} + +/* There is one instance of the following structure for each +** associative array of type "x2". +*/ +struct s_x2 { + int size; /* The number of available slots. */ + /* Must be a power of 2 greater than or */ + /* equal to 1 */ + int count; /* Number of currently slots filled */ + struct s_x2node *tbl; /* The data stored here */ + struct s_x2node **ht; /* Hash table for lookups */ +}; + +/* There is one instance of this structure for every data element +** in an associative array of type "x2". +*/ +typedef struct s_x2node { + struct symbol *data; /* The data */ + const char *key; /* The key */ + struct s_x2node *next; /* Next entry with the same hash */ + struct s_x2node **from; /* Previous link */ +} x2node; + +/* There is only one instance of the array, which is the following */ +static struct s_x2 *x2a; + +/* Allocate a new associative array */ +void Symbol_init() { + if (x2a) return; + x2a = (struct s_x2*)malloc( sizeof(struct s_x2) ); + if (x2a) { + x2a->size = 128; + x2a->count = 0; + x2a->tbl = (x2node *)calloc(128, sizeof(x2node) + sizeof(x2node *)); + if (x2a->tbl == 0) { + free(x2a); + x2a = 0; + } else{ + int i; + x2a->ht = (x2node **)&(x2a->tbl[128]); + for(i = 0 ; i < 128; i++) x2a->ht[i] = 0; + } + } +} +/* Insert a new record into the array. Return TRUE if successful. +** Prior data with the same key is NOT overwritten */ +int Symbol_insert(struct symbol *data, const char *key) +{ + x2node *np; + unsigned h; + unsigned ph; + + if (x2a == 0) return 0; + ph = strhash(key); + h = ph & (x2a->size-1); + np = x2a->ht[h]; + while(np) { + if (strcmp(np->key,key) == 0) { + /* An existing entry with the same key is found. */ + /* Fail because overwrite is not allows. */ + return 0; + } + np = np->next; + } + if (x2a->count>=x2a->size) { + /* Need to make the hash table bigger */ + int i,arrSize; + struct s_x2 array; + array.size = arrSize = x2a->size *2; + array.count = x2a->count; + array.tbl = (x2node *)calloc(arrSize, sizeof(x2node) + sizeof(x2node *)); + if (array.tbl == 0) return 0; /* Fail due to malloc failure */ + array.ht = (x2node **)&(array.tbl[arrSize]); + for(i = 0 ; i < arrSize; i++) array.ht[i] = 0; + for(i = 0 ; i < x2a->count; i++) { + x2node *oldnp, *newnp; + oldnp = &(x2a->tbl[i]); + h = strhash(oldnp->key) & (arrSize-1); + newnp = &(array.tbl[i]); + if (array.ht[h] ) array.ht[h]->from = &(newnp->next); + newnp->next = array.ht[h]; + newnp->key = oldnp->key; + newnp->data = oldnp->data; + newnp->from = &(array.ht[h]); + array.ht[h] = newnp; + } + free(x2a->tbl); + *x2a = array; + } + /* Insert the new data */ + h = ph & (x2a->size-1); + np = &(x2a->tbl[x2a->count++]); + np->key = key; + np->data = data; + if (x2a->ht[h] ) x2a->ht[h]->from = &(np->next); + np->next = x2a->ht[h]; + x2a->ht[h] = np; + np->from = &(x2a->ht[h]); + return 1; +} + +/* Return a pointer to data assigned to the given key. Return NULL +** if no such key. */ +struct symbol *Symbol_find(const char *key) +{ + unsigned h; + x2node *np; + + if (x2a == 0) return 0; + h = strhash(key) & (x2a->size-1); + np = x2a->ht[h]; + while(np) { + if (strcmp(np->key,key) == 0) break; + np = np->next; + } + return np ? np->data : 0; +} + +/* Return the n-th data. Return NULL if n is out of range. */ +struct symbol *Symbol_Nth(int n) +{ + struct symbol *data; + if (x2a && n>0 && n<=x2a->count) { + data = x2a->tbl[n-1].data; + } else{ + data = 0; + } + return data; +} + +/* Return the size of the array */ +int Symbol_count() +{ + return x2a ? x2a->count : 0; +} + +/* Return an array of pointers to all data in the table. +** The array is obtained from malloc. Return NULL if memory allocation +** problems, or if the array is empty. */ +struct symbol **Symbol_arrayof() +{ + struct symbol **array; + int i,arrSize; + if (x2a == 0) return 0; + arrSize = x2a->count; + array = (struct symbol **)calloc(arrSize, sizeof(struct symbol *)); + if (array) { + for(i = 0 ; i < arrSize; i++) array[i] = x2a->tbl[i].data; + } + return array; +} + +/* Compare two configurations */ +int Configcmp(const char *_a,const char *_b) +{ + const struct config *a = (struct config *) _a; + const struct config *b = (struct config *) _b; + int x; + x = a->rp->index - b->rp->index; + if (x == 0) x = a->dot - b->dot; + return x; +} + +/* Compare two states */ +PRIVATE int statecmp(struct config *a, struct config *b) +{ + int rc; + for(rc=0; rc == 0 && a && b; a=a->bp, b=b->bp) { + rc = a->rp->index - b->rp->index; + if (rc == 0) rc = a->dot - b->dot; + } + if (rc == 0) { + if (a) rc = 1; + if (b) rc = -1; + } + return rc; +} + +/* Hash a state */ +PRIVATE unsigned statehash(struct config *a) +{ + unsigned h=0; + while(a) { + h = h *571 + a->rp->index *37 + a->dot; + a = a->bp; + } + return h; +} + +/* Allocate a new state structure */ +struct state *State_new() +{ + struct state *newstate; + newstate = (struct state *)calloc(1, sizeof(struct state) ); + MemoryCheck(newstate); + return newstate; +} + +/* There is one instance of the following structure for each +** associative array of type "x3". +*/ +struct s_x3 { + int size; /* The number of available slots. */ + /* Must be a power of 2 greater than or */ + /* equal to 1 */ + int count; /* Number of currently slots filled */ + struct s_x3node *tbl; /* The data stored here */ + struct s_x3node **ht; /* Hash table for lookups */ +}; + +/* There is one instance of this structure for every data element +** in an associative array of type "x3". +*/ +typedef struct s_x3node { + struct state *data; /* The data */ + struct config *key; /* The key */ + struct s_x3node *next; /* Next entry with the same hash */ + struct s_x3node **from; /* Previous link */ +} x3node; + +/* There is only one instance of the array, which is the following */ +static struct s_x3 *x3a; + +/* Allocate a new associative array */ +void State_init() { + if (x3a) return; + x3a = (struct s_x3*)malloc( sizeof(struct s_x3) ); + if (x3a) { + x3a->size = 128; + x3a->count = 0; + x3a->tbl = (x3node *)calloc(128, sizeof(x3node) + sizeof(x3node *)); + if (x3a->tbl == 0) { + free(x3a); + x3a = 0; + } else{ + int i; + x3a->ht = (x3node **)&(x3a->tbl[128]); + for(i = 0 ; i < 128; i++) x3a->ht[i] = 0; + } + } +} +/* Insert a new record into the array. Return TRUE if successful. +** Prior data with the same key is NOT overwritten */ +int State_insert(struct state *data, struct config *key) +{ + x3node *np; + unsigned h; + unsigned ph; + + if (x3a == 0) return 0; + ph = statehash(key); + h = ph & (x3a->size-1); + np = x3a->ht[h]; + while(np) { + if (statecmp(np->key,key) == 0) { + /* An existing entry with the same key is found. */ + /* Fail because overwrite is not allows. */ + return 0; + } + np = np->next; + } + if (x3a->count>=x3a->size) { + /* Need to make the hash table bigger */ + int i,arrSize; + struct s_x3 array; + array.size = arrSize = x3a->size *2; + array.count = x3a->count; + array.tbl = (x3node *)calloc(arrSize, sizeof(x3node) + sizeof(x3node *)); + if (array.tbl == 0) return 0; /* Fail due to malloc failure */ + array.ht = (x3node **)&(array.tbl[arrSize]); + for(i = 0 ; i < arrSize; i++) array.ht[i] = 0; + for(i = 0 ; i < x3a->count; i++) { + x3node *oldnp, *newnp; + oldnp = &(x3a->tbl[i]); + h = statehash(oldnp->key) & (arrSize-1); + newnp = &(array.tbl[i]); + if (array.ht[h] ) array.ht[h]->from = &(newnp->next); + newnp->next = array.ht[h]; + newnp->key = oldnp->key; + newnp->data = oldnp->data; + newnp->from = &(array.ht[h]); + array.ht[h] = newnp; + } + free(x3a->tbl); + *x3a = array; + } + /* Insert the new data */ + h = ph & (x3a->size-1); + np = &(x3a->tbl[x3a->count++]); + np->key = key; + np->data = data; + if (x3a->ht[h] ) x3a->ht[h]->from = &(np->next); + np->next = x3a->ht[h]; + x3a->ht[h] = np; + np->from = &(x3a->ht[h]); + return 1; +} + +/* Return a pointer to data assigned to the given key. Return NULL +** if no such key. */ +struct state *State_find(struct config *key) +{ + unsigned h; + x3node *np; + + if (x3a == 0) return 0; + h = statehash(key) & (x3a->size-1); + np = x3a->ht[h]; + while(np) { + if (statecmp(np->key,key) == 0) break; + np = np->next; + } + return np ? np->data : 0; +} + +/* Return an array of pointers to all data in the table. +** The array is obtained from malloc. Return NULL if memory allocation +** problems, or if the array is empty. */ +struct state **State_arrayof() +{ + struct state **array; + int i,arrSize; + if (x3a == 0) return 0; + arrSize = x3a->count; + array = (struct state **)calloc(arrSize, sizeof(struct state *)); + if (array) { + for(i = 0 ; i < arrSize; i++) array[i] = x3a->tbl[i].data; + } + return array; +} + +/* Hash a configuration */ +PRIVATE unsigned confighash(struct config *a) +{ + unsigned h=0; + h = h *571 + a->rp->index *37 + a->dot; + return h; +} + +/* There is one instance of the following structure for each +** associative array of type "x4". +*/ +struct s_x4 { + int size; /* The number of available slots. */ + /* Must be a power of 2 greater than or */ + /* equal to 1 */ + int count; /* Number of currently slots filled */ + struct s_x4node *tbl; /* The data stored here */ + struct s_x4node **ht; /* Hash table for lookups */ +}; + +/* There is one instance of this structure for every data element +** in an associative array of type "x4". +*/ +typedef struct s_x4node { + struct config *data; /* The data */ + struct s_x4node *next; /* Next entry with the same hash */ + struct s_x4node **from; /* Previous link */ +} x4node; + +/* There is only one instance of the array, which is the following */ +static struct s_x4 *x4a; + +/* Allocate a new associative array */ +void Configtable_init() { + if (x4a) return; + x4a = (struct s_x4*)malloc( sizeof(struct s_x4) ); + if (x4a) { + x4a->size = 64; + x4a->count = 0; + x4a->tbl = (x4node *)calloc(64, sizeof(x4node) + sizeof(x4node *)); + if (x4a->tbl == 0) { + free(x4a); + x4a = 0; + } else{ + int i; + x4a->ht = (x4node **)&(x4a->tbl[64]); + for(i = 0 ; i < 64; i++) x4a->ht[i] = 0; + } + } +} +/* Insert a new record into the array. Return TRUE if successful. +** Prior data with the same key is NOT overwritten */ +int Configtable_insert(struct config *data) +{ + x4node *np; + unsigned h; + unsigned ph; + + if (x4a == 0) return 0; + ph = confighash(data); + h = ph & (x4a->size-1); + np = x4a->ht[h]; + while(np) { + if (Configcmp((const char *) np->data,(const char *) data) == 0) { + /* An existing entry with the same key is found. */ + /* Fail because overwrite is not allows. */ + return 0; + } + np = np->next; + } + if (x4a->count>=x4a->size) { + /* Need to make the hash table bigger */ + int i,arrSize; + struct s_x4 array; + array.size = arrSize = x4a->size *2; + array.count = x4a->count; + array.tbl = (x4node *)calloc(arrSize, sizeof(x4node) + sizeof(x4node *)); + if (array.tbl == 0) return 0; /* Fail due to malloc failure */ + array.ht = (x4node **)&(array.tbl[arrSize]); + for(i = 0 ; i < arrSize; i++) array.ht[i] = 0; + for(i = 0 ; i < x4a->count; i++) { + x4node *oldnp, *newnp; + oldnp = &(x4a->tbl[i]); + h = confighash(oldnp->data) & (arrSize-1); + newnp = &(array.tbl[i]); + if (array.ht[h] ) array.ht[h]->from = &(newnp->next); + newnp->next = array.ht[h]; + newnp->data = oldnp->data; + newnp->from = &(array.ht[h]); + array.ht[h] = newnp; + } + free(x4a->tbl); + *x4a = array; + } + /* Insert the new data */ + h = ph & (x4a->size-1); + np = &(x4a->tbl[x4a->count++]); + np->data = data; + if (x4a->ht[h] ) x4a->ht[h]->from = &(np->next); + np->next = x4a->ht[h]; + x4a->ht[h] = np; + np->from = &(x4a->ht[h]); + return 1; +} + +/* Return a pointer to data assigned to the given key. Return NULL +** if no such key. */ +struct config *Configtable_find(struct config *key) +{ + int h; + x4node *np; + + if (x4a == 0) return 0; + h = confighash(key) & (x4a->size-1); + np = x4a->ht[h]; + while(np) { + if (Configcmp((const char *) np->data,(const char *) key) == 0) break; + np = np->next; + } + return np ? np->data : 0; +} + +/* Remove all data from the table. Pass each data to the function "f" +** as it is removed. ("f" may be null to avoid this step.) */ +void Configtable_clear(int(*f)(struct config *)) +{ + int i; + if (x4a == 0 || x4a->count == 0) return; + if (f) for(i = 0 ; i < x4a->count; i++) (*f)(x4a->tbl[i].data); + for(i = 0 ; i < x4a->size; i++) x4a->ht[i] = 0; + x4a->count = 0; + return; +} diff --git a/tools/lempar.c b/tools/lempar.c new file mode 100644 index 0000000..73a7b16 --- /dev/null +++ b/tools/lempar.c @@ -0,0 +1,929 @@ +/* +** 2000-05-29 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Driver template for the LEMON parser generator. +** +** The "lemon" program processes an LALR(1) input grammar file, then uses +** this template to construct a parser. The "lemon" program inserts text +** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the +** interstitial "-" characters) contained in this template is changed into +** the value of the %name directive from the grammar. Otherwise, the content +** of this template is copied straight through into the generate parser +** source file. +** +** The following is the concatenation of all %include directives from the +** input grammar file: +*/ +#include +/************ Begin %include sections from the grammar ************************/ +%% +/**************** End of %include directives **********************************/ +/* These constants specify the various numeric values for terminal symbols +** in a format understandable to "makeheaders". This section is blank unless +** "lemon" is run with the "-m" command-line option. +***************** Begin makeheaders token definitions *************************/ +%% +/**************** End makeheaders token definitions ***************************/ + +/* The next sections is a series of control #defines. +** various aspects of the generated parser. +** YYCODETYPE is the data type used to store the integer codes +** that represent terminal and non-terminal symbols. +** "unsigned char" is used if there are fewer than +** 256 symbols. Larger types otherwise. +** YYNOCODE is a number of type YYCODETYPE that is not used for +** any terminal or nonterminal symbol. +** YYFALLBACK If defined, this indicates that one or more tokens +** (also known as: "terminal symbols") have fall-back +** values which should be used if the original symbol +** would not parse. This permits keywords to sometimes +** be used as identifiers, for example. +** YYACTIONTYPE is the data type used for "action codes" - numbers +** that indicate what to do in response to the next +** token. +** ParseTOKENTYPE is the data type used for minor type for terminal +** symbols. Background: A "minor type" is a semantic +** value associated with a terminal or non-terminal +** symbols. For example, for an "ID" terminal symbol, +** the minor type might be the name of the identifier. +** Each non-terminal can have a different minor type. +** Terminal symbols all have the same minor type, though. +** This macros defines the minor type for terminal +** symbols. +** YYMINORTYPE is the data type used for all minor types. +** This is typically a union of many types, one of +** which is ParseTOKENTYPE. The entry in the union +** for terminal symbols is called "yy0". +** YYSTACKDEPTH is the maximum depth of the parser's stack. If +** zero the stack is dynamically sized using realloc() +** ParseARG_SDECL A static variable declaration for the %extra_argument +** ParseARG_PDECL A parameter declaration for the %extra_argument +** ParseARG_STORE Code to store %extra_argument into yypParser +** ParseARG_FETCH Code to extract %extra_argument from yypParser +** YYERRORSYMBOL is the code number of the error symbol. If not +** defined, then do no error processing. +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YY_MAX_SHIFT Maximum value for shift actions +** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions +** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions +** YY_MIN_REDUCE Maximum value for reduce actions +** YY_ERROR_ACTION The yy_action[] code for syntax error +** YY_ACCEPT_ACTION The yy_action[] code for accept +** YY_NO_ACTION The yy_action[] code for no-op +*/ +#ifndef INTERFACE +# define INTERFACE 1 +#endif +/************* Begin control #defines *****************************************/ +%% +/************* End control #defines *******************************************/ + +/* Define the yytestcase() macro to be a no-op if is not already defined +** otherwise. +** +** Applications can choose to define yytestcase() in the %include section +** to a macro that can assist in verifying code coverage. For production +** code the yytestcase() macro should be turned off. But it is useful +** for testing. +*/ +#ifndef yytestcase +# define yytestcase(X) +#endif + + +/* Next are the tables used to determine what action to take based on the +** current state and lookahead token. These tables are used to implement +** functions that take a state number and lookahead value and return an +** action integer. +** +** Suppose the action integer is N. Then the action is determined as +** follows +** +** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead +** token onto the stack and goto state N. +** +** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then +** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. +** +** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE +** and YY_MAX_REDUCE + +** N == YY_ERROR_ACTION A syntax error has occurred. +** +** N == YY_ACCEPT_ACTION The parser accepts its input. +** +** N == YY_NO_ACTION No such action. Denotes unused +** slots in the yy_action[] table. +** +** The action table is constructed as a single large table named yy_action[]. +** Given state S and lookahead X, the action is computed as +** +** yy_action[ yy_shift_ofst[S] + X ] +** +** If the index value yy_shift_ofst[S]+X is out of range or if the value +** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X or if yy_shift_ofst[S] +** is equal to YY_SHIFT_USE_DFLT, it means that the action is not in the table +** and that yy_default[S] should be used instead. +** +** The formula above is for computing the action when the lookahead is +** a terminal symbol. If the lookahead is a non-terminal (as occurs after +** a reduce action) then the yy_reduce_ofst[] array is used in place of +** the yy_shift_ofst[] array and YY_REDUCE_USE_DFLT is used in place of +** YY_SHIFT_USE_DFLT. +** +** The following are the tables generated in this section: +** +** yy_action[] A single table containing all actions. +** yy_lookahead[] A table containing the lookahead for each entry in +** yy_action. Used to detect hash collisions. +** yy_shift_ofst[] For each state, the offset into yy_action for +** shifting terminals. +** yy_reduce_ofst[] For each state, the offset into yy_action for +** shifting non-terminals after a reduce. +** yy_default[] Default action for each state. +** +*********** Begin parsing tables **********************************************/ +%% +/********** End of lemon-generated parsing tables *****************************/ + +/* The next table maps tokens (terminal symbols) into fallback tokens. +** If a construct like the following: +** +** %fallback ID X Y Z. +** +** appears in the grammar, then ID becomes a fallback token for X, Y, +** and Z. Whenever one of the tokens X, Y, or Z is input to the parser +** but it does not parse, the type of the token is changed to ID and +** the parse is retried before an error is thrown. +** +** This feature can be used, for example, to cause some keywords in a language +** to revert to identifiers if they keyword does not apply in the context where +** it appears. +*/ +#ifdef YYFALLBACK +static const YYCODETYPE yyFallback[] = { +%% +}; +#endif /* YYFALLBACK */ + +/* The following structure represents a single element of the +** parser's stack. Information stored includes: +** +** + The state number for the parser at this level of the stack. +** +** + The value of the token stored at this level of the stack. +** (In other words, the "major" token.) +** +** + The semantic value stored at this level of the stack. This is +** the information used by the action routines in the grammar. +** It is sometimes called the "minor" token. +** +** After the "shift" half of a SHIFTREDUCE action, the stateno field +** actually contains the reduce action for the second half of the +** SHIFTREDUCE. +*/ +struct yyStackEntry { + YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ + YYCODETYPE major; /* The major token value. This is the code + ** number for the token at this stack level */ + YYMINORTYPE minor; /* The user-supplied minor token value. This + ** is the value of the token */ +}; +typedef struct yyStackEntry yyStackEntry; + +/* The state of the parser is completely contained in an instance of +** the following structure */ +struct yyParser { + int yyidx; /* Index of top element in stack */ +#ifdef YYTRACKMAXSTACKDEPTH + int yyidxMax; /* Maximum value of yyidx */ +#endif +#ifndef YYNOERRORRECOVERY + int yyerrcnt; /* Shifts left before out of the error */ +#endif + ParseARG_SDECL /* A place to hold %extra_argument */ +#if YYSTACKDEPTH<=0 + int yystksz; /* Current side of the stack */ + yyStackEntry *yystack; /* The parser's stack */ +#else + yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ +#endif +}; +typedef struct yyParser yyParser; + +#ifndef NDEBUG +#include +static FILE *yyTraceFILE = 0; +static char *yyTracePrompt = 0; +#endif /* NDEBUG */ + +#ifndef NDEBUG +/* +** Turn parser tracing on by giving a stream to which to write the trace +** and a prompt to preface each trace message. Tracing is turned off +** by making either argument NULL +** +** Inputs: +**
    +**
  • A FILE *to which trace output should be written. +** If NULL, then tracing is turned off. +**
  • A prefix string written at the beginning of every +** line of trace output. If NULL, then tracing is +** turned off. +**
+** +** Outputs: +** None. +*/ +void ParseTrace(FILE *TraceFILE, char *zTracePrompt) { + yyTraceFILE = TraceFILE; + yyTracePrompt = zTracePrompt; + if (yyTraceFILE == 0) yyTracePrompt = 0; + else if (yyTracePrompt == 0) yyTraceFILE = 0; +} +#endif /* NDEBUG */ + +#ifndef NDEBUG +/* For tracing shifts, the names of all terminals and nonterminals +** are required. The following table supplies these names */ +static const char *const yyTokenName[] = { +%% +}; +#endif /* NDEBUG */ + +#ifndef NDEBUG +/* For tracing reduce actions, the names of all rules are required. +*/ +static const char *const yyRuleName[] = { +%% +}; +#endif /* NDEBUG */ + + +#if YYSTACKDEPTH<=0 +/* +** Try to increase the size of the parser stack. +*/ +static void yyGrowStack(yyParser *p) { + int newSize; + yyStackEntry *pNew; + + newSize = p->yystksz *2 + 100; + pNew = realloc(p->yystack, newSize *sizeof(pNew[0])); + if (pNew) { + p->yystack = pNew; + p->yystksz = newSize; +#ifndef NDEBUG + if (yyTraceFILE) { + fprintf(yyTraceFILE,"%sStack grows to %d entries!\n", + yyTracePrompt, p->yystksz); + } +#endif + } +} +#endif + +/* Datatype of the argument to the memory allocated passed as the +** second argument to ParseAlloc() below. This can be changed by +** putting an appropriate #define in the %include section of the input +** grammar. +*/ +#ifndef YYMALLOCARGTYPE +# define YYMALLOCARGTYPE size_t +#endif + +/* +** This function allocates a new parser. +** The only argument is a pointer to a function which works like +** malloc. +** +** Inputs: +** A pointer to the function used to allocate memory. +** +** Outputs: +** A pointer to a parser. This pointer is used in subsequent calls +** to Parse and ParseFree. +*/ +void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)) { + yyParser *pParser; + pParser = (yyParser *)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); + if (pParser) { + pParser->yyidx = -1; +#ifdef YYTRACKMAXSTACKDEPTH + pParser->yyidxMax = 0; +#endif +#if YYSTACKDEPTH<=0 + pParser->yystack = NULL; + pParser->yystksz = 0; + yyGrowStack(pParser); +#endif + } + return pParser; +} + +/* The following function deletes the "minor type" or semantic value +** associated with a symbol. The symbol can be either a terminal +** or nonterminal. "yymajor" is the symbol code, and "yypminor" is +** a pointer to the value to be deleted. The code used to do the +** deletions is derived from the %destructor and/or %token_destructor +** directives of the input grammar. +*/ +static void yy_destructor( + yyParser *yypParser, /* The parser */ + YYCODETYPE yymajor, /* Type code for object to destroy */ + YYMINORTYPE *yypminor /* The object to be destroyed */ +) { + ParseARG_FETCH; + switch(yymajor) { + /* Here is inserted the actions which take place when a + ** terminal or non-terminal is destroyed. This can happen + ** when the symbol is popped from the stack during a + ** reduce or during error processing or when a parser is + ** being destroyed before it is finished parsing. + ** + ** Note: during a reduce, the only symbols destroyed are those + ** which appear on the RHS of the rule, but which are *not *used + ** inside the C code. + */ +/********* Begin destructor definitions ***************************************/ +%% +/********* End destructor definitions *****************************************/ + default: break; /* If no destructor action specified: do nothing */ + } +} + +/* +** Pop the parser's stack once. +** +** If there is a destructor routine associated with the token which +** is popped from the stack, then call it. +*/ +static void yy_pop_parser_stack(yyParser *pParser) { + yyStackEntry *yytos; + assert( pParser->yyidx>=0); + yytos = &pParser->yystack[pParser->yyidx--]; +#ifndef NDEBUG + if (yyTraceFILE) { + fprintf(yyTraceFILE,"%sPopping %s\n", + yyTracePrompt, + yyTokenName[yytos->major]); + } +#endif + yy_destructor(pParser, yytos->major, &yytos->minor); +} + +/* +** Deallocate and destroy a parser. Destructors are called for +** all stack elements before shutting the parser down. +** +** If the YYPARSEFREENEVERNULL macro exists (for example because it +** is defined in a %include section of the input grammar) then it is +** assumed that the input pointer is never NULL. +*/ +void ParseFree( + void *p, /* The parser to be deleted */ + void (*freeProc)(void *) /* Function used to reclaim memory */ +) { + yyParser *pParser = (yyParser *)p; +#ifndef YYPARSEFREENEVERNULL + if (pParser == 0) return; +#endif + while(pParser->yyidx>=0) yy_pop_parser_stack(pParser); +#if YYSTACKDEPTH<=0 + free(pParser->yystack); +#endif + (*freeProc)((void *)pParser); +} + +/* +** Return the peak depth of the stack for a parser. +*/ +#ifdef YYTRACKMAXSTACKDEPTH +int ParseStackPeak(void *p) { + yyParser *pParser = (yyParser *)p; + return pParser->yyidxMax; +} +#endif + +/* +** Find the appropriate action for a parser given the terminal +** look-ahead token iLookAhead. +*/ +static unsigned int yy_find_shift_action( + yyParser *pParser, /* The parser */ + YYCODETYPE iLookAhead /* The look-ahead token */ +) { + int i; + int stateno = pParser->yystack[pParser->yyidx].stateno; + + if (stateno>=YY_MIN_REDUCE) return stateno; + assert( stateno <= YY_SHIFT_COUNT); + do{ + i = yy_shift_ofst[stateno]; + if (i == YY_SHIFT_USE_DFLT) return yy_default[stateno]; + assert( iLookAhead!=YYNOCODE); + i += iLookAhead; + if (i < 0 || i >= YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead) { + if (iLookAhead>0) { +#ifdef YYFALLBACK + YYCODETYPE iFallback; /* Fallback token */ + if (iLookAhead %s\n", + yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); + } +#endif + assert( yyFallback[iFallback] == 0); /* Fallback loop must terminate */ + iLookAhead = iFallback; + continue; + } +#endif +#ifdef YYWILDCARD + { + int j = i - iLookAhead + YYWILDCARD; + if ( +#if YY_SHIFT_MIN+YYWILDCARD<0 + j>=0 && +#endif +#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT + j %s\n", + yyTracePrompt, yyTokenName[iLookAhead], + yyTokenName[YYWILDCARD]); + } +#endif /* NDEBUG */ + return yy_action[j]; + } + } +#endif /* YYWILDCARD */ + } + return yy_default[stateno]; + } else{ + return yy_action[i]; + } + }while(1); +} + +/* +** Find the appropriate action for a parser given the non-terminal +** look-ahead token iLookAhead. +*/ +static int yy_find_reduce_action( + int stateno, /* Current state number */ + YYCODETYPE iLookAhead /* The look-ahead token */ +) { + int i; +#ifdef YYERRORSYMBOL + if (stateno>YY_REDUCE_COUNT) { + return yy_default[stateno]; + } +#else + assert( stateno<=YY_REDUCE_COUNT); +#endif + i = yy_reduce_ofst[stateno]; + assert( i!=YY_REDUCE_USE_DFLT); + assert( iLookAhead!=YYNOCODE); + i += iLookAhead; +#ifdef YYERRORSYMBOL + if (i < 0 || i >= YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead) { + return yy_default[stateno]; + } +#else + assert( i >= 0 && i < YY_ACTTAB_COUNT); + assert( yy_lookahead[i] == iLookAhead); +#endif + return yy_action[i]; +} + +/* +** The following routine is called if the stack overflows. +*/ +static void yyStackOverflow(yyParser *yypParser) { + ParseARG_FETCH; + yypParser->yyidx--; +#ifndef NDEBUG + if (yyTraceFILE) { + fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt); + } +#endif + while(yypParser->yyidx>=0) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will execute if the parser + ** stack every overflows */ +/******** Begin %stack_overflow code ******************************************/ +%% +/******** End %stack_overflow code ********************************************/ + ParseARG_STORE; /* Suppress warning about unused %extra_argument var */ +} + +/* +** Print tracing information for a SHIFT action +*/ +#ifndef NDEBUG +static void yyTraceShift(yyParser *yypParser, int yyNewState) { + if (yyTraceFILE) { + if (yyNewStateyystack[yypParser->yyidx].major], + yyNewState); + } else{ + fprintf(yyTraceFILE,"%sShift '%s'\n", + yyTracePrompt,yyTokenName[yypParser->yystack[yypParser->yyidx].major]); + } + } +} +#else +# define yyTraceShift(X,Y) +#endif + +/* +** Perform a shift action. +*/ +static void yy_shift( + yyParser *yypParser, /* The parser to be shifted */ + int yyNewState, /* The new state to shift in */ + int yyMajor, /* The major token to shift in */ + ParseTOKENTYPE yyMinor /* The minor token to shift in */ +) { + yyStackEntry *yytos; + yypParser->yyidx++; +#ifdef YYTRACKMAXSTACKDEPTH + if (yypParser->yyidx>yypParser->yyidxMax) { + yypParser->yyidxMax = yypParser->yyidx; + } +#endif +#if YYSTACKDEPTH>0 + if (yypParser->yyidx>=YYSTACKDEPTH) { + yyStackOverflow(yypParser); + return; + } +#else + if (yypParser->yyidx>=yypParser->yystksz) { + yyGrowStack(yypParser); + if (yypParser->yyidx>=yypParser->yystksz) { + yyStackOverflow(yypParser); + return; + } + } +#endif + yytos = &yypParser->yystack[yypParser->yyidx]; + yytos->stateno = (YYACTIONTYPE)yyNewState; + yytos->major = (YYCODETYPE)yyMajor; + yytos->minor.yy0 = yyMinor; + yyTraceShift(yypParser, yyNewState); +} + +/* The following table contains information about every rule that +** is used during the reduce. +*/ +static const struct { + YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ + unsigned char nrhs; /* Number of right-hand side symbols in the rule */ +} yyRuleInfo[] = { +%% +}; + +static void yy_accept(yyParser *); /* Forward Declaration */ + +/* +** Perform a reduce action and the shift that must immediately +** follow the reduce. +*/ +static void yy_reduce( + yyParser *yypParser, /* The parser */ + unsigned int yyruleno /* Number of the rule by which to reduce */ +) { + int yygoto; /* The next state */ + int yyact; /* The next action */ + yyStackEntry *yymsp; /* The top of the parser's stack */ + int yysize; /* Amount to pop the stack */ + ParseARG_FETCH; + yymsp = &yypParser->yystack[yypParser->yyidx]; +#ifndef NDEBUG + if (yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ) { + yysize = yyRuleInfo[yyruleno].nrhs; + fprintf(yyTraceFILE, "%sReduce [%s], go to state %d.\n", yyTracePrompt, + yyRuleName[yyruleno], yymsp[-yysize].stateno); + } +#endif /* NDEBUG */ + + /* Check that the stack is large enough to grow by a single entry + ** if the RHS of the rule is empty. This ensures that there is room + ** enough on the stack to push the LHS value */ + if (yyRuleInfo[yyruleno].nrhs == 0) { +#ifdef YYTRACKMAXSTACKDEPTH + if (yypParser->yyidx>yypParser->yyidxMax) { + yypParser->yyidxMax = yypParser->yyidx; + } +#endif +#if YYSTACKDEPTH>0 + if (yypParser->yyidx>=YYSTACKDEPTH-1) { + yyStackOverflow(yypParser); + return; + } +#else + if (yypParser->yyidx>=yypParser->yystksz-1) { + yyGrowStack(yypParser); + if (yypParser->yyidx>=yypParser->yystksz-1) { + yyStackOverflow(yypParser); + return; + } + } +#endif + } + + switch(yyruleno) { + /* Beginning here are the reduction cases. A typical example + ** follows: + ** case 0: + ** #line + ** { ... } // User supplied code + ** #line + ** break; + */ +/********** Begin reduce actions **********************************************/ +%% +/********** End reduce actions ************************************************/ + }; + assert( yyrulenoYY_MAX_SHIFT) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + yypParser->yyidx -= yysize - 1; + yymsp -= yysize-1; + yymsp->stateno = (YYACTIONTYPE)yyact; + yymsp->major = (YYCODETYPE)yygoto; + yyTraceShift(yypParser, yyact); + } else{ + assert( yyact == YY_ACCEPT_ACTION); + yypParser->yyidx -= yysize; + yy_accept(yypParser); + } +} + +/* +** The following code executes when the parse fails +*/ +#ifndef YYNOERRORRECOVERY +static void yy_parse_failed( + yyParser *yypParser /* The parser */ +) { + ParseARG_FETCH; +#ifndef NDEBUG + if (yyTraceFILE) { + fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); + } +#endif + while(yypParser->yyidx>=0) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will be executed whenever the + ** parser fails */ +/************ Begin %parse_failure code ***************************************/ +%% +/************ End %parse_failure code *****************************************/ + ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ +} +#endif /* YYNOERRORRECOVERY */ + +/* +** The following code executes when a syntax error first occurs. +*/ +static void yy_syntax_error( + yyParser *yypParser, /* The parser */ + int yymajor, /* The major type of the error token */ + ParseTOKENTYPE yyminor /* The minor type of the error token */ +) { + ParseARG_FETCH; +#define TOKEN yyminor +/************ Begin %syntax_error code ****************************************/ +%% +/************ End %syntax_error code ******************************************/ + ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ +} + +/* +** The following is executed when the parser accepts +*/ +static void yy_accept( + yyParser *yypParser /* The parser */ +) { + ParseARG_FETCH; +#ifndef NDEBUG + if (yyTraceFILE) { + fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); + } +#endif + while(yypParser->yyidx>=0) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will be executed whenever the + ** parser accepts */ +/*********** Begin %parse_accept code *****************************************/ +%% +/*********** End %parse_accept code *******************************************/ + ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ +} + +/* The main parser program. +** The first argument is a pointer to a structure obtained from +** "ParseAlloc" which describes the current state of the parser. +** The second argument is the major token number. The third is +** the minor token. The fourth optional argument is whatever the +** user wants (and specified in the grammar) and is available for +** use by the action routines. +** +** Inputs: +**
    +**
  • A pointer to the parser (an opaque structure.) +**
  • The major token number. +**
  • The minor token number. +**
  • An option argument of a grammar-specified type. +**
+** +** Outputs: +** None. +*/ +void Parse( + void *yyp, /* The parser */ + int yymajor, /* The major token code number */ + ParseTOKENTYPE yyminor /* The value for the token */ + ParseARG_PDECL /* Optional %extra_argument parameter */ +) { + YYMINORTYPE yyminorunion; + unsigned int yyact; /* The parser action. */ +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + int yyendofinput; /* True if we are at the end of input */ +#endif +#ifdef YYERRORSYMBOL + int yyerrorhit = 0; /* True if yymajor has invoked an error */ +#endif + yyParser *yypParser; /* The parser */ + + /* (re)initialize the parser, if necessary */ + yypParser = (yyParser *)yyp; + if (yypParser->yyidx<0) { +#if YYSTACKDEPTH<=0 + if (yypParser->yystksz <=0) { + yyStackOverflow(yypParser); + return; + } +#endif + yypParser->yyidx = 0; +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + yypParser->yystack[0].stateno = 0; + yypParser->yystack[0].major = 0; +#ifndef NDEBUG + if (yyTraceFILE) { + fprintf(yyTraceFILE,"%sInitialize. Empty stack. State 0\n", + yyTracePrompt); + } +#endif + } +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + yyendofinput = (yymajor == 0); +#endif + ParseARG_STORE; + +#ifndef NDEBUG + if (yyTraceFILE) { + fprintf(yyTraceFILE,"%sInput '%s'\n",yyTracePrompt,yyTokenName[yymajor]); + } +#endif + + do{ + yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); + if (yyact <= YY_MAX_SHIFTREDUCE) { + if (yyact > YY_MAX_SHIFT) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + yy_shift(yypParser,yyact,yymajor,yyminor); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt--; +#endif + yymajor = YYNOCODE; + } else if (yyact <= YY_MAX_REDUCE) { + yy_reduce(yypParser,yyact-YY_MIN_REDUCE); + } else{ + assert( yyact == YY_ERROR_ACTION); + yyminorunion.yy0 = yyminor; +#ifdef YYERRORSYMBOL + int yymx; +#endif +#ifndef NDEBUG + if (yyTraceFILE) { + fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt); + } +#endif +#ifdef YYERRORSYMBOL + /* A syntax error has occurred. + ** The response to an error depends upon whether or not the + ** grammar defines an error token "ERROR". + ** + ** This is what we do if the grammar does define ERROR: + ** + ** * Call the %syntax_error function. + ** + ** * Begin popping the stack until we enter a state where + ** it is legal to shift the error symbol, then shift + ** the error symbol. + ** + ** * Set the error count to three. + ** + ** * Begin accepting and shifting new tokens. No new error + ** processing will occur until three tokens have been + ** shifted successfully. + ** + */ + if (yypParser->yyerrcnt<0) { + yy_syntax_error(yypParser,yymajor,yyminor); + } + yymx = yypParser->yystack[yypParser->yyidx].major; + if (yymx == YYERRORSYMBOL || yyerrorhit) { +#ifndef NDEBUG + if (yyTraceFILE) { + fprintf(yyTraceFILE,"%sDiscard input token %s\n", + yyTracePrompt,yyTokenName[yymajor]); + } +#endif + yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion); + yymajor = YYNOCODE; + } else{ + while( + yypParser->yyidx >= 0 && + yymx != YYERRORSYMBOL && + (yyact = yy_find_reduce_action( + yypParser->yystack[yypParser->yyidx].stateno, + YYERRORSYMBOL)) >= YY_MIN_REDUCE + ) { + yy_pop_parser_stack(yypParser); + } + if (yypParser->yyidx < 0 || yymajor == 0) { + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + yy_parse_failed(yypParser); + yymajor = YYNOCODE; + } else if (yymx!=YYERRORSYMBOL) { + yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor); + } + } + yypParser->yyerrcnt = 3; + yyerrorhit = 1; +#elif defined(YYNOERRORRECOVERY) + /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to + ** do any kind of error recovery. Instead, simply invoke the syntax + ** error routine and continue going as if nothing had happened. + ** + ** Applications can set this macro (for example inside %include) if + ** they intend to abandon the parse upon the first syntax error seen. + */ + yy_syntax_error(yypParser,yymajor, yyminor); + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + yymajor = YYNOCODE; + +#else /* YYERRORSYMBOL is not defined */ + /* This is what we do if the grammar does not define ERROR: + ** + ** * Report an error message, and throw away the input token. + ** + ** * If the input token is $, then fail the parse. + ** + ** As before, subsequent error messages are suppressed until + ** three input tokens have been successfully shifted. + */ + if (yypParser->yyerrcnt<=0) { + yy_syntax_error(yypParser,yymajor, yyminor); + } + yypParser->yyerrcnt = 3; + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + if (yyendofinput) { + yy_parse_failed(yypParser); + } + yymajor = YYNOCODE; +#endif + } + }while(yymajor!=YYNOCODE && yypParser->yyidx>=0); +#ifndef NDEBUG + if (yyTraceFILE) { + int i; + fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt); + for(i = 1; i <= yypParser->yyidx; i++) + fprintf(yyTraceFILE,"%c%s", i == 1 ? '[' : ' ', + yyTokenName[yypParser->yystack[i].major]); + fprintf(yyTraceFILE,"]\n"); + } +#endif + return; +}